Mario de Frutos
GitHub
commit
79c450f63f
6
.github/PULL_REQUEST_TEMPLATE.md
vendored
6
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -2,6 +2,12 @@
|
||||
|
||||
I'd like to request a new data observatory extension deploy: dump + extension
|
||||
|
||||
## Performance comparison to last deployment
|
||||
|
||||
Please include link here to comparison perftests:
|
||||
|
||||
http://52.71.151.140/perftest/#oldsha..newsha
|
||||
|
||||
## Dump database id to be deployed
|
||||
|
||||
Please put here the dump id to be deployed: <dump_id>
|
||||
|
||||
66
NEWS.md
66
NEWS.md
@ -1,3 +1,69 @@
|
||||
1.3.0 (2017-01-17)
|
||||
|
||||
__API Changes__
|
||||
|
||||
* `OBS_GetMeasureDataMulti()` is now called `OBS_GetData()`
|
||||
* `OBS_GetMeasureMetaMulti()` is now called `OBS_GetMeta()`
|
||||
* Additional signature for `OBS_GetData` which can take an array of `TEXT`,
|
||||
mimicking functionality of `OBS_GetMeasureByID`
|
||||
|
||||
__Improvements__
|
||||
|
||||
* Generate fixtures from `obs_meta`
|
||||
* Remove unused table-level code
|
||||
* Refactor all augmentation and geometry functions to obtain data from
|
||||
`OBS_GetMeta()` and `OBS_GetData()`.
|
||||
* Improvements to `OBS_GetMeta()` so it can still fill in metadata in cases
|
||||
where only a geometry is being requested.
|
||||
* `OBS_GetData()` returns two-column table instead of anonymous record.
|
||||
* `OBS_GetData()` can return categorical (text) and geometries
|
||||
|
||||
__Bugfixes__
|
||||
|
||||
* Remove unnecessary dependency on `postgres_fdw`
|
||||
* `OBS_GetData()` now aggregates measures with mixed geoms correctly
|
||||
|
||||
__API Changes__
|
||||
|
||||
1.2.1 (2017-01-17)
|
||||
|
||||
__Improvements__
|
||||
|
||||
* Support Point/LineString in responses from `OBS_GetBoundary`.
|
||||
([#243](https://github.com/CartoDB/observatory-extension/pull/233))
|
||||
|
||||
1.2.0 (2016-12-28)
|
||||
|
||||
__API Changes__
|
||||
|
||||
* Added `OBS_GetMeasureDataMulti`, which takes an array of geomvals and
|
||||
parameters as JSON, and returns a set of RECORDs keyed by the vals of the
|
||||
geomvals.
|
||||
* Added `OBS_GetMeasureMetaMulti`, which takes sparse metadata as JSON (for
|
||||
example, the measure ID) and returns a filled-out version of the metadata
|
||||
sufficient for use with `OBS_GetMeasureDataMulti`.
|
||||
|
||||
__Improvements__
|
||||
|
||||
* Move tests to 2015
|
||||
* Fixes to `_OBS_GetGeometryScores` to avoid spamming NOTICEs about all pixels
|
||||
for a band being NULL
|
||||
* Tests for `_OBS_GetGeometryScores` with complex geometries
|
||||
* Performance tests for `OBS_GetMeasureDataMulti`
|
||||
* Return both `table_id` and `column_id` from `_OBS_GetGeometryScores`
|
||||
|
||||
1.1.7 (2016-12-15)
|
||||
|
||||
__Improvements__
|
||||
|
||||
* Use simpler raster table and simplified `_OBSGetGeometryScores` functions to
|
||||
improve performance
|
||||
* In cases where geometry passed into geometry scoring function has greater
|
||||
than 10K points, simply use its buffer instead
|
||||
* Add `IMMUTABLE` to `_OBSGetGeometryScores`
|
||||
* Add tests explicitly for `_OBSGetGeometryScores` in perftest.py
|
||||
* Yields a ~50% improvement in performance for `_OBSGetGeomeryScores`.
|
||||
|
||||
1.1.6 (2016-12-08)
|
||||
|
||||
__Bugfixes__
|
||||
|
||||
2121
release/observatory--1.3.0.sql
Normal file
2121
release/observatory--1.3.0.sql
Normal file
File diff suppressed because one or more lines are too long
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB Observatory backend extension'
|
||||
default_version = '1.1.6'
|
||||
requires = 'postgis, postgres_fdw'
|
||||
default_version = '1.3.0'
|
||||
requires = 'postgis'
|
||||
superuser = true
|
||||
schema = cdb_observatory
|
||||
|
||||
@ -38,30 +38,22 @@ def get_tablename_query(column_id, boundary_id, timespan):
|
||||
timespan, give back the current table hash from the data observatory
|
||||
"""
|
||||
return """
|
||||
SELECT t.tablename, geoid_ct.colname colname, t.id table_id
|
||||
FROM observatory.obs_table t,
|
||||
observatory.obs_column_table geoid_ct,
|
||||
observatory.obs_column_table data_ct
|
||||
WHERE
|
||||
t.id = geoid_ct.table_id AND
|
||||
t.id = data_ct.table_id AND
|
||||
geoid_ct.column_id =
|
||||
(SELECT source_id
|
||||
FROM observatory.obs_column_to_column
|
||||
WHERE target_id = '{boundary_id}'
|
||||
AND reltype = 'geom_ref'
|
||||
) AND
|
||||
data_ct.column_id = '{column_id}' AND
|
||||
timespan = '{timespan}'
|
||||
""".format(column_id=column_id,
|
||||
boundary_id=boundary_id,
|
||||
timespan=timespan)
|
||||
SELECT numer_tablename, numer_geomref_colname, numer_tid,
|
||||
geom_tablename, geom_geomref_colname, geom_tid
|
||||
FROM observatory.obs_meta
|
||||
WHERE numer_id = '{numer_id}' AND
|
||||
geom_id = '{geom_id}' AND
|
||||
numer_timespan = '{numer_timespan}'
|
||||
""".format(numer_id=column_id,
|
||||
geom_id=boundary_id,
|
||||
numer_timespan=timespan)
|
||||
|
||||
|
||||
METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
|
||||
'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta',
|
||||
'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom',
|
||||
'obs_meta_timespan', 'obs_column_table_tile', ]
|
||||
'obs_meta_timespan', 'obs_column_table_tile',
|
||||
'obs_column_table_tile_simple']
|
||||
|
||||
FIXTURES = [
|
||||
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
@ -75,7 +67,7 @@ FIXTURES = [
|
||||
('us.census.acs.B05001006_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08006001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08006002_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08006008_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08301010_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08006009_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08006011_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08006015_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
@ -129,7 +121,6 @@ FIXTURES = [
|
||||
('us.census.acs.B15003022', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B15003023', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19013001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19083001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19301001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B25001001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B25002003', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
@ -142,15 +133,6 @@ FIXTURES = [
|
||||
('us.census.acs.B25081002', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134002', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134003', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134004', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134005', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134006', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134007', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134008', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134009', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08134010', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B08135001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19001002', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19001003', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19001004', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
@ -167,24 +149,72 @@ FIXTURES = [
|
||||
('us.census.acs.B19001015', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19001016', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B19001017', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'),
|
||||
('us.census.tiger.block_group', 'us.census.tiger.block_group', '2014'),
|
||||
('us.census.tiger.zcta5', 'us.census.tiger.zcta5', '2014'),
|
||||
('us.census.tiger.county', 'us.census.tiger.county', '2014'),
|
||||
('us.census.acs.B01001002', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B01003001', 'us.census.tiger.block_group', '2010 - 2014'),
|
||||
('us.census.spielman_singleton_segments.X2', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B01001002', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B01001026', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B01002001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002003', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002004', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002006', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002012', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002005', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002008', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002009', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B03002002', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B11001001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003017', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003019', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003020', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003021', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003022', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B15003023', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19013001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19083001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19301001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25001001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25002003', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25004002', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25004004', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25058001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25071001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25075001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25075025', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B25081002', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08134001', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08134002', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08134008', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08134008', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B08134010', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001002', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001003', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001004', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001005', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001006', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001007', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001008', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001009', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001010', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001011', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001012', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001013', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001014', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001015', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001016', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.acs.B19001017', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.spielman_singleton_segments.X31', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.census.spielman_singleton_segments.X55', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2014-01'),
|
||||
('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2016-06'),
|
||||
('whosonfirst.wof_country_geom', 'whosonfirst.wof_country_geom', '2016'),
|
||||
('us.census.tiger.zcta5_clipped', 'us.census.tiger.zcta5_clipped', '2014'),
|
||||
('us.census.tiger.block_group_clipped', 'us.census.tiger.block_group_clipped', '2014'),
|
||||
('us.census.tiger.census_tract_clipped', 'us.census.tiger.census_tract_clipped', '2014'),
|
||||
('whosonfirst.wof_country_name', 'whosonfirst.wof_country_geom', '2016'),
|
||||
('us.census.acs.B01003001', 'us.census.tiger.zcta5_clipped', '2010 - 2014'),
|
||||
('us.census.acs.B01003001', 'us.census.tiger.block_group_clipped', '2010 - 2014'),
|
||||
('us.census.acs.B01003001', 'us.census.tiger.census_tract_clipped', '2010 - 2014'),
|
||||
('us.census.tiger.fullname', 'us.census.tiger.pointlm_geom', '2016'),
|
||||
('us.census.tiger.fullname', 'us.census.tiger.prisecroads_geom', '2016'),
|
||||
('us.census.tiger.name', 'us.census.tiger.county', '2015'),
|
||||
('us.census.tiger.name', 'us.census.tiger.block_group', '2015'),
|
||||
]
|
||||
|
||||
OUTFILE_PATH = os.path.join(os.path.dirname(__file__), '..',
|
||||
@ -203,6 +233,7 @@ def dump(cols, tablename, where=''):
|
||||
' | sed "s:SET search_path.*::" '
|
||||
' | sed "s:CREATE TABLE :CREATE TABLE observatory.:" '
|
||||
' | sed "s:ALTER TABLE.*OWNER.*::" '
|
||||
' | sed "s:SET idle_in_transaction_session_timeout.*::" '
|
||||
' >> {outfile}'.format(
|
||||
tablename=tablename,
|
||||
outfile=OUTFILE_PATH,
|
||||
@ -233,15 +264,19 @@ def main():
|
||||
tablename_query = get_tablename_query(column_id, boundary_id, timespan)
|
||||
resp = query(tablename_query).fetchone()
|
||||
if resp:
|
||||
tablename, colname, table_id = resp
|
||||
numer_tablename, numer_colname, numer_table_id = resp[0:3]
|
||||
geom_tablename, geom_colname, geom_table_id = resp[3:6]
|
||||
else:
|
||||
print("Could not find table for {}, {}, {}".format(
|
||||
raise Exception("Could not find table for {}, {}, {}".format(
|
||||
column_id, boundary_id, timespan))
|
||||
continue
|
||||
table_colname = (tablename, colname, boundary_id, table_id, )
|
||||
if table_colname not in unique_tables:
|
||||
print(table_colname)
|
||||
unique_tables.add(table_colname)
|
||||
numer = (numer_tablename, numer_colname, numer_table_id, )
|
||||
geom = (geom_tablename, geom_colname, geom_table_id, )
|
||||
if numer not in unique_tables:
|
||||
print(numer)
|
||||
unique_tables.add(numer)
|
||||
if geom not in unique_tables:
|
||||
print(geom)
|
||||
unique_tables.add(geom)
|
||||
|
||||
print unique_tables
|
||||
|
||||
@ -290,13 +325,14 @@ def main():
|
||||
"column_id IN ('{}', '{}')".format(numer_id, geom_id)
|
||||
for numer_id, geom_id, timespan in FIXTURES
|
||||
])
|
||||
elif tablename in ('obs_column_table', 'obs_column_table_tile', ):
|
||||
where = 'WHERE column_id IN ({numer_ids}) ' \
|
||||
'OR column_id IN ({geom_ids}) ' \
|
||||
'OR table_id IN ({table_ids}) '.format(
|
||||
elif tablename in ('obs_column_table', 'obs_column_table_tile',
|
||||
'obs_column_table_tile_simple'):
|
||||
where = '''WHERE table_id IN ({table_ids}) AND
|
||||
(column_id IN ({numer_ids}) OR column_id IN ({geom_ids}))
|
||||
'''.format(
|
||||
numer_ids=','.join(["'{}'".format(x) for x, _, _ in FIXTURES]),
|
||||
geom_ids=','.join(["'{}'".format(x) for _, x, _ in FIXTURES]),
|
||||
table_ids=','.join(["'{}'".format(x) for _, _, _, x in unique_tables])
|
||||
table_ids=','.join(["'{}'".format(x) for _, _, x in unique_tables])
|
||||
)
|
||||
elif tablename == 'obs_column_to_column':
|
||||
where = "WHERE " + " OR ".join([
|
||||
@ -308,17 +344,20 @@ def main():
|
||||
where = 'WHERE timespan IN ({timespans}) ' \
|
||||
'OR id IN ({table_ids}) '.format(
|
||||
timespans=','.join(["'{}'".format(x) for _, _, x in FIXTURES]),
|
||||
table_ids=','.join(["'{}'".format(x) for _, _, _, x in unique_tables])
|
||||
table_ids=','.join(["'{}'".format(x) for _, _, x in unique_tables])
|
||||
)
|
||||
else:
|
||||
where = ''
|
||||
dump('*', tablename, where)
|
||||
|
||||
for tablename, colname, boundary_id, table_id in unique_tables:
|
||||
if 'zcta5' in boundary_id:
|
||||
for tablename, colname, table_id in unique_tables:
|
||||
if 'zcta5' in table_id or 'zillow_zip' in table_id:
|
||||
where = '\'11%\''
|
||||
compare = 'LIKE'
|
||||
elif 'whosonfirst' in boundary_id:
|
||||
elif 'pri_sec_roads' in table_id or 'point_landmark' in table_id:
|
||||
dump('*', tablename, 'WHERE geom && ST_MakeEnvelope(-74,40.69,-73.9,40.72, 4326)')
|
||||
continue
|
||||
elif 'whosonfirst' in table_id:
|
||||
where = '(\'85632785\',\'85633051\',\'85633111\',\'85633147\',\'85633253\',\'85633267\')'
|
||||
compare = 'IN'
|
||||
else:
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB Observatory backend extension'
|
||||
default_version = '1.1.6'
|
||||
requires = 'postgis, postgres_fdw'
|
||||
default_version = '1.3.0'
|
||||
requires = 'postgis'
|
||||
superuser = true
|
||||
schema = cdb_observatory
|
||||
|
||||
@ -1,67 +0,0 @@
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_ConnectRemoteTable(fdw_name text, schema_name text, user_dbname text, user_hostname text, username text, user_tablename text, user_schema text)
|
||||
RETURNS void
|
||||
AS $$
|
||||
DECLARE
|
||||
row record;
|
||||
option record;
|
||||
connection_str json;
|
||||
BEGIN
|
||||
-- Build connection string
|
||||
connection_str := '{"server":{"extensions":"postgis", "dbname":"'
|
||||
|| user_dbname ||'", "host":"' || user_hostname ||'", "port":"6432"}, "users":{"public"'
|
||||
|| ':{"user":"' || username ||'", "password":""} } }';
|
||||
|
||||
-- This function tries to be as idempotent as possible, by not creating anything more than once
|
||||
-- (not even using IF NOT EXIST to avoid throwing warnings)
|
||||
IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
|
||||
CREATE EXTENSION postgres_fdw;
|
||||
END IF;
|
||||
-- Create FDW first if it does not exist
|
||||
IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name)
|
||||
THEN
|
||||
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Set FDW settings
|
||||
FOR row IN SELECT p.key, p.value from lateral json_each_text(connection_str->'server') p
|
||||
LOOP
|
||||
IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key)
|
||||
THEN
|
||||
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value);
|
||||
ELSE
|
||||
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value);
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- Create user mappings
|
||||
FOR row IN SELECT p.key, p.value from lateral json_each(connection_str->'users') p LOOP
|
||||
-- Check if entry on pg_user_mappings exists
|
||||
IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = row.key ) THEN
|
||||
EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', row.key, fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Update user mapping settings
|
||||
FOR option IN SELECT o.key, o.value from lateral json_each_text(row.value) o LOOP
|
||||
IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = row.key) SELECT * from a where options = option.key) THEN
|
||||
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', row.key, fdw_name, option.key, option.value);
|
||||
ELSE
|
||||
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', row.key, fdw_name, option.key, option.value);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
|
||||
-- Create schema if it does not exist.
|
||||
IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN
|
||||
EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Bring the remote cdb_tablemetadata
|
||||
IF NOT EXISTS ( SELECT * FROM PG_CLASS WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname=fdw_name) and relname='cdb_tablemetadata') THEN
|
||||
EXECUTE FORMAT ('CREATE FOREIGN TABLE %I.cdb_tablemetadata (tabname text, updated_at timestamp with time zone) SERVER %I OPTIONS (table_name ''cdb_tablemetadata_text'', schema_name ''public'', updatable ''false'')', fdw_name, fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Import target table
|
||||
EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) from SERVER %I INTO %I', user_schema, user_tablename, fdw_name, schema_name);
|
||||
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL;
|
||||
@ -219,4 +219,3 @@ CREATE AGGREGATE cdb_observatory.FIRST (
|
||||
basetype = anyelement,
|
||||
stype = anyelement
|
||||
);
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -298,7 +298,7 @@ BEGIN
|
||||
) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms,
|
||||
percentfill, estnumgeoms, meanmediansize
|
||||
FROM available_geoms, scores
|
||||
WHERE available_geoms.geom_id = scores.geom_id
|
||||
WHERE available_geoms.geom_id = scores.column_id
|
||||
$string$, geom_clause)
|
||||
USING numer_id, denom_id, timespan, filter_tags, bounds;
|
||||
RETURN;
|
||||
@ -412,14 +412,16 @@ BEGIN
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
|
||||
bounds Geometry(Geometry, 4326) DEFAULT NULL,
|
||||
filter_geom_ids TEXT[] DEFAULT NULL,
|
||||
desired_num_geoms INTEGER DEFAULT 3000
|
||||
desired_num_geoms INTEGER DEFAULT NULL
|
||||
) RETURNS TABLE (
|
||||
score NUMERIC,
|
||||
numtiles BIGINT,
|
||||
geom_id TEXT,
|
||||
table_id TEXT,
|
||||
column_id TEXT,
|
||||
notnull_percent NUMERIC,
|
||||
numgeoms NUMERIC,
|
||||
percentfill NUMERIC,
|
||||
@ -427,57 +429,66 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
|
||||
meanmediansize NUMERIC
|
||||
) AS $$
|
||||
BEGIN
|
||||
IF desired_num_geoms IS NULL THEN
|
||||
desired_num_geoms := 3000;
|
||||
END IF;
|
||||
filter_geom_ids := COALESCE(filter_geom_ids, (ARRAY[])::TEXT[]);
|
||||
-- Very complex geometries simply fail. For a boundary check, we can
|
||||
-- comfortably get away with the simplicity of an envelope
|
||||
IF ST_Npoints(bounds) > 10000 THEN
|
||||
bounds := ST_Envelope(bounds);
|
||||
END IF;
|
||||
RETURN QUERY
|
||||
EXECUTE format($string$
|
||||
SELECT
|
||||
((100.0 / (1+abs(log(1 + $3) - log(1 + numgeoms)))) * percentfill)::Numeric
|
||||
AS score, *
|
||||
FROM (
|
||||
EXECUTE $string$
|
||||
WITH clipped_geom AS (
|
||||
SELECT column_id, table_id
|
||||
, CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True)
|
||||
, CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True) -- -20
|
||||
ELSE tile END clipped_tile
|
||||
, tile
|
||||
FROM observatory.obs_column_table_tile
|
||||
FROM observatory.obs_column_table_tile_simple
|
||||
WHERE ($1 IS NULL OR ST_Intersects($1, tile))
|
||||
AND (column_id = ANY($2) OR cardinality($2) = 0)
|
||||
), clipped_geom_countagg AS (
|
||||
SELECT column_id, table_id
|
||||
, ST_CountAgg(clipped_tile, 2, True)::Numeric notnull_pixels
|
||||
, ST_CountAgg(clipped_tile, 2, False)::Numeric pixels
|
||||
, BOOL_AND(ST_BandIsNoData(clipped_tile, 1)) nodata
|
||||
, ST_CountAgg(clipped_tile, 1, False)::Numeric pixels -- -10
|
||||
FROM clipped_geom
|
||||
GROUP BY column_id, table_id
|
||||
) SELECT
|
||||
count(*)::BIGINT, a.column_id
|
||||
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
|
||||
THEN cdb_observatory.FIRST(notnull_pixels) / cdb_observatory.FIRST(pixels)
|
||||
ELSE 1
|
||||
END)::Numeric AS notnull_percent
|
||||
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
|
||||
THEN (ST_SummaryStatsAgg(clipped_tile, 2, True)).sum
|
||||
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0) * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(cdb_observatory.FIRST(tile), 0, 0)) * cdb_observatory.FIRST(pixels))
|
||||
END)::Numeric AS numgeoms
|
||||
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
|
||||
THEN (ST_SummaryStatsAgg(clipped_tile, 3, True)).mean
|
||||
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 3, ST_PointOnSurface($1)), 0)
|
||||
END)::Numeric AS percentfill
|
||||
, ((ST_Area(ST_Transform($1, 3857)) / 1000000) / NullIf(
|
||||
CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
|
||||
THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
|
||||
ELSE Coalesce(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
|
||||
END, 0))::Numeric AS estnumgeoms
|
||||
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
|
||||
THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
|
||||
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
|
||||
END)::Numeric AS meanmediansize
|
||||
), clipped_geom_reagg AS (
|
||||
SELECT COUNT(*)::BIGINT cnt, a.column_id, a.table_id,
|
||||
cdb_observatory.FIRST(nodata) first_nodata,
|
||||
cdb_observatory.FIRST(pixels) first_pixel,
|
||||
cdb_observatory.FIRST(tile) first_tile,
|
||||
(ST_SummaryStatsAgg(clipped_tile, 1, False)).sum::Numeric sum_geoms, -- ND
|
||||
(ST_SummaryStatsAgg(clipped_tile, 2, False)).mean::Numeric / 255 mean_fill --ND
|
||||
FROM clipped_geom_countagg a, clipped_geom b
|
||||
WHERE a.table_id = b.table_id
|
||||
AND a.column_id = b.column_id
|
||||
GROUP BY a.column_id, a.table_id
|
||||
ORDER BY a.column_id, a.table_id
|
||||
) foo
|
||||
$string$) USING bounds, filter_geom_ids, desired_num_geoms;
|
||||
), final AS (
|
||||
SELECT
|
||||
cnt, table_id, column_id
|
||||
, NULL::Numeric AS notnull_percent
|
||||
, (CASE WHEN first_nodata IS FALSE
|
||||
THEN sum_geoms
|
||||
ELSE COALESCE(ST_Value(first_tile, 1, ST_PointOnSurface($1)), 0)
|
||||
* (ST_Area($1) / ST_Area(ST_PixelAsPolygon(first_tile, 0, 0))
|
||||
* first_pixel) -- -20
|
||||
END)::Numeric
|
||||
AS numgeoms
|
||||
, (CASE WHEN first_nodata IS FALSE
|
||||
THEN mean_fill
|
||||
ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2
|
||||
END)::Numeric
|
||||
AS percentfill
|
||||
, null::numeric estnumgeoms
|
||||
, null::numeric meanmediansize
|
||||
FROM clipped_geom_reagg
|
||||
) SELECT
|
||||
((100.0 / (1+abs(log(0.0001 + $3) - log(0.0001 + numgeoms::Numeric)))) * percentfill)::Numeric
|
||||
AS score, *
|
||||
FROM final
|
||||
$string$ USING bounds, filter_geom_ids, desired_num_geoms;
|
||||
RETURN;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||
|
||||
@ -40,44 +40,11 @@ BEGIN
|
||||
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_Point''', ST_GeometryType(geom);
|
||||
END IF;
|
||||
|
||||
-- choose appropriate table based on time_span
|
||||
IF time_span IS NULL
|
||||
THEN
|
||||
SELECT x.target_tables INTO target_table
|
||||
FROM cdb_observatory._OBS_SearchTables(boundary_id,
|
||||
time_span) As x(target_tables,
|
||||
timespans)
|
||||
ORDER BY x.timespans DESC
|
||||
LIMIT 1;
|
||||
ELSE
|
||||
-- TODO: modify for only one table returned instead of arbitrarily choosing
|
||||
-- one with LIMIT 1 (could be conflict between clipped vs non-clipped
|
||||
-- boundaries in the metadata tables)
|
||||
SELECT x.target_tables INTO target_table
|
||||
FROM cdb_observatory._OBS_SearchTables(boundary_id,
|
||||
time_span) As x(target_tables,
|
||||
timespans)
|
||||
WHERE x.timespans = time_span
|
||||
LIMIT 1;
|
||||
END IF;
|
||||
|
||||
-- if no tables are found, raise notice and return null
|
||||
IF target_table IS NULL
|
||||
THEN
|
||||
--RAISE NOTICE 'No boundaries found for ''%'' in ''%''', ST_AsText(geom), boundary_id;
|
||||
RETURN NULL::geometry;
|
||||
END IF;
|
||||
|
||||
--RAISE NOTICE 'target_table: %', target_table;
|
||||
|
||||
-- return the first boundary in intersections
|
||||
EXECUTE format(
|
||||
'SELECT the_geom
|
||||
FROM observatory.%I
|
||||
WHERE ST_Intersects($1, the_geom)
|
||||
LIMIT 1', target_table)
|
||||
INTO boundary
|
||||
USING geom;
|
||||
EXECUTE $query$
|
||||
SELECT * FROM cdb_observatory._OBS_GetBoundariesByGeometry($1, $2, $3) LIMIT 1
|
||||
$query$ INTO boundary
|
||||
USING geom, boundary_id, time_span;
|
||||
|
||||
RETURN boundary;
|
||||
|
||||
@ -111,67 +78,17 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundaryId(
|
||||
RETURNS text
|
||||
AS $$
|
||||
DECLARE
|
||||
output_id text;
|
||||
target_table text;
|
||||
geoid_colname text;
|
||||
result TEXT;
|
||||
BEGIN
|
||||
|
||||
-- If not point, raise error
|
||||
IF ST_GeometryType(geom) != 'ST_Point'
|
||||
THEN
|
||||
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_Point''', ST_GeometryType(geom);
|
||||
END IF;
|
||||
|
||||
-- choose appropriate table based on time_span
|
||||
IF time_span IS NULL
|
||||
THEN
|
||||
SELECT x.target_tables INTO target_table
|
||||
FROM cdb_observatory._OBS_SearchTables(boundary_id,
|
||||
time_span) As x(target_tables,
|
||||
timespans)
|
||||
ORDER BY x.timespans DESC
|
||||
LIMIT 1;
|
||||
ELSE
|
||||
SELECT x.target_tables INTO target_table
|
||||
FROM cdb_observatory._OBS_SearchTables(boundary_id,
|
||||
time_span) As x(target_tables,
|
||||
timespans)
|
||||
WHERE x.timespans = time_span
|
||||
LIMIT 1;
|
||||
END IF;
|
||||
|
||||
-- if no tables are found, raise notice and return null
|
||||
IF target_table IS NULL
|
||||
THEN
|
||||
--RAISE NOTICE 'Warning: No boundaries found for ''%''', boundary_id;
|
||||
RETURN NULL::text;
|
||||
END IF;
|
||||
|
||||
EXECUTE
|
||||
format('SELECT ct.colname
|
||||
FROM observatory.obs_column_to_column c2c,
|
||||
observatory.obs_column_table ct,
|
||||
observatory.obs_table t
|
||||
WHERE c2c.reltype = ''geom_ref''
|
||||
AND ct.column_id = c2c.source_id
|
||||
AND ct.table_id = t.id
|
||||
AND t.tablename = %L'
|
||||
, target_table)
|
||||
INTO geoid_colname;
|
||||
|
||||
--RAISE NOTICE 'target_table: %, geoid_colname: %', target_table, geoid_colname;
|
||||
|
||||
-- return geometry id column value
|
||||
EXECUTE format(
|
||||
'SELECT %I::text
|
||||
FROM observatory.%I
|
||||
WHERE ST_Intersects($1, the_geom)
|
||||
LIMIT 1', geoid_colname, target_table)
|
||||
INTO output_id
|
||||
USING geom;
|
||||
|
||||
RETURN output_id;
|
||||
EXECUTE $query$
|
||||
SELECT geom_refs FROM cdb_observatory._OBS_GetBoundariesByGeometry(
|
||||
$1, $2, $3) LIMIT 1
|
||||
$query$
|
||||
INTO result
|
||||
USING geom, boundary_id, time_span;
|
||||
|
||||
RETURN result;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
@ -203,35 +120,21 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundaryById(
|
||||
RETURNS geometry(geometry, 4326)
|
||||
AS $$
|
||||
DECLARE
|
||||
boundary geometry(geometry, 4326);
|
||||
target_table text;
|
||||
geoid_colname text;
|
||||
geom_colname text;
|
||||
result GEOMETRY;
|
||||
BEGIN
|
||||
|
||||
SELECT * INTO geoid_colname, target_table, geom_colname
|
||||
FROM cdb_observatory._OBS_GetGeometryMetadata(boundary_id);
|
||||
|
||||
--RAISE NOTICE '%', target_table;
|
||||
|
||||
IF target_table IS NULL
|
||||
THEN
|
||||
--RAISE NOTICE 'No geometries found';
|
||||
RETURN NULL::geometry;
|
||||
END IF;
|
||||
|
||||
-- retrieve boundary
|
||||
EXECUTE
|
||||
format(
|
||||
'SELECT %I
|
||||
FROM observatory.%I
|
||||
WHERE %I = $1
|
||||
LIMIT 1', geom_colname, target_table, geoid_colname)
|
||||
INTO boundary
|
||||
USING geometry_id;
|
||||
|
||||
RETURN boundary;
|
||||
EXECUTE $query$
|
||||
SELECT (data->0->>'value')::Geometry
|
||||
FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[$1],
|
||||
cdb_observatory.OBS_GetMeta(
|
||||
ST_MakeEnvelope(-180, -90, 180, 90, 4326),
|
||||
('[{"geom_id": "' || $2 || '"}]')::JSON))
|
||||
$query$
|
||||
INTO result
|
||||
USING geometry_id, boundary_id;
|
||||
|
||||
RETURN result;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
@ -245,13 +148,12 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetBoundariesByGeometry(
|
||||
boundary_id text,
|
||||
time_span text DEFAULT NULL,
|
||||
overlap_type text DEFAULT NULL)
|
||||
RETURNS TABLE(the_geom geometry, geom_refs text)
|
||||
AS $$
|
||||
RETURNS TABLE (
|
||||
the_geom geometry,
|
||||
geom_refs text
|
||||
) AS $$
|
||||
DECLARE
|
||||
boundary geometry(Geometry, 4326);
|
||||
geom_colname text;
|
||||
geoid_colname text;
|
||||
target_table text;
|
||||
meta JSON;
|
||||
BEGIN
|
||||
overlap_type := COALESCE(overlap_type, 'intersects');
|
||||
-- check inputs
|
||||
@ -259,34 +161,27 @@ BEGIN
|
||||
THEN
|
||||
-- recognized overlap type (map to ST_Contains, ST_Intersects, and ST_Within)
|
||||
RAISE EXCEPTION 'Overlap type ''%'' is not an accepted type (choose intersects, within, or contains)', overlap_type;
|
||||
ELSIF ST_GeometryType(geom) NOT IN ('ST_Polygon', 'ST_MultiPolygon')
|
||||
THEN
|
||||
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom);
|
||||
END IF;
|
||||
|
||||
-- TODO: add timespan in search
|
||||
-- TODO: add overlap info in search
|
||||
SELECT * INTO geoid_colname, target_table, geom_colname
|
||||
FROM cdb_observatory._OBS_GetGeometryMetadata(boundary_id);
|
||||
EXECUTE $query$
|
||||
SELECT cdb_observatory.OBS_GetMeta($1, JSON_Build_Array(JSON_Build_Object(
|
||||
'geom_id', $2, 'geom_timespan', $3)))
|
||||
$query$
|
||||
INTO meta
|
||||
USING geom, boundary_id, time_span;
|
||||
|
||||
-- if no tables are found, raise notice and return null
|
||||
IF target_table IS NULL
|
||||
THEN
|
||||
--RAISE NOTICE 'No boundaries found for bounding box ''%'' in ''%''', ST_AsText(geom), boundary_id;
|
||||
RETURN QUERY SELECT NULL::geometry, NULL::text;
|
||||
IF meta->0->>'geom_id' IS NULL THEN
|
||||
RETURN QUERY EXECUTE 'SELECT NULL::Geometry, NULL::Text LIMIT 0';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
--RAISE NOTICE 'target_table: %', target_table;
|
||||
|
||||
-- return first boundary in intersections
|
||||
RETURN QUERY
|
||||
EXECUTE format(
|
||||
'SELECT %I, %I::text
|
||||
FROM observatory.%I
|
||||
WHERE ST_%s($1, the_geom)
|
||||
', geom_colname, geoid_colname, target_table, overlap_type)
|
||||
USING geom;
|
||||
RETURN QUERY EXECUTE $query$
|
||||
SELECT (data->0->>'value')::Geometry the_geom, data->0->>'geomref' geom_refs
|
||||
FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[($1, 1)::geomval], $2, False
|
||||
)
|
||||
$query$ USING geom, meta;
|
||||
RETURN;
|
||||
|
||||
END;
|
||||
@ -414,27 +309,11 @@ BEGIN
|
||||
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom);
|
||||
END IF;
|
||||
|
||||
SELECT * INTO geoid_colname, target_table, geom_colname
|
||||
FROM cdb_observatory._OBS_GetGeometryMetadata(boundary_id);
|
||||
|
||||
-- if no tables are found, raise notice and return null
|
||||
IF target_table IS NULL
|
||||
THEN
|
||||
--RAISE NOTICE 'No boundaries found for bounding box ''%'' in ''%''', ST_AsText(geom), boundary_id;
|
||||
RETURN QUERY SELECT NULL::geometry, NULL::text;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
--RAISE NOTICE 'target_table: %', target_table;
|
||||
|
||||
-- return first boundary in intersections
|
||||
RETURN QUERY
|
||||
EXECUTE format(
|
||||
'SELECT ST_PointOnSurface(%I) As %s, %I::text
|
||||
FROM observatory.%I
|
||||
WHERE ST_%s($1, the_geom)
|
||||
', geom_colname, geom_colname, geoid_colname, target_table, overlap_type)
|
||||
USING geom;
|
||||
RETURN QUERY EXECUTE $query$
|
||||
SELECT ST_PointOnSurface(the_geom), geom_refs
|
||||
FROM cdb_observatory._OBS_GetBoundariesByGeometry($1, $2)
|
||||
$query$ USING geom, boundary_id;
|
||||
RETURN;
|
||||
|
||||
END;
|
||||
@ -534,44 +413,3 @@ BEGIN
|
||||
RETURN;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
-- _OBS_GetGeometryMetadata()
|
||||
-- TODO: add timespan in search
|
||||
-- TODO: add choice of clipped versus not clipped
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryMetadata(boundary_id text)
|
||||
RETURNS table(geoid_colname text, target_table text, geom_colname text)
|
||||
AS $$
|
||||
BEGIN
|
||||
|
||||
RETURN QUERY
|
||||
EXECUTE
|
||||
format($string$
|
||||
SELECT geoid_ct.colname::text As geoid_colname,
|
||||
tablename::text,
|
||||
geom_ct.colname::text As geom_colname
|
||||
FROM observatory.obs_column_table As geoid_ct,
|
||||
observatory.obs_table As geom_t,
|
||||
observatory.obs_column_table As geom_ct,
|
||||
observatory.obs_column As geom_c
|
||||
WHERE geoid_ct.column_id
|
||||
IN (
|
||||
SELECT source_id
|
||||
FROM observatory.obs_column_to_column
|
||||
WHERE reltype = 'geom_ref'
|
||||
AND target_id = '%s'
|
||||
)
|
||||
AND geoid_ct.table_id = geom_t.id AND
|
||||
geom_t.id = geom_ct.table_id AND
|
||||
geom_ct.column_id = geom_c.id AND
|
||||
geom_c.type ILIKE 'geometry' AND
|
||||
geom_c.id = '%s'
|
||||
$string$, boundary_id, boundary_id);
|
||||
RETURN;
|
||||
-- AND geom_t.timespan = '%s' <-- put in requested year
|
||||
-- TODO: filter by clipped vs. not so appropriate tablename are unique
|
||||
-- so the limit 1 can be removed
|
||||
RETURN;
|
||||
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
@ -1,82 +0,0 @@
|
||||
CREATE TYPE cdb_observatory.ds_fdw_metadata as (schemaname text, tabname text, servername text);
|
||||
CREATE TYPE cdb_observatory.ds_return_metadata as (colnames text[], coltypes text[]);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
|
||||
RETURNS cdb_observatory.ds_fdw_metadata
|
||||
AS $$
|
||||
DECLARE
|
||||
fdw_server text;
|
||||
fdw_import_schema text;
|
||||
connection_str json;
|
||||
import_foreign_schema_q text;
|
||||
epoch_timestamp text;
|
||||
BEGIN
|
||||
|
||||
SELECT extract(epoch from now() at time zone 'utc')::int INTO epoch_timestamp;
|
||||
fdw_server := 'fdw_server_' || username || '_' || epoch_timestamp;
|
||||
fdw_import_schema:= fdw_server;
|
||||
|
||||
-- Import foreign table
|
||||
EXECUTE FORMAT ('SELECT cdb_observatory._OBS_ConnectRemoteTable(%L, %L, %L, %L, %L, %L, %L)', fdw_server, fdw_import_schema, dbname, host_addr, user_db_role, table_name, input_schema);
|
||||
|
||||
RETURN (fdw_import_schema::text, table_name::text, fdw_server::text);
|
||||
|
||||
EXCEPTION
|
||||
WHEN others THEN
|
||||
-- Disconnect user imported table. Delete schema and FDW server.
|
||||
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".' || table_name;
|
||||
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".cdb_tablemetadata';
|
||||
EXECUTE 'DROP SCHEMA IF EXISTS "' || fdw_import_schema || '"';
|
||||
EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || fdw_server || '"';
|
||||
EXECUTE 'DROP SERVER IF EXISTS "' || fdw_server || '"';
|
||||
|
||||
RETURN (null, null, null);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
|
||||
RETURNS cdb_observatory.ds_return_metadata
|
||||
AS $$
|
||||
DECLARE
|
||||
colnames text[];
|
||||
coltypes text[];
|
||||
BEGIN
|
||||
EXECUTE FORMAT('SELECT r.colnames::text[], r.coltypes::text[] FROM cdb_observatory._%sResultMetadata(%L::json) r', function_name, params::text)
|
||||
INTO colnames, coltypes;
|
||||
|
||||
RETURN (colnames::text[], coltypes::text[]);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
|
||||
RETURNS SETOF record
|
||||
AS $$
|
||||
DECLARE
|
||||
data_query text;
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
|
||||
EXECUTE FORMAT('SELECT cdb_observatory._%sQuery(%L, %L, %L::json)', function_name, table_schema, table_name, params::text)
|
||||
INTO data_query;
|
||||
|
||||
FOR rec IN EXECUTE data_query
|
||||
LOOP
|
||||
RETURN NEXT rec;
|
||||
END LOOP;
|
||||
RETURN;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
|
||||
RETURNS boolean
|
||||
AS $$
|
||||
BEGIN
|
||||
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".' || table_name;
|
||||
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".cdb_tablemetadata';
|
||||
EXECUTE 'DROP SCHEMA IF EXISTS "' || table_schema || '"';
|
||||
EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || servername || '"';
|
||||
EXECUTE 'DROP SERVER IF EXISTS "' || servername || '"';
|
||||
RETURN true;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
@ -1,79 +0,0 @@
|
||||
--
|
||||
--
|
||||
-- OBS_GetMeasure
|
||||
--
|
||||
--
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureResultMetadata(params json)
|
||||
RETURNS cdb_observatory.ds_return_metadata
|
||||
AS $$
|
||||
DECLARE
|
||||
colnames text[]; -- Array to store the name of the measures to be returned
|
||||
coltypes text[]; -- Array to store the type of the measures to be returned
|
||||
requested_measures text[];
|
||||
measure_id text;
|
||||
BEGIN
|
||||
-- By definition, all the measure results for the OBS_GetMeasure API are numeric values
|
||||
SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO requested_measures;
|
||||
|
||||
FOREACH measure_id IN ARRAY requested_measures
|
||||
LOOP
|
||||
SELECT array_append(colnames, measure_id) INTO colnames;
|
||||
SELECT array_append(coltypes, 'numeric'::text) INTO coltypes;
|
||||
END LOOP;
|
||||
|
||||
RETURN (colnames::text[], coltypes::text[]);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureQuery(table_schema text, table_name text, params json)
|
||||
RETURNS text
|
||||
AS $$
|
||||
DECLARE
|
||||
data_query text;
|
||||
measure_ids_arr text[];
|
||||
measure_id text;
|
||||
measures_list text;
|
||||
measures_query text;
|
||||
normalize text;
|
||||
boundary_id text;
|
||||
time_span text;
|
||||
geom_table_name text;
|
||||
data_table_name text;
|
||||
BEGIN
|
||||
measures_query := '';
|
||||
-- SELECT table_name from obs_meta WHERE boundary_id = {bound} AND [...] INTO geom_table_name
|
||||
geom_table_name := 'observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308';
|
||||
-- SELECT table_name from obs_meta WHERE time_span = {time} AND [...] INTO data_table_name
|
||||
data_table_name := 'observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd';
|
||||
|
||||
-- Get measure_ids array from JSON
|
||||
SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO measure_ids_arr;
|
||||
|
||||
-- Get a comma-separated list of measures ("total_pop, over_16_pop") to be used in SELECTs
|
||||
SELECT array_to_string(measure_ids_arr, ',') INTO measures_list;
|
||||
|
||||
FOREACH measure_id IN ARRAY measure_ids_arr
|
||||
LOOP
|
||||
-- Build query to compute each value and normalize
|
||||
-- Assumes the default normalization method, the normalize parameter given in the JSON
|
||||
-- should be checked in order to build the final query
|
||||
SELECT measures_query || ' sum(' || measure_id || '/fraction)::numeric as ' || measure_id || ', ' INTO measures_query;
|
||||
END LOOP;
|
||||
|
||||
-- Data query should select the measures and the cartodb_id of the user table, in that order.
|
||||
data_query := '(WITH _areas AS(SELECT ST_Area(a.the_geom::geography)'
|
||||
|| '/ (1000 * 1000) as fraction, a.geoid, b.cartodb_id FROM '
|
||||
|| geom_table_name || ' as a, '
|
||||
|| table_schema || '.' || table_name || ' AS b '
|
||||
|| 'WHERE b.the_geom && a.the_geom ), values AS (SELECT geoid, '
|
||||
|| measures_list
|
||||
|| ' FROM ' || data_table_name || ' ) '
|
||||
|| 'SELECT '
|
||||
|| measures_query
|
||||
|| ' cartodb_id::int FROM _areas, values '
|
||||
|| 'WHERE values.geoid = _areas.geoid GROUP BY cartodb_id);';
|
||||
RETURN data_query;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
-- Install dependencies
|
||||
CREATE EXTENSION postgis;
|
||||
CREATE EXTENSION postgres_fdw;
|
||||
-- Install the extension
|
||||
CREATE EXTENSION observatory VERSION 'dev';
|
||||
|
||||
@ -3,36 +3,12 @@
|
||||
obs_getdemographicsnapshot_test_no_returns
|
||||
t
|
||||
(1 row)
|
||||
obs_get_median_income_at_test_point
|
||||
t
|
||||
(1 row)
|
||||
obs_get_median_income_at_null_island
|
||||
t
|
||||
(1 row)
|
||||
obs_getpoints_for_test_point_value|obs_getpoints_for_test_point_name|obs_getpoints_for_test_point_tablename|obs_getpoints_for_test_point_aggregate|obs_getpoints_for_test_point_type|obs_getpoints_for_test_point_description
|
||||
t|t|t|t|t|t
|
||||
(1 row)
|
||||
obs_getpoints_for_null_island
|
||||
t
|
||||
(1 row)
|
||||
obs_getpolygons_for_test_point
|
||||
t
|
||||
(1 row)
|
||||
obs_getpolygons_for_null_island
|
||||
t
|
||||
(1 row)
|
||||
test_point_segmentation
|
||||
t
|
||||
(1 row)
|
||||
null_island_segmentation
|
||||
t
|
||||
(1 row)
|
||||
getcategories_at_test_point_1
|
||||
t
|
||||
(1 row)
|
||||
getcategories_at_null_island
|
||||
t
|
||||
(1 row)
|
||||
obs_getmeasure_zhvi_point_test
|
||||
t
|
||||
(1 row)
|
||||
@ -129,3 +105,144 @@ t
|
||||
obs_getmeasurebyid_null_id
|
||||
t
|
||||
(1 row)
|
||||
obs_getmeta_null_null_is_null
|
||||
t
|
||||
(1 row)
|
||||
obs_getmeta_null_empty_is_null
|
||||
t
|
||||
(1 row)
|
||||
obs_getmeta_nullisland_null_is_null
|
||||
t
|
||||
(1 row)
|
||||
obs_getmeta_nullisland_empty_is_null
|
||||
t
|
||||
(1 row)
|
||||
obs_getmeta_nullisland_us_measure_is_null
|
||||
t
|
||||
(1 row)
|
||||
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|geom_id|normalization
|
||||
t|t|t|t|t|t|t|t|t|t|t
|
||||
(1 row)
|
||||
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
|
||||
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
|
||||
(1 row)
|
||||
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|geom_id|normalization
|
||||
t|t|t|t|t|t|t|t|t|t|t
|
||||
(1 row)
|
||||
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
|
||||
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
|
||||
(1 row)
|
||||
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization|id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
|
||||
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
|
||||
(1 row)
|
||||
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
|
||||
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
|
||||
(1 row)
|
||||
obs_getmeta_conflicting_metadata
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_geomval_empty_null
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_text_empty_null
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_geomval_empty_one_measure
|
||||
t
|
||||
(1 row)
|
||||
id|data_point_measure_null|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_null|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_point_measure_area|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_area|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_point_measure_predenominated|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_predenominated|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_point_measure_impossible_denominated|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_impossible_denominated|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_point_measure_denominated|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_denominated|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_one_null|data_polygon_measure_two_null
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_one_predenom|data_polygon_measure_two_predenom
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_one_area|data_polygon_measure_two_area
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_polygon_measure_tract|data_polygon_measure_bg
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_point_categorical|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_poly_categorical|nullcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|data_poly_categorical|valcol
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|correct_num_geoms
|
||||
t|t
|
||||
(1 row)
|
||||
id|correct_num_geoms|correct_pop
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|correct_num_geoms|correct_pop|correct_bg_names
|
||||
t|t|t|t
|
||||
(1 row)
|
||||
id|correct_num_points
|
||||
t|t
|
||||
(1 row)
|
||||
id|correct_num_points|pointgeom_names
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|obs_getdata_by_id_one_measure_null
|
||||
t|t
|
||||
(1 row)
|
||||
id|obs_getdata_by_id_one_measure_predenom
|
||||
t|t
|
||||
(1 row)
|
||||
id|obs_getdata_by_id_one_measure_null|obs_getdata_by_id_two_measure_null
|
||||
t|t|t
|
||||
(1 row)
|
||||
id|obs_getdata_by_id_categorical
|
||||
t|t
|
||||
(1 row)
|
||||
id|obs_getdata_by_id_geometry
|
||||
t|t
|
||||
(1 row)
|
||||
obs_getdata_api_geomvals_no_args
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_api_geomvals_args_numer_return
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_api_geomvals_args_string_return
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_api_geomrefs_args_numer_return
|
||||
t
|
||||
(1 row)
|
||||
obs_getdata_api_geomrefs_args_string_return
|
||||
t
|
||||
(1 row)
|
||||
|
||||
@ -114,7 +114,7 @@ t
|
||||
_obs_getavailablegeometries_foobarbaz_denom_not_in_usa_bg
|
||||
t
|
||||
(1 row)
|
||||
_obs_getavailablegeometries_bg_2014
|
||||
_obs_getavailablegeometries_bg_2015
|
||||
t
|
||||
(1 row)
|
||||
_obs_getavailablegeometries_bg_not_1996
|
||||
|
||||
@ -42,6 +42,12 @@ t
|
||||
obs_getboundarybyid_boundary_id_mismatch_geom_id
|
||||
t
|
||||
(1 row)
|
||||
_obs_getboundariesbygeometry_roads_around_cartodb
|
||||
t
|
||||
(1 row)
|
||||
_obs_getboundariesbygeometry_points_around_cartodb
|
||||
t
|
||||
(1 row)
|
||||
_obs_getboundariesbygeometry_tracts_around_cartodb
|
||||
t
|
||||
(1 row)
|
||||
@ -87,9 +93,3 @@ t
|
||||
obs_getpointsbypointandradius_around_null_island
|
||||
t
|
||||
(1 row)
|
||||
geoid_name_matches|table_name_matches|geom_name_matches
|
||||
t|t|t
|
||||
(1 row)
|
||||
geoid_name_matches|table_name_matches|geom_name_matches
|
||||
t|t|t
|
||||
(1 row)
|
||||
|
||||
24
src/pg/test/fixtures/drop_fixtures.sql
vendored
24
src/pg/test/fixtures/drop_fixtures.sql
vendored
@ -13,17 +13,21 @@ DROP TABLE IF EXISTS observatory.obs_meta_denom;
|
||||
DROP TABLE IF EXISTS observatory.obs_meta_geom;
|
||||
DROP TABLE IF EXISTS observatory.obs_meta_timespan;
|
||||
DROP TABLE IF EXISTS observatory.obs_column_table_tile;
|
||||
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
|
||||
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
|
||||
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
|
||||
DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103;
|
||||
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;
|
||||
DROP TABLE IF EXISTS observatory.obs_1babf5a26a1ecda5fb74963e88408f71d0364b81;
|
||||
DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple;
|
||||
DROP TABLE IF EXISTS observatory.obs_78fb6c1d6ff6505225175922c2c389ce48d7632c;
|
||||
DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d;
|
||||
DROP TABLE IF EXISTS observatory.obs_144e8b4f906885b2e057ac4842644a553ae49c6e;
|
||||
DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7;
|
||||
DROP TABLE IF EXISTS observatory.obs_1746e37b7cd28cb131971ea4187d42d71f09c5f3;
|
||||
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
|
||||
DROP TABLE IF EXISTS observatory.obs_c4411eba732408d47d73281772dbf03d60645dec;
|
||||
DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd;
|
||||
DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103;
|
||||
DROP TABLE IF EXISTS observatory.obs_a01cd5d8ccaa6531cef715071e9307e6b1987ec3;
|
||||
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
|
||||
DROP TABLE IF EXISTS observatory.obs_0310c639744a2014bb1af82709228f05b59e7d3d;
|
||||
DROP TABLE IF EXISTS observatory.obs_87a814e485deabe3b12545a537f693d16ca702c2;
|
||||
DROP TABLE IF EXISTS observatory.obs_e32f8e59c7c8861ee5ee4029b3ace2af9a5c9caf;
|
||||
DROP TABLE IF EXISTS observatory.obs_1ea93bbc109c87c676b3270789dacf7a1430db6c;
|
||||
DROP TABLE IF EXISTS observatory.obs_b393b5b88c6adda634b2071a8005b03c551b609a;
|
||||
DROP TABLE IF EXISTS observatory.obs_1746e37b7cd28cb131971ea4187d42d71f09c5f3;
|
||||
DROP TABLE IF EXISTS observatory.obs_a01cd5d8ccaa6531cef715071e9307e6b1987ec3;
|
||||
DROP TABLE IF EXISTS observatory.obs_8e30e6b3792430b410ba5b9e49cdc6a0d404d48f;
|
||||
DROP TABLE IF EXISTS observatory.obs_08025e1287e3af2b5de571d06562ba8d3bdb48e9;
|
||||
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;
|
||||
|
||||
21631
src/pg/test/fixtures/load_fixtures.sql
vendored
21631
src/pg/test/fixtures/load_fixtures.sql
vendored
File diff suppressed because one or more lines are too long
@ -1,6 +1,5 @@
|
||||
-- Install dependencies
|
||||
CREATE EXTENSION postgis;
|
||||
CREATE EXTENSION postgres_fdw;
|
||||
|
||||
-- Install the extension
|
||||
CREATE EXTENSION observatory VERSION 'dev';
|
||||
|
||||
@ -11,8 +11,8 @@ SELECT
|
||||
cdb_observatory._OBS_GeomTable(
|
||||
ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326),
|
||||
'us.census.tiger.census_tract',
|
||||
'2014'
|
||||
) = 'obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7' As _obs_geomtable_with_returned_table;
|
||||
'2015'
|
||||
) = 'obs_87a814e485deabe3b12545a537f693d16ca702c2' As _obs_geomtable_with_returned_table;
|
||||
|
||||
-- get null for unknown geometry_id
|
||||
-- should give back null
|
||||
|
||||
@ -5,128 +5,21 @@ SET client_min_messages TO WARNING;
|
||||
--
|
||||
WITH result as(
|
||||
Select count(coalesce(OBS_GetDemographicSnapshot->>'value', 'foo')) expected_columns
|
||||
FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint())
|
||||
FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint(), '2010 - 2014')
|
||||
) select expected_columns = 52 as OBS_GetDemographicSnapshot_test_no_returns
|
||||
FROM result;
|
||||
|
||||
WITH result as (
|
||||
SELECT _OBS_Get::text as expected FROM
|
||||
cdb_observatory._OBS_Get(
|
||||
cdb_observatory._TestPoint(),
|
||||
Array['us.census.acs.B19013001']::text[],
|
||||
'2010 - 2014',
|
||||
'us.census.tiger.block_group'
|
||||
)
|
||||
) SELECT expected = '{"value":79292,"name":"Median Household Income in the past 12 Months","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"median","type":"Numeric","description":"Within a geographic area, the median income received by every household on a regular basis before payments for personal income taxes, social security, union dues, medicare deductions, etc. It includes income received from wages, salary, commissions, bonuses, and tips; self-employment income from own nonfarm or farm businesses, including proprietorships and partnerships; interest, dividends, net rental income, royalty income, or income from estates and trusts; Social Security or Railroad Retirement income; Supplemental Security Income (SSI); any cash public assistance or welfare payments from the state or local welfare office; retirement, survivor, or disability benefits; and any other sources of income received regularly such as Veterans'' (VA) payments, unemployment and/or worker''s compensation, child support, and alimony."}'
|
||||
As OBS_Get_median_income_at_test_point
|
||||
FROM result;
|
||||
|
||||
-- median income at null island
|
||||
WITH result as (
|
||||
SELECT count(_OBS_Get) as expected FROM
|
||||
cdb_observatory._OBS_Get(
|
||||
ST_SetSRID(ST_Point(0, 0), 4326),
|
||||
Array['us.census.acs.B19013001']::text[],
|
||||
'2010 - 2014',
|
||||
'us.census.tiger.block_group'
|
||||
)
|
||||
) select expected = 0 as OBS_Get_median_income_at_null_island
|
||||
from result;
|
||||
|
||||
-- OBS_GetPoints
|
||||
-- obs_getpoints
|
||||
-- --------------------
|
||||
-- {4809.33511352425}
|
||||
|
||||
-- SELECT
|
||||
-- (cdb_observatory._OBS_GetPoints(
|
||||
-- cdb_observatory._TestPoint(),
|
||||
-- 'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- block groups (see _obs_geomtable)
|
||||
-- (Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
|
||||
-- ))[1]::text = '{"value":10923.093200390833950,"name":"Total Population","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'
|
||||
-- as OBS_GetPoints_for_test_point;
|
||||
WITH cte As (
|
||||
SELECT
|
||||
(cdb_observatory._OBS_GetPoints(
|
||||
cdb_observatory._TestPoint(),
|
||||
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- block groups (see _obs_geomtable)
|
||||
(Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
|
||||
))[1]
|
||||
as OBS_GetPoints_for_test_point)
|
||||
SELECT
|
||||
(abs((OBS_GetPoints_for_test_point ->> 'value')::numeric - 10923.093200390833950) / 10923.093200390833950) < 0.001 As OBS_GetPoints_for_test_point_value,
|
||||
(OBS_GetPoints_for_test_point ->> 'name') = 'Total Population' As OBS_GetPoints_for_test_point_name,
|
||||
(OBS_GetPoints_for_test_point ->> 'tablename') = 'obs_1a098da56badf5f32e336002b0a81708c40d29cd' As OBS_GetPoints_for_test_point_tablename,
|
||||
(OBS_GetPoints_for_test_point ->> 'aggregate') = 'sum' As OBS_GetPoints_for_test_point_aggregate,
|
||||
(OBS_GetPoints_for_test_point ->> 'type') = 'Numeric' As OBS_GetPoints_for_test_point_type,
|
||||
(OBS_GetPoints_for_test_point ->> 'description') = 'The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.' As OBS_GetPoints_for_test_point_description
|
||||
FROM cte;
|
||||
|
||||
-- what happens at null island
|
||||
|
||||
SELECT
|
||||
(cdb_observatory._OBS_GetPoints(
|
||||
ST_SetSRID(ST_Point(0, 0), 4326),
|
||||
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- see example in obs_geomtable
|
||||
(Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
|
||||
))[1]::text is null
|
||||
as OBS_GetPoints_for_null_island;
|
||||
|
||||
-- OBS_GetPolygons
|
||||
-- obs_getpolygons
|
||||
-- --------------------
|
||||
-- {12996.8172420752}
|
||||
|
||||
SELECT
|
||||
(cdb_observatory._OBS_GetPolygons(
|
||||
cdb_observatory._TestArea(),
|
||||
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- see example in obs_geomtable
|
||||
Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json]
|
||||
))[1]::text = '{"value":12327.3133495107,"name":"Total Population","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'
|
||||
as OBS_GetPolygons_for_test_point;
|
||||
|
||||
-- see what happens around null island
|
||||
SELECT
|
||||
((cdb_observatory._OBS_GetPolygons(
|
||||
ST_Buffer(ST_SetSRID(ST_Point(0, 0), 4326)::geography, 500)::geometry,
|
||||
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- see example in obs_geomtable
|
||||
Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
|
||||
)[1]->>'value') is null
|
||||
as OBS_GetPolygons_for_null_island;
|
||||
|
||||
SELECT cdb_observatory.OBS_GetSegmentSnapshot(
|
||||
cdb_observatory._TestPoint(),
|
||||
'us.census.tiger.census_tract'
|
||||
)::text =
|
||||
'{"x10_segment":"Wealthy, urban without Kids","x55_segment":"Wealthy transplants displacing long-term local residents","us.census.acs.B01003001_quantile":"0.3235","us.census.acs.B01001002_quantile":"0.494716216216216","us.census.acs.B01001026_quantile":"0.183756756756757","us.census.acs.B01002001_quantile":"0.0752837837837838","us.census.acs.B03002003_quantile":"0.293162162162162","us.census.acs.B03002004_quantile":"0.455527027027027","us.census.acs.B03002006_quantile":"0.656405405405405","us.census.acs.B03002012_quantile":"0.840081081081081","us.census.acs.B05001006_quantile":"0.727135135135135","us.census.acs.B08006001_quantile":"0.688635135135135","us.census.acs.B08006002_quantile":"0.0204459459459459","us.census.acs.B08006008_quantile":"0.679324324324324","us.census.acs.B08006009_quantile":"0.996716216216216","us.census.acs.B08006011_quantile":"0.967418918918919","us.census.acs.B08006015_quantile":"0.512945945945946","us.census.acs.B08006017_quantile":"0.0504864864864865","us.census.acs.B09001001_quantile":"0.192405405405405","us.census.acs.B11001001_quantile":"0.331702702702703","us.census.acs.B14001001_quantile":"0.296283783783784","us.census.acs.B14001002_quantile":"0.045472972972973","us.census.acs.B14001005_quantile":"0.0442702702702703","us.census.acs.B14001006_quantile":"0.0829054054054054","us.census.acs.B14001007_quantile":"0.701135135135135","us.census.acs.B14001008_quantile":"0.404527027027027","us.census.acs.B15003001_quantile":"0.191824324324324","us.census.acs.B15003017_quantile":"0.864162162162162","us.census.acs.B15003022_quantile":"0.754297297297297","us.census.acs.B15003023_quantile":"0.350054054054054","us.census.acs.B16001001_quantile":"0.217635135135135","us.census.acs.B16001002_quantile":"0.85972972972973","us.census.acs.B16001003_quantile":"0.342851351351351","us.census.acs.B17001001_quantile":"0.51204054054054","us.census.acs.B17001002_quantile":"0.813540540540541","us.census.acs.B19013001_quantile":"0.0948648648648649","us.census.acs.B19083001_quantile":"0.678351351351351","us.census.acs.B19301001_quantile":"0.146108108108108","us.census.acs.B25001001_quantile":"0.149067567567568","us.census.acs.B25002003_quantile":"0","us.census.acs.B25004002_quantile":"0","us.census.acs.B25004004_quantile":"0.944554054054054","us.census.acs.B25058001_quantile":"0.398040540540541","us.census.acs.B25071001_quantile":"0.0596081081081081","us.census.acs.B25075001_quantile":"0","us.census.acs.B25075025_quantile":null}' as test_point_segmentation;
|
||||
)::JSONB =
|
||||
'{"x10_segment": "Wealthy, urban without Kids", "x55_segment": "Wealthy transplants displacing long-term local residents", "us.census.acs.B01001002_quantile": "0.494716216216216", "us.census.acs.B01001026_quantile": "0.183756756756757", "us.census.acs.B01002001_quantile": "0.0752837837837838", "us.census.acs.B01003001_quantile": "0.3235", "us.census.acs.B03002003_quantile": "0.293162162162162", "us.census.acs.B03002004_quantile": "0.455527027027027", "us.census.acs.B03002006_quantile": "0.656405405405405", "us.census.acs.B03002012_quantile": "0.840081081081081", "us.census.acs.B05001006_quantile": "0.727135135135135", "us.census.acs.B08006001_quantile": "0.688635135135135", "us.census.acs.B08006002_quantile": "0.0204459459459459", "us.census.acs.B08006009_quantile": "0.679324324324324", "us.census.acs.B08006011_quantile": "0.996716216216216", "us.census.acs.B08006015_quantile": "0.967418918918919", "us.census.acs.B08006017_quantile": "0.512945945945946", "us.census.acs.B08301010_quantile": "0.994743243243243", "us.census.acs.B09001001_quantile": "0.0504864864864865", "us.census.acs.B11001001_quantile": "0.192405405405405", "us.census.acs.B14001001_quantile": "0.331702702702703", "us.census.acs.B14001002_quantile": "0.296283783783784", "us.census.acs.B14001005_quantile": "0.045472972972973", "us.census.acs.B14001006_quantile": "0.0442702702702703", "us.census.acs.B14001007_quantile": "0.0829054054054054", "us.census.acs.B14001008_quantile": "0.701135135135135", "us.census.acs.B15003001_quantile": "0.404527027027027", "us.census.acs.B15003017_quantile": "0.191824324324324", "us.census.acs.B15003022_quantile": "0.864162162162162", "us.census.acs.B15003023_quantile": "0.754297297297297", "us.census.acs.B16001001_quantile": "0.350054054054054", "us.census.acs.B16001002_quantile": "0.217635135135135", "us.census.acs.B16001003_quantile": "0.85972972972973", "us.census.acs.B17001001_quantile": "0.342851351351351", "us.census.acs.B17001002_quantile": "0.51204054054054", "us.census.acs.B19013001_quantile": "0.813540540540541", "us.census.acs.B19083001_quantile": "0.0948648648648649", "us.census.acs.B19301001_quantile": "0.678351351351351", "us.census.acs.B25001001_quantile": "0.146108108108108", "us.census.acs.B25002003_quantile": "0.149067567567568", "us.census.acs.B25004002_quantile": "0", "us.census.acs.B25004004_quantile": "0", "us.census.acs.B25058001_quantile": "0.944554054054054", "us.census.acs.B25071001_quantile": "0.398040540540541", "us.census.acs.B25075001_quantile": "0.0596081081081081", "us.census.acs.B25075025_quantile": "0"}'::JSONB as test_point_segmentation;
|
||||
|
||||
-- segmentation around null island
|
||||
SELECT cdb_observatory.OBS_GetSegmentSnapshot(
|
||||
ST_SetSRID(ST_Point(0, 0), 4326),
|
||||
'us.census.tiger.census_tract'
|
||||
)::text = '{"x10_segment":null,"x55_segment":null,"us.census.acs.B01003001_quantile":null,"us.census.acs.B01001002_quantile":null,"us.census.acs.B01001026_quantile":null,"us.census.acs.B01002001_quantile":null,"us.census.acs.B03002003_quantile":null,"us.census.acs.B03002004_quantile":null,"us.census.acs.B03002006_quantile":null,"us.census.acs.B03002012_quantile":null,"us.census.acs.B05001006_quantile":null,"us.census.acs.B08006001_quantile":null,"us.census.acs.B08006002_quantile":null,"us.census.acs.B08006008_quantile":null,"us.census.acs.B08006009_quantile":null,"us.census.acs.B08006011_quantile":null,"us.census.acs.B08006015_quantile":null,"us.census.acs.B08006017_quantile":null,"us.census.acs.B09001001_quantile":null,"us.census.acs.B11001001_quantile":null,"us.census.acs.B14001001_quantile":null,"us.census.acs.B14001002_quantile":null,"us.census.acs.B14001005_quantile":null,"us.census.acs.B14001006_quantile":null,"us.census.acs.B14001007_quantile":null,"us.census.acs.B14001008_quantile":null,"us.census.acs.B15003001_quantile":null,"us.census.acs.B15003017_quantile":null,"us.census.acs.B15003022_quantile":null,"us.census.acs.B15003023_quantile":null,"us.census.acs.B16001001_quantile":null,"us.census.acs.B16001002_quantile":null,"us.census.acs.B16001003_quantile":null,"us.census.acs.B17001001_quantile":null,"us.census.acs.B17001002_quantile":null,"us.census.acs.B19013001_quantile":null,"us.census.acs.B19083001_quantile":null,"us.census.acs.B19301001_quantile":null,"us.census.acs.B25001001_quantile":null,"us.census.acs.B25002003_quantile":null,"us.census.acs.B25004002_quantile":null,"us.census.acs.B25004004_quantile":null,"us.census.acs.B25058001_quantile":null,"us.census.acs.B25071001_quantile":null,"us.census.acs.B25075001_quantile":null,"us.census.acs.B25075025_quantile":null}' as null_island_segmentation;
|
||||
|
||||
WITH result as (
|
||||
SELECT array_agg(_OBS_GetCategories) as expected FROM
|
||||
cdb_observatory._OBS_GetCategories(
|
||||
cdb_observatory._TestPoint(),
|
||||
Array['us.census.spielman_singleton_segments.X10'],
|
||||
'us.census.tiger.census_tract'
|
||||
)
|
||||
)
|
||||
select (expected)[1]::text = '{"category":"Wealthy, urban without Kids","name":"Spielman-Singleton Segments: 10 Clusters","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"type":"Text","description":"Sociodemographic classes from Spielman and Singleton 2015, 10 clusters"}' as GetCategories_at_test_point_1
|
||||
from result;
|
||||
|
||||
WITH result as (
|
||||
SELECT array_agg(_OBS_GetCategories) as expected FROM
|
||||
cdb_observatory._OBS_GetCategories(
|
||||
ST_SetSRID(ST_Point(0,0), 4326),
|
||||
Array['us.census.spielman_singleton_segments.X10'],
|
||||
'us.census.tiger.census_tract'
|
||||
)
|
||||
)
|
||||
select expected[0] is NULL as GetCategories_at_null_island
|
||||
from result;
|
||||
)::text is null as null_island_segmentation;
|
||||
|
||||
-- Point-based OBS_GetMeasure with zillow
|
||||
SELECT abs(OBS_GetMeasure_zhvi_point - 597900) / 597900 < 5.0 AS OBS_GetMeasure_zhvi_point_test FROM cdb_observatory.OBS_GetMeasure(
|
||||
@ -196,7 +89,7 @@ SELECT (abs(cdb_observatory.OBS_GetMeasure(
|
||||
-- Poly-based OBS_GetMeasure with denominator normalization
|
||||
SELECT abs(cdb_observatory.OBS_GetMeasure(
|
||||
cdb_observatory._TestArea(),
|
||||
'us.census.acs.B01001002', 'denominator') - 0.49026340444793965457) / 0.49026340444793965457 < 0.001 As OBS_GetMeasure_total_male_poly_denominator;
|
||||
'us.census.acs.B01001002', 'denominator', null, '2010 - 2014') - 0.49026340444793965457) / 0.49026340444793965457 < 0.001 As OBS_GetMeasure_total_male_poly_denominator;
|
||||
|
||||
-- Poly-based OBS_GetMeasure with one very bad geom
|
||||
SELECT abs(cdb_observatory.OBS_GetMeasure(
|
||||
@ -219,7 +112,7 @@ SELECT cdb_observatory.OBS_GetCategory(
|
||||
|
||||
-- Poly-based OBS_GetCategory
|
||||
SELECT cdb_observatory.OBS_GetCategory(
|
||||
cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Wealthy, urban without Kids' As obs_getcategory_polygon;
|
||||
cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Hispanic and Young' As obs_getcategory_polygon;
|
||||
|
||||
-- NULL Input OBS_GetCategory
|
||||
SELECT cdb_observatory.OBS_GetCategory(
|
||||
@ -276,7 +169,7 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory(
|
||||
|
||||
-- Area-based OBS_GetUSCensusCategory
|
||||
SELECT cdb_observatory.OBS_GetUSCensusCategory(
|
||||
cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Wealthy, urban without Kids' As OBS_GetUSCensusCategory_polygon;
|
||||
cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Hispanic and Young' As OBS_GetUSCensusCategory_polygon;
|
||||
|
||||
-- Null-input OBS_GetUSCensusCategory
|
||||
SELECT cdb_observatory.OBS_GetUSCensusCategory(
|
||||
@ -323,3 +216,535 @@ SELECT cdb_observatory.OBS_GetMeasureById(
|
||||
'us.census.tiger.block_group',
|
||||
'2010 - 2014'
|
||||
) IS NULL As OBS_GetMeasureById_null_id;
|
||||
|
||||
-- OBS_GetMeta null/null
|
||||
SELECT cdb_observatory.OBS_GetMeta(NULL, NULL) IS NULL
|
||||
AS OBS_GetMeta_null_null_is_null;
|
||||
|
||||
-- OBS_GetMeta null/empty array
|
||||
SELECT cdb_observatory.OBS_GetMeta(NULL, '[]') IS NULL
|
||||
AS OBS_GetMeta_null_empty_is_null;
|
||||
|
||||
-- OBS_GetMeta nullisland/null
|
||||
SELECT cdb_observatory.OBS_GetMeta(ST_Point(0, 0), NULL) IS NULL
|
||||
AS OBS_GetMeta_nullisland_null_is_null;
|
||||
|
||||
-- OBS_GetMeta nullisland/empty array
|
||||
SELECT cdb_observatory.OBS_GetMeta(ST_Point(0, 0), '[]') IS NULL
|
||||
AS OBS_GetMeta_nullisland_empty_is_null;
|
||||
|
||||
-- OBS_GetMeta nullisland/us_measure data
|
||||
SELECT cdb_observatory.OBS_GetMeta(ST_Point(0, 0),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}]') IS NULL
|
||||
AS OBS_GetMeta_nullisland_us_measure_is_null;
|
||||
|
||||
-- OBS_GetMeta for point completes one partial measure with "best" metadata
|
||||
-- with no denominator
|
||||
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}]') meta)
|
||||
SELECT
|
||||
(meta->0->>'id')::integer = 1 id,
|
||||
(meta->0->>'numer_id') = 'us.census.acs.B01003001' numer_id,
|
||||
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->0->>'score_rank')::integer = 1 score_rank,
|
||||
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->0->>'numer_colname') = 'total_pop' numer_colname,
|
||||
(meta->0->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->0->>'numer_name') = 'Total Population' numer_name,
|
||||
(meta->0->>'denom_id') IS NULL denom_id,
|
||||
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
|
||||
(meta->0->>'normalization') IS NULL normalization
|
||||
FROM meta;
|
||||
|
||||
-- OBS_GetMeta for point completes one partial measure with "best" metadata
|
||||
-- with a denominator
|
||||
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01001002"}]') meta)
|
||||
SELECT
|
||||
(meta->0->>'id')::integer = 1 id,
|
||||
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
|
||||
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->0->>'score_rank')::integer = 1 score_rank,
|
||||
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
|
||||
(meta->0->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->0->>'numer_name') = 'Male Population' numer_name,
|
||||
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
|
||||
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
|
||||
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
|
||||
(meta->0->>'denom_type') = 'Numeric' denom_type,
|
||||
(meta->0->>'denom_name') = 'Total Population' denom_name,
|
||||
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
|
||||
(meta->0->>'normalization') IS NULL normalization
|
||||
FROM meta;
|
||||
|
||||
-- OBS_GetMeta for polygon completes one partial measure with "best" metadata
|
||||
-- with no denominator
|
||||
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}]') meta)
|
||||
SELECT
|
||||
(meta->0->>'id')::integer = 1 id,
|
||||
(meta->0->>'numer_id') = 'us.census.acs.B01003001' numer_id,
|
||||
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->0->>'score_rank')::integer = 1 score_rank,
|
||||
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->0->>'numer_colname') = 'total_pop' numer_colname,
|
||||
(meta->0->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->0->>'numer_name') = 'Total Population' numer_name,
|
||||
(meta->0->>'denom_id') IS NULL denom_id,
|
||||
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
|
||||
(meta->0->>'normalization') IS NULL normalization
|
||||
FROM meta;
|
||||
|
||||
-- OBS_GetMeta for polygon completes one partial measure with "best" metadata
|
||||
-- with a denominator
|
||||
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01001002"}]') meta)
|
||||
SELECT
|
||||
(meta->0->>'id')::integer = 1 id,
|
||||
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
|
||||
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->0->>'score_rank')::integer = 1 score_rank,
|
||||
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
|
||||
(meta->0->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->0->>'numer_name') = 'Male Population' numer_name,
|
||||
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
|
||||
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
|
||||
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
|
||||
(meta->0->>'denom_type') = 'Numeric' denom_type,
|
||||
(meta->0->>'denom_name') = 'Total Population' denom_name,
|
||||
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
|
||||
(meta->0->>'normalization') IS NULL normalization
|
||||
FROM meta;
|
||||
|
||||
-- OBS_GetMeta for point completes several partial measures with "best"
|
||||
-- metadata, includes geom alternatives if asked
|
||||
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01001002"}]', null, 2) meta)
|
||||
SELECT
|
||||
(meta->0->>'id')::integer = 1 id,
|
||||
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
|
||||
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->0->>'score_rank')::integer = 1 score_rank,
|
||||
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
|
||||
(meta->0->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->0->>'numer_name') = 'Male Population' numer_name,
|
||||
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
|
||||
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
|
||||
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
|
||||
(meta->0->>'denom_type') = 'Numeric' denom_type,
|
||||
(meta->0->>'denom_name') = 'Total Population' denom_name,
|
||||
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
|
||||
(meta->0->>'normalization') IS NULL normalization,
|
||||
(meta->1->>'id')::integer = 1 id,
|
||||
(meta->1->>'numer_id') = 'us.census.acs.B01001002' numer_id,
|
||||
(meta->1->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->1->>'score_rank')::integer = 2 score_rank,
|
||||
(meta->1->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->1->>'numer_colname') = 'male_pop' numer_colname,
|
||||
(meta->1->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->1->>'numer_name') = 'Male Population' numer_name,
|
||||
(meta->1->>'denom_id') = 'us.census.acs.B01003001' denom_id,
|
||||
(meta->1->>'denom_aggregate') = 'sum' denom_aggregate,
|
||||
(meta->1->>'denom_colname') = 'total_pop' denom_colname,
|
||||
(meta->1->>'denom_type') = 'Numeric' denom_type,
|
||||
(meta->1->>'denom_name') = 'Total Population' denom_name,
|
||||
(meta->1->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
|
||||
(meta->1->>'normalization') IS NULL normalization
|
||||
FROM meta;
|
||||
|
||||
-- OBS_GetMeta for point completes several partial measures with "best" metadata
|
||||
-- with pre-computed geom
|
||||
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01001002", "geom_id": "us.census.tiger.census_tract"}]') meta)
|
||||
SELECT
|
||||
(meta->0->>'id')::integer = 1 id,
|
||||
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
|
||||
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
|
||||
(meta->0->>'score_rank')::integer = 1 score_rank,
|
||||
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
|
||||
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
|
||||
(meta->0->>'numer_type') = 'Numeric' numer_type,
|
||||
(meta->0->>'numer_name') = 'Male Population' numer_name,
|
||||
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
|
||||
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
|
||||
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
|
||||
(meta->0->>'denom_type') = 'Numeric' denom_type,
|
||||
(meta->0->>'denom_name') = 'Total Population' denom_name,
|
||||
(meta->0->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
|
||||
(meta->0->>'normalization') IS NULL normalization
|
||||
FROM meta;
|
||||
|
||||
-- OBS_GetMeta for point completes several partial measures with conflicting
|
||||
-- metadata
|
||||
SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01001002", "denom_id": "us.census.acs.B01001002", "geom_id": "us.census.tiger.census_tract"}]') IS NULL
|
||||
AS obs_getmeta_conflicting_metadata;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by id with empty list/null
|
||||
WITH data AS (SELECT * FROM cdb_observatory.OBS_GetData(ARRAY[]::TEXT[], null))
|
||||
SELECT ARRAY_AGG(data) IS NULL AS obs_getdata_geomval_empty_null FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with empty list/null
|
||||
WITH data AS (SELECT * FROM cdb_observatory.OBS_GetData(ARRAY[]::GEOMVAL[], null))
|
||||
SELECT ARRAY_AGG(data) IS NULL AS obs_getdata_text_empty_null FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with empty list
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(ARRAY[]::GEOMVAL[],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT ARRAY_AGG(data) IS NULL AS obs_getdata_geomval_empty_one_measure FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by point geom with one standard measure NULL
|
||||
-- normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 10923) / 10923 < 0.001 data_point_measure_null,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by polygon geom with one standard measure NULL
|
||||
-- normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_null,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by point geom with one standard measure area
|
||||
-- normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "area"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 10923) / 10923 < 0.001 data_point_measure_area,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by polygon geom with one standard measure area
|
||||
-- normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "area"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_area,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by point geom with one standard measure predenom
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "predenominated"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 1900) / 1900 < 0.001 data_point_measure_predenominated,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by polygon geom with one standard measure predenom
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "predenominated"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 12327) / 12327 < 0.001 data_polygon_measure_predenominated,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by point geom with impossible denom
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "denominated"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
data->0->>'value' IS NULL data_point_measure_impossible_denominated,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by polygon geom with one impossible denom
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "denominated"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
data->0->>'value' IS NULL data_polygon_measure_impossible_denominated,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by point geom with denom
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.acs.B01001002", "normalization": "denominated"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 0.6215) / 0.6215 < 0.001 data_point_measure_denominated,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by polygon geom with one denom measure
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01001002", "normalization": "denominated"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_denominated,
|
||||
data->1 IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with two standard measures NULL normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001"}, {"numer_id": "us.census.acs.B01001002"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_one_null,
|
||||
abs((data->1->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_two_null
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with two standard measures predenom normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "predenom"}, {"numer_id": "us.census.acs.B01001002", "normalization": "predenom"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 12327) / 12327 < 0.001 data_polygon_measure_one_predenom,
|
||||
abs((data->1->>'value')::Numeric - 6043) / 6043 < 0.001 data_polygon_measure_two_predenom
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with two standard measures area normalization
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "normalization": "area"}, {"numer_id": "us.census.acs.B01001002", "normalization": "area"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_one_area,
|
||||
abs((data->1->>'value')::Numeric - 7739) / 7739 < 0.001 data_polygon_measure_two_area
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with two standard measures different geoms
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.census_tract"}, {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
abs((data->0->>'value')::Numeric - 16960) / 16960 < 0.001 data_polygon_measure_tract,
|
||||
abs((data->1->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_bg
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by point geom with one categorical
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
|
||||
'[{"numer_id": "us.census.spielman_singleton_segments.X55"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
data->0->>'value' = 'Wealthy transplants displacing long-term local residents' data_point_categorical,
|
||||
data->1->>'value' IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by polygon geom with one categorical
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.spielman_singleton_segments.X55"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
data->0->>'value' = 'Hispanic Black mix multilingual, high poverty, renters, uses public transport' data_poly_categorical,
|
||||
data->1->>'value' IS NULL nullcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with one categorical and one measure
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"numer_id": "us.census.spielman_singleton_segments.X55"}, {"numer_id": "us.census.acs.B01003001"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = 1 id,
|
||||
data->0->>'value' = 'Hispanic Black mix multilingual, high poverty, renters, uses public transport' data_poly_categorical,
|
||||
abs((data->1->>'value')::Numeric - 15787) / 15787 < 0.0001 valcol
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with polygons inside a polygon
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.block_group"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta), false))
|
||||
SELECT every(id = 1) is TRUE id,
|
||||
count(distinct (data->0->>'value')::geometry) = 16 correct_num_geoms
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with polygons inside a polygon + one measure
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.block_group"}, {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta), false))
|
||||
SELECT every(id = 1) is TRUE id,
|
||||
count(distinct (data->0->>'value')::geometry) = 16 correct_num_geoms,
|
||||
abs(sum((data->1->>'value')::numeric) - 15787) / 15787 < 0.001 correct_pop
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with polygons inside a polygon + one measure + one text
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.block_group"}, {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group"}, {"numer_id": "us.census.tiger.name", "geom_id": "us.census.tiger.block_group"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta), false))
|
||||
SELECT every(id = 1) is TRUE id,
|
||||
count(distinct (data->0->>'value')::geometry) = 16 correct_num_geoms,
|
||||
abs(sum((data->1->>'value')::numeric) - 15787) / 15787 < 0.001 correct_pop,
|
||||
array_agg(distinct data->2->>'value') = '{"Block Group 1","Block Group 2","Block Group 3","Block Group 4","Block Group 5"}' correct_bg_names
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with points inside a polygon
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.pointlm_geom"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta), false))
|
||||
SELECT every(id = 1) AS id,
|
||||
count(distinct (data->0->>'value')::geometry(point, 4326)) = 3 correct_num_points
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by geom with points inside a polygon + one text
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.pointlm_geom"}, {"geom_id": "us.census.tiger.pointlm_geom", "numer_id": "us.census.tiger.fullname"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
|
||||
(SELECT meta FROM meta), false))
|
||||
SELECT every(id = 1) AS id,
|
||||
count(distinct (data->0->>'value')::geometry(point, 4326)) = 3 correct_num_points,
|
||||
array_agg(data->1->>'value') = '{"Bushwick Yards","Edward Block Square","Bushwick Houses"}' pointgeom_names
|
||||
FROM data;
|
||||
|
||||
|
||||
-- OBS_GetData by id with one standard measure
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01003001"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY['36047048500'],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = '36047048500' AS id,
|
||||
(abs((data->0->>'value')::numeric) - 5578) / 5578 < 0.001 obs_getdata_by_id_one_measure_null
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData by id with one standard measure, predenominated
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"normalization": "predenominated", "geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01003001"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY['36047048500'],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = '36047048500' AS id,
|
||||
(abs((data->0->>'value')::numeric) - 3241) / 3241 < 0.001 obs_getdata_by_id_one_measure_predenom
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by id with two standard measures
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01003001"}, {"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01001002"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY['36047048500'],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = '36047048500' AS id,
|
||||
(abs((data->0->>'value')::numeric) - 5578) / 5578 < 0.001 obs_getdata_by_id_one_measure_null,
|
||||
(abs((data->1->>'value')::numeric) - 0.6053) / 0.6053 < 0.001 obs_getdata_by_id_two_measure_null
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by id with one categorical
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.spielman_singleton_segments.X55"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY['36047048500'],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = '36047048500' AS id,
|
||||
data->0->>'value' = 'Wealthy transplants displacing long-term local residents' obs_getdata_by_id_categorical
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData/OBS_GetMeta by id with one geometry
|
||||
WITH
|
||||
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
|
||||
'[{"geom_id": "us.census.tiger.census_tract"}]') meta),
|
||||
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
|
||||
ARRAY['36047048500'],
|
||||
(SELECT meta FROM meta)))
|
||||
SELECT id = '36047048500' AS id,
|
||||
ST_GeometryType((data->0->>'value')::geometry) = 'ST_MultiPolygon' obs_getdata_by_id_geometry
|
||||
FROM data;
|
||||
|
||||
-- OBS_GetData with an API + geomvals, no args
|
||||
SELECT ARRAY['us.census.tiger.census_tract'] <@ array_agg(data->0->>'value') AS OBS_GetData_API_geomvals_no_args
|
||||
FROM cdb_observatory.obs_getdata(array[(cdb_observatory._testarea(), 1)::geomval],
|
||||
'[{"numer_type": "text", "numer_colname": "boundary_id", "api_method": "obs_getavailableboundaries", "geom_geomref_colname": "boundary_id"}]',
|
||||
false);
|
||||
|
||||
-- OBS_GetData with an API + geomvals, args, numeric
|
||||
SELECT json_typeof(data->0->'value') = 'number' AS OBS_GetData_API_geomvals_args_numer_return
|
||||
FROM cdb_observatory.obs_getdata(array[(cdb_observatory._testarea(), 1)::geomval],
|
||||
'[{"numer_type": "numeric", "numer_colname": "obs_getmeasure", "api_method": "obs_getmeasure", "api_args": ["us.census.acs.B01003001"]}]', false);
|
||||
|
||||
-- OBS_GetData with an API + geomvals, args, text
|
||||
SELECT json_typeof(data->0->'value') = 'string' AS OBS_GetData_API_geomvals_args_string_return
|
||||
FROM cdb_observatory.obs_getdata(array[(cdb_observatory._testarea(), 1)::geomval],
|
||||
'[{"numer_type": "text", "numer_colname": "obs_getcategory", "api_method": "obs_getcategory", "api_args": ["us.census.spielman_singleton_segments.X55"]}]', false);
|
||||
|
||||
-- OBS_GetData with an API + geomrefs, args, numeric
|
||||
SELECT json_typeof(data->0->'value') = 'number' AS OBS_GetData_API_geomrefs_args_numer_return
|
||||
FROM cdb_observatory.obs_getdata(array['36047076200'],
|
||||
'[{"numer_type": "numeric", "numer_colname": "obs_getmeasurebyid", "api_method": "obs_getmeasurebyid", "api_args": ["us.census.acs.B01003001", "us.census.tiger.census_tract"]}]');
|
||||
|
||||
-- OBS_GetData with an API + geomrefs, args, text
|
||||
SELECT json_typeof(data->0->'value') = 'string' AS OBS_GetData_API_geomrefs_args_string_return
|
||||
FROM cdb_observatory.obs_getdata(array['36047'],
|
||||
'[{"numer_type": "text", "numer_colname": "obs_getboundarybyid", "api_method": "obs_getboundarybyid", "api_args": ["us.census.tiger.county"]}]');
|
||||
|
||||
@ -10,11 +10,11 @@ SET client_min_messages TO WARNING;
|
||||
|
||||
-- _OBS_SearchTables tests
|
||||
SELECT
|
||||
t.table_name = 'obs_1babf5a26a1ecda5fb74963e88408f71d0364b81' As _OBS_SearchTables_tables_match,
|
||||
t.timespan = '2014' As _OBS_SearchTables_timespan_matches
|
||||
t.table_name = 'obs_0310c639744a2014bb1af82709228f05b59e7d3d' As _OBS_SearchTables_tables_match,
|
||||
t.timespan = '2015' As _OBS_SearchTables_timespan_matches
|
||||
FROM cdb_observatory._OBS_SearchTables(
|
||||
'us.census.tiger.county',
|
||||
'2014'
|
||||
'2015'
|
||||
) As t(table_name, timespan);
|
||||
|
||||
-- _OBS_SearchTables tests
|
||||
@ -278,9 +278,9 @@ AS _obs_getavailablegeometries_foobarbaz_denom_not_in_usa_bg;
|
||||
SELECT 'us.census.tiger.block_group' IN (SELECT geom_id
|
||||
FROM cdb_observatory.OBS_GetAvailableGeometries(
|
||||
ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
|
||||
NULL, NULL, NULL, '2014'
|
||||
NULL, NULL, NULL, '2015'
|
||||
) WHERE valid_timespan = True)
|
||||
AS _obs_getavailablegeometries_bg_2014;
|
||||
AS _obs_getavailablegeometries_bg_2015;
|
||||
|
||||
SELECT 'us.census.tiger.block_group' NOT IN (SELECT geom_id
|
||||
FROM cdb_observatory.OBS_GetAvailableGeometries(
|
||||
@ -350,126 +350,140 @@ AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014;
|
||||
-- _OBS_GetGeometryScores tests
|
||||
--
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.county', 'us.census.tiger.zcta5']
|
||||
AS _obs_geometryscores_500m_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.county', 'us.census.tiger.zcta5']);
|
||||
'us.census.tiger.county', 'us.census.tiger.zcta5'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.county', 'us.census.tiger.zcta5']
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']
|
||||
AS _obs_geometryscores_5km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.county', 'us.census.tiger.zcta5']);
|
||||
'us.census.tiger.county', 'us.census.tiger.zcta5'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.block_group',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']
|
||||
AS _obs_geometryscores_50km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY[ 'us.census.tiger.zcta5', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.county', 'us.census.tiger.block_group']
|
||||
'us.census.tiger.county', 'us.census.tiger.block_group' ]
|
||||
AS _obs_geometryscores_500km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
|
||||
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
|
||||
AS _obs_geometryscores_2500km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
|
||||
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text
|
||||
= '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }'
|
||||
AS _obs_geometryscores_numgeoms_500m_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
|
||||
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 880, "us.census.tiger.census_tract" : 310, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
|
||||
AS _obs_geometryscores_numgeoms_5km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 13 }'
|
||||
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 11531, "us.census.tiger.census_tract" : 3601, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }'
|
||||
AS _obs_geometryscores_numgeoms_50km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 48415, "us.census.tiger.census_tract" : 15776, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 295 }'
|
||||
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 48917, "us.census.tiger.census_tract" : 15969, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 314 }'
|
||||
AS _obs_geometryscores_numgeoms_500km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 165489, "us.census.tiger.census_tract" : 55152, "us.census.tiger.zcta5" : 26500, "us.census.tiger.county" : 2551 }'
|
||||
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
|
||||
'{ "us.census.tiger.block_group" : 169191, "us.census.tiger.census_tract" : 56469, "us.census.tiger.zcta5" : 26525, "us.census.tiger.county" : 2753 }'
|
||||
AS _obs_geometryscores_numgeoms_2500km_buffer
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'])
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.county', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.block_group']
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
|
||||
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
|
||||
AS _obs_geometryscores_500km_buffer_50_geoms
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 50);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 50)
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC)
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC)
|
||||
= ARRAY['us.census.tiger.zcta5', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.block_group', 'us.census.tiger.county']
|
||||
'us.census.tiger.county', 'us.census.tiger.block_group']
|
||||
AS _obs_geometryscores_500km_buffer_500_geoms
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 500);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 500)
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.block_group',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']
|
||||
AS _obs_geometryscores_500km_buffer_2500_geoms
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500)
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
|
||||
SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county']
|
||||
AS _obs_geometryscores_500km_buffer_25000_geoms
|
||||
FROM cdb_observatory._OBS_GetGeometryScores(
|
||||
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
|
||||
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000);
|
||||
'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000)
|
||||
WHERE table_id LIKE '%2015%';
|
||||
|
||||
--
|
||||
-- OBS_LegacyBuilderMetadata tests
|
||||
|
||||
@ -42,7 +42,7 @@ SELECT cdb_observatory.OBS_GetBoundary(
|
||||
SELECT cdb_observatory.OBS_GetBoundary(
|
||||
cdb_observatory._TestPoint(),
|
||||
'us.census.tiger.census_tract',
|
||||
'2014'
|
||||
'2015'
|
||||
) = :'cartodb_census_tract_geometry' As OBS_GetBoundary_year_census_tract;
|
||||
|
||||
-- should return null
|
||||
@ -65,7 +65,7 @@ SELECT cdb_observatory.OBS_GetBoundaryId(
|
||||
SELECT cdb_observatory.OBS_GetBoundaryId(
|
||||
cdb_observatory._TestPoint(),
|
||||
'us.census.tiger.census_tract',
|
||||
'2014'
|
||||
'2015'
|
||||
) = '36047048500'::text As OBS_GetBoundaryId_cartodb_census_tract_with_year;
|
||||
|
||||
-- should give back '36047', the geoid of cartodb's county (King's/
|
||||
@ -73,7 +73,7 @@ SELECT cdb_observatory.OBS_GetBoundaryId(
|
||||
SELECT cdb_observatory.OBS_GetBoundaryId(
|
||||
cdb_observatory._TestPoint(),
|
||||
'us.census.tiger.county',
|
||||
'2014'
|
||||
'2015'
|
||||
) = '36047'::text As OBS_GetBoundaryId_cartodb_county_with_year;
|
||||
|
||||
-- should give back null since there is not a census tract at null island
|
||||
@ -104,7 +104,7 @@ SELECT cdb_observatory.OBS_GetBoundaryById(
|
||||
SELECT cdb_observatory.OBS_GetBoundaryById(
|
||||
'36047',
|
||||
'us.census.tiger.county',
|
||||
'2014'
|
||||
'2015'
|
||||
) = :'cartodb_county_geometry' OBS_GetBoundaryById_boundary_id_mismatch_geom_id;
|
||||
|
||||
-- should give null since boundary_id does not match geometry reference id
|
||||
@ -115,6 +115,34 @@ SELECT cdb_observatory.OBS_GetBoundaryById(
|
||||
|
||||
-- _OBS_GetBoundariesByGeometry
|
||||
|
||||
SELECT array_agg(geom_refs) = Array[ '1104486618765', '1104486642837',
|
||||
'1104991798384', '1105044325367',
|
||||
'1105089330200', '1105089331758']
|
||||
As _OBS_GetBoundariesByGeometry_roads_around_cartodb
|
||||
FROM (
|
||||
SELECT *
|
||||
FROM cdb_observatory._OBS_GetBoundariesByGeometry(
|
||||
-- near CartoDB's office
|
||||
ST_MakeEnvelope(-74,40.69,-73.99,40.7,
|
||||
4326),
|
||||
'us.census.tiger.prisecroads_geom')
|
||||
ORDER BY geom_refs ASC
|
||||
) As m(the_geom, geom_refs);
|
||||
|
||||
SELECT
|
||||
array_agg(geom_refs) = Array['1102654301684', '1102654307106',
|
||||
'1102654326686', '1102654351507' ]
|
||||
As _OBS_GetBoundariesByGeometry_points_around_cartodb
|
||||
FROM (
|
||||
SELECT *
|
||||
FROM cdb_observatory._OBS_GetBoundariesByGeometry(
|
||||
-- near CartoDB's office
|
||||
ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,
|
||||
4326),
|
||||
'us.census.tiger.pointlm_geom')
|
||||
ORDER BY geom_refs ASC
|
||||
) As m(the_geom, geom_refs);
|
||||
|
||||
-- check that all census tracts intersecting with the geometry are returned
|
||||
-- order them to ensure that the same values are returned
|
||||
SELECT
|
||||
@ -270,7 +298,7 @@ FROM (
|
||||
-73.9280319214,40.7101254524,
|
||||
4326),
|
||||
'us.census.tiger.census_tract',
|
||||
'2014')
|
||||
'2015')
|
||||
ORDER BY geom_refs ASC
|
||||
) As m(the_geom, geom_refs);
|
||||
|
||||
@ -313,7 +341,7 @@ FROM (
|
||||
cdb_observatory._testpoint(),
|
||||
500,
|
||||
'us.census.tiger.census_tract',
|
||||
'2014')
|
||||
'2015')
|
||||
ORDER BY geom_refs ASC
|
||||
) As m(the_geom, geom_refs);
|
||||
|
||||
@ -330,21 +358,4 @@ FROM (
|
||||
ORDER BY geom_refs ASC
|
||||
) As m(the_geom, geom_refs);
|
||||
|
||||
-- _OBS_GetGeometryMetadata
|
||||
-- get metadata for census tracts
|
||||
|
||||
SELECT
|
||||
geoid_colname = 'geoid' As geoid_name_matches,
|
||||
target_table = 'obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7' As table_name_matches,
|
||||
geom_colname = 'the_geom' As geom_name_matches
|
||||
FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract')
|
||||
As m(geoid_colname, target_table, geom_colname);
|
||||
|
||||
-- get metadata for boundaries with clipped geometries
|
||||
SELECT
|
||||
geoid_colname = 'geoid' As geoid_name_matches,
|
||||
target_table = 'obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963' As table_name_matches,
|
||||
geom_colname = 'the_geom' As geom_name_matches
|
||||
FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract_clipped') As m(geoid_colname, target_table, geom_colname);
|
||||
|
||||
\i test/fixtures/drop_fixtures.sql
|
||||
|
||||
@ -65,7 +65,14 @@ SKIP_COLUMNS = set([
|
||||
u'mx.inegi_columns.VIV41',
|
||||
u'mx.inegi_columns.VIV40',
|
||||
u'mx.inegi_columns.VIV17',
|
||||
u'mx.inegi_columns.EDU10'
|
||||
u'mx.inegi_columns.VIV25',
|
||||
u'mx.inegi_columns.EDU10',
|
||||
u'whosonfirst.wof_disputed_name',
|
||||
u'us.census.tiger.fullname',
|
||||
u'whosonfirst.wof_marinearea_name',
|
||||
u'us.census.tiger.mtfcc',
|
||||
u'whosonfirst.wof_county_name',
|
||||
u'whosonfirst.wof_region_name',
|
||||
])
|
||||
|
||||
#def default_geometry_id(column_id):
|
||||
|
||||
@ -23,11 +23,11 @@ for q in (
|
||||
ST_Translate(the_geom, -0.1, 0.1) offset_geom,
|
||||
geom_refs AS name
|
||||
FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
|
||||
st_makeenvelope(-74.05437469482422,40.66319159533881,
|
||||
-73.81885528564453,40.745696344339564, 4326),
|
||||
st_makeenvelope(-74.1, 40.5,
|
||||
-73.8, 40.9, 4326),
|
||||
'us.census.tiger.census_tract_clipped')) foo
|
||||
ORDER BY ST_NPoints(the_geom) ASC
|
||||
LIMIT 500''',
|
||||
LIMIT 1000''',
|
||||
'DROP TABLE IF EXISTS obs_perftest_complex',
|
||||
'''CREATE TABLE obs_perftest_complex (cartodb_id SERIAL PRIMARY KEY,
|
||||
point GEOMETRY,
|
||||
@ -45,25 +45,57 @@ for q in (
|
||||
'us.census.tiger.county_clipped')) foo
|
||||
ORDER BY ST_NPoints(the_geom) DESC
|
||||
LIMIT 50;''',
|
||||
'DROP TABLE IF EXISTS obs_perftest_country_simple',
|
||||
'''CREATE TABLE obs_perftest_country_simple (cartodb_id SERIAL PRIMARY KEY,
|
||||
geom GEOMETRY,
|
||||
name TEXT) ''',
|
||||
'''INSERT INTO obs_perftest_country_simple (geom, name)
|
||||
SELECT the_geom geom,
|
||||
geom_refs AS name
|
||||
FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
|
||||
st_makeenvelope(-179,-89, 179,89, 4326),
|
||||
'whosonfirst.wof_country_geom')) foo
|
||||
ORDER BY ST_NPoints(the_geom) ASC
|
||||
LIMIT 50;''',
|
||||
'DROP TABLE IF EXISTS obs_perftest_country_complex',
|
||||
'''CREATE TABLE obs_perftest_country_complex (cartodb_id SERIAL PRIMARY KEY,
|
||||
geom GEOMETRY,
|
||||
name TEXT) ''',
|
||||
'''INSERT INTO obs_perftest_country_complex (geom, name)
|
||||
SELECT the_geom geom,
|
||||
geom_refs AS name
|
||||
FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
|
||||
st_makeenvelope(-179,-89, 179,89, 4326),
|
||||
'whosonfirst.wof_country_geom')) foo
|
||||
ORDER BY ST_NPoints(the_geom) DESC
|
||||
LIMIT 50;''',
|
||||
#'''SET statement_timeout = 5000;'''
|
||||
):
|
||||
query(q.format(
|
||||
q_formatted = q.format(
|
||||
schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
))
|
||||
)
|
||||
resp = query(q_formatted)
|
||||
if q.lower().startswith('insert'):
|
||||
if resp.rowcount == 0:
|
||||
raise Exception('''Performance fixture creation "{}" inserted 0 rows,
|
||||
this will break tests. Check the query to determine
|
||||
what is going wrong.'''.format(q_formatted))
|
||||
commit()
|
||||
|
||||
|
||||
ARGS = {
|
||||
('OBS_GetMeasureByID', None): "name, 'us.census.acs.B01001002', '{}'",
|
||||
('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001'",
|
||||
('OBS_GetMeasure', 'area'): "{}, 'us.census.acs.B01001002', 'area'",
|
||||
('OBS_GetMeasure', 'denominator'): "{}, 'us.census.acs.B01001002', 'denominator'",
|
||||
('OBS_GetCategory', None): "{}, 'us.census.spielman_singleton_segments.X10'",
|
||||
('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001', null, {}",
|
||||
('OBS_GetMeasure', 'area'): "{}, 'us.census.acs.B01001002', 'area', {}",
|
||||
('OBS_GetMeasure', 'denominator'): "{}, 'us.census.acs.B01001002', 'denominator', {}",
|
||||
('OBS_GetCategory', None): "{}, 'us.census.spielman_singleton_segments.X10', {}",
|
||||
('_OBS_GetGeometryScores', None): "{}, NULL"
|
||||
}
|
||||
|
||||
|
||||
def record(params, results):
|
||||
sha = os.environ['OBS_EXTENSION_SHA']
|
||||
msg = os.environ.get('OBS_EXTENSION_MSG')
|
||||
fpath = os.path.join(os.environ['OBS_PERFTEST_DIR'], sha + '.json')
|
||||
if os.path.isfile(fpath):
|
||||
tests = json.load(open(fpath, 'r'))
|
||||
@ -76,53 +108,129 @@ def record(params, results):
|
||||
}
|
||||
json.dump(tests, fhandle)
|
||||
|
||||
@parameterized([
|
||||
('simple', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('simple', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('simple', '_OBS_GetGeometryScores', 'NULL', 3000),
|
||||
|
||||
('complex', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('complex', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('complex', '_OBS_GetGeometryScores', 'NULL', 3000),
|
||||
|
||||
('country_simple', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('country_simple', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000),
|
||||
|
||||
('country_complex', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('country_complex', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000),
|
||||
])
|
||||
def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms):
|
||||
print api_method, geom_complexity, filters, target_geoms
|
||||
|
||||
rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,)
|
||||
results = []
|
||||
for rows in rownums:
|
||||
stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms})
|
||||
FROM obs_perftest_{complexity}
|
||||
WHERE cartodb_id <= {n}'''.format(
|
||||
complexity=geom_complexity,
|
||||
schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
api_method=api_method,
|
||||
filters=filters,
|
||||
target_geoms=target_geoms,
|
||||
n=rows)
|
||||
start = time()
|
||||
query(stmt)
|
||||
end = time()
|
||||
qps = (rows / (end - start))
|
||||
results.append({
|
||||
'rows': rows,
|
||||
'qps': qps,
|
||||
'stmt': stmt
|
||||
})
|
||||
print rows, ': ', qps, ' QPS'
|
||||
|
||||
if 'OBS_RECORD_TEST' in os.environ:
|
||||
record({
|
||||
'geom_complexity': geom_complexity,
|
||||
'api_method': api_method,
|
||||
'filters': filters,
|
||||
'target_geoms': target_geoms
|
||||
}, results)
|
||||
|
||||
@parameterized([
|
||||
('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract'),
|
||||
('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county'),
|
||||
('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None),
|
||||
('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None),
|
||||
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'point'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'geom'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'point'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'geom'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'offset_geom'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'point'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'geom'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom'),
|
||||
('simple', 'OBS_GetCategory', None, 'point'),
|
||||
('simple', 'OBS_GetCategory', None, 'geom'),
|
||||
('simple', 'OBS_GetCategory', None, 'offset_geom'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
|
||||
('simple', 'OBS_GetCategory', None, 'point', 'NULL'),
|
||||
('simple', 'OBS_GetCategory', None, 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
|
||||
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'point'),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'geom'),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'point'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'geom'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'offset_geom'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'point'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'geom'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom'),
|
||||
('complex', 'OBS_GetCategory', None, 'point'),
|
||||
('complex', 'OBS_GetCategory', None, 'geom'),
|
||||
('complex', 'OBS_GetCategory', None, 'offset_geom'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
|
||||
('complex', 'OBS_GetCategory', None, 'point', 'NULL'),
|
||||
('complex', 'OBS_GetCategory', None, 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
|
||||
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
|
||||
('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
|
||||
('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
])
|
||||
def test_performance(geom_complexity, api_method, normalization, geom):
|
||||
print api_method, geom_complexity, normalization, geom
|
||||
def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary):
|
||||
print api_method, geom_complexity, normalization, geom, boundary
|
||||
col = 'measure' if 'measure' in api_method.lower() else 'category'
|
||||
results = []
|
||||
|
||||
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50 )
|
||||
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, )
|
||||
for rows in rownums:
|
||||
stmt = '''UPDATE obs_perftest_{complexity}
|
||||
SET {col} = {schema}{api_method}({args})
|
||||
WHERE cartodb_id < {n}'''.format(
|
||||
WHERE cartodb_id <= {n}'''.format(
|
||||
col=col,
|
||||
complexity=geom_complexity,
|
||||
schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
api_method=api_method,
|
||||
args=ARGS[api_method, normalization].format(geom),
|
||||
n=rows+1)
|
||||
args=ARGS[api_method, normalization].format(geom, boundary),
|
||||
n=rows)
|
||||
start = time()
|
||||
query(stmt)
|
||||
end = time()
|
||||
@ -139,5 +247,104 @@ def test_performance(geom_complexity, api_method, normalization, geom):
|
||||
'geom_complexity': geom_complexity,
|
||||
'api_method': api_method,
|
||||
'normalization': normalization,
|
||||
'boundary': boundary,
|
||||
'geom': geom
|
||||
}, results)
|
||||
|
||||
|
||||
@parameterized([
|
||||
('simple', 'predenominated', 'point', 'null'),
|
||||
('simple', 'predenominated', 'geom', 'null'),
|
||||
('simple', 'predenominated', 'offset_geom', 'null'),
|
||||
('simple', 'area', 'point', 'null'),
|
||||
('simple', 'area', 'geom', 'null'),
|
||||
('simple', 'area', 'offset_geom', 'null'),
|
||||
('simple', 'denominator', 'point', 'null'),
|
||||
('simple', 'denominator', 'geom', 'null'),
|
||||
('simple', 'denominator', 'offset_geom', 'null'),
|
||||
|
||||
('simple', 'predenominated', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'predenominated', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'area', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'area', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'area', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'denominator', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'denominator', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
|
||||
('complex', 'predenominated', 'point', 'null'),
|
||||
('complex', 'predenominated', 'geom', 'null'),
|
||||
('complex', 'predenominated', 'offset_geom', 'null'),
|
||||
('complex', 'area', 'point', 'null'),
|
||||
('complex', 'area', 'geom', 'null'),
|
||||
('complex', 'area', 'offset_geom', 'null'),
|
||||
('complex', 'denominator', 'point', 'null'),
|
||||
('complex', 'denominator', 'geom', 'null'),
|
||||
('complex', 'denominator', 'offset_geom', 'null'),
|
||||
|
||||
('complex', 'predenominated', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'predenominated', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'predenominated', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'area', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'area', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'area', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'denominator', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'denominator', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'denominator', 'offset_geom', "'us.census.tiger.county'"),
|
||||
])
|
||||
def test_getmeasure_split_performance(geom_complexity, normalization, geom, boundary):
|
||||
print geom_complexity, normalization, geom, boundary
|
||||
results = []
|
||||
|
||||
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (10, 50, 100)
|
||||
for rows in rownums:
|
||||
stmt = '''
|
||||
with data as (
|
||||
SELECT id, data FROM {schema}OBS_GetData(
|
||||
(SELECT array_agg(({geom}, cartodb_id)::geomval)
|
||||
FROM obs_perftest_{complexity}
|
||||
WHERE cartodb_id <= {n}),
|
||||
(SELECT {schema}OBS_GetMeta(
|
||||
(SELECT st_setsrid(st_extent({geom}), 4326)
|
||||
FROM obs_perftest_{complexity}
|
||||
WHERE cartodb_id <= {n}),
|
||||
'[{{
|
||||
"numer_id": "us.census.acs.B01001002",
|
||||
"normalization": "{normalization}",
|
||||
"geom_id": {boundary}
|
||||
}}]'::JSON
|
||||
))
|
||||
))
|
||||
UPDATE obs_perftest_{complexity}
|
||||
SET measure = (data->0->>'value')::Numeric
|
||||
FROM data
|
||||
WHERE obs_perftest_{complexity}.cartodb_id = data.id
|
||||
;
|
||||
'''.format(
|
||||
point_or_poly='point' if geom == 'point' else 'polygon',
|
||||
complexity=geom_complexity,
|
||||
schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
normalization=normalization,
|
||||
geom=geom,
|
||||
boundary=boundary.replace("'", '"'),
|
||||
n=rows)
|
||||
start = time()
|
||||
query(stmt)
|
||||
end = time()
|
||||
qps = (rows / (end - start))
|
||||
results.append({
|
||||
'rows': rows,
|
||||
'qps': qps,
|
||||
'stmt': stmt
|
||||
})
|
||||
print rows, ': ', qps, ' QPS'
|
||||
|
||||
if 'OBS_RECORD_TEST' in os.environ:
|
||||
record({
|
||||
'geom_complexity': geom_complexity,
|
||||
'api_method': 'OBS_GetData',
|
||||
'normalization': normalization,
|
||||
'boundary': boundary,
|
||||
'geom': geom
|
||||
}, results)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user