cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

keep track of table_id in obs_meta and geometryscores, use obs_getmeasure*multi for obs_getmeasure

Browse Source
This commit is contained in:
John Krauss 2016-12-21 21:53:53 +00:00
parent 24587b7e03
commit d3a57e637c
10 changed files with 6746 additions and 4414 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/generate_fixtures.py
View File

@ -168,10 +168,11 @@ FIXTURES = [
('us.census.acs.B19001015', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001015', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001016', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001016', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001017', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001017', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2015'),
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'), ('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'),
('us.census.tiger.block_group', 'us.census.tiger.block_group', '2014'), ('us.census.tiger.block_group', 'us.census.tiger.block_group', '2015'),
('us.census.tiger.zcta5', 'us.census.tiger.zcta5', '2014'), ('us.census.tiger.zcta5', 'us.census.tiger.zcta5', '2015'),
('us.census.tiger.county', 'us.census.tiger.county', '2014'), ('us.census.tiger.county', 'us.census.tiger.county', '2015'),
('us.census.acs.B01001002', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B01001002', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
@ -204,6 +205,7 @@ def dump(cols, tablename, where=''):
' | sed "s:SET search_path.*::" ' ' | sed "s:SET search_path.*::" '
' | sed "s:CREATE TABLE :CREATE TABLE observatory.:" ' ' | sed "s:CREATE TABLE :CREATE TABLE observatory.:" '
' | sed "s:ALTER TABLE.*OWNER.*::" ' ' | sed "s:ALTER TABLE.*OWNER.*::" '
' | sed "s:SET idle_in_transaction_session_timeout.*::" '
' >> {outfile}'.format( ' >> {outfile}'.format(
tablename=tablename, tablename=tablename,
outfile=OUTFILE_PATH, outfile=OUTFILE_PATH,

389
src/pg/sql/41_observatory_augmentation.sql
View File

@ -361,7 +361,7 @@ BEGIN
(unnest($3))->>'geom_id' geom_id, (unnest($3))->>'geom_id' geom_id,
(unnest($3))->>'timespan' timespan (unnest($3))->>'timespan' timespan
), meta AS (SELECT ), meta AS (SELECT
id, id, geom_tid,
numer_aggregate, numer_colname, numer_geomref_colname, numer_aggregate, numer_colname, numer_geomref_colname,
numer_tablename, denom_aggregate, denom_colname, denom_geomref_colname, numer_tablename, denom_aggregate, denom_colname, denom_geomref_colname,
denom_tablename, geom_colname, geom_geomref_colname, denom_tablename, geom_colname, geom_geomref_colname,
@ -409,7 +409,8 @@ BEGIN
'geom_id', meta.geom_id 'geom_id', meta.geom_id
) metadata ) metadata
FROM meta, scores FROM meta, scores
WHERE meta.geom_id = scores.geom_id WHERE meta.geom_id = scores.column_id
AND meta.geom_tid = scores.table_id
) SELECT JSON_AGG(metadata ORDER BY id) ) SELECT JSON_AGG(metadata ORDER BY id)
FROM groups FROM groups
WHERE timespan_rank <= $4 WHERE timespan_rank <= $4
@ -432,146 +433,150 @@ END;
$$ LANGUAGE plpgsql IMMUTABLE; $$ LANGUAGE plpgsql IMMUTABLE;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureData( --CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureData(
geom geometry(Geometry, 4326), -- geom geometry(Geometry, 4326),
geom_type TEXT, -- geom_type TEXT,
normalize TEXT, -- normalize TEXT,
numer_aggregate TEXT, -- numer_aggregate TEXT,
numer_colname TEXT, -- numer_colname TEXT,
numer_geomref_colname TEXT, -- numer_geomref_colname TEXT,
numer_tablename TEXT, -- numer_tablename TEXT,
denom_colname TEXT, -- denom_colname TEXT,
denom_geomref_colname TEXT, -- denom_geomref_colname TEXT,
denom_tablename TEXT, -- denom_tablename TEXT,
geom_colname TEXT, -- geom_colname TEXT,
geom_geomref_colname TEXT, -- geom_geomref_colname TEXT,
geom_tablename TEXT -- geom_tablename TEXT
) --)
RETURNS NUMERIC --RETURNS NUMERIC
AS $$ --AS $$
DECLARE --DECLARE
sql TEXT; -- sql TEXT;
map_type TEXT; -- map_type TEXT;
result NUMERIC; -- result NUMERIC;
BEGIN --BEGIN
--
IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN -- IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN
map_type := 'areaNormalized'; -- map_type := 'areaNormalized';
ELSIF normalize ILIKE 'denominator' THEN -- ELSIF normalize ILIKE 'denominator' THEN
map_type := 'denominated'; -- map_type := 'denominated';
ELSE -- ELSE
-- defaults: area normalization for point if it's possible and none for -- -- defaults: area normalization for point if it's possible and none for
-- polygon or non-summable point -- -- polygon or non-summable point
IF geom_type = 'point' AND numer_aggregate ILIKE 'sum' THEN -- IF geom_type = 'point' AND numer_aggregate ILIKE 'sum' THEN
map_type := 'areaNormalized'; -- map_type := 'areaNormalized';
ELSE -- ELSE
map_type := 'predenominated'; -- map_type := 'predenominated';
END IF; -- END IF;
END IF; -- END IF;
--
IF geom_type = 'point' THEN -- IF geom_type = 'point' THEN
IF map_type = 'areaNormalized' THEN -- IF map_type = 'areaNormalized' THEN
sql = format('WITH _geom AS (SELECT ST_Area(geom.%I::Geography) / 1000000 area, geom.%I geom_ref -- sql = format('WITH _geom AS (SELECT ST_Area(geom.%I::Geography) / 1000000 area, geom.%I geom_ref
FROM observatory.%I geom -- FROM observatory.%I geom
WHERE ST_Within($1, geom.%I) -- WHERE ST_Within($1, geom.%I)
LIMIT 1) -- LIMIT 1)
SELECT numer.%I / (SELECT area FROM _geom) -- SELECT numer.%I / (SELECT area FROM _geom)
FROM observatory.%I numer -- FROM observatory.%I numer
WHERE numer.%I = (SELECT geom_ref FROM _geom)', -- WHERE numer.%I = (SELECT geom_ref FROM _geom)',
geom_colname, geom_geomref_colname, geom_tablename, -- geom_colname, geom_geomref_colname, geom_tablename,
geom_colname, numer_colname, numer_tablename, -- geom_colname, numer_colname, numer_tablename,
numer_geomref_colname); -- numer_geomref_colname);
ELSIF map_type = 'denominated' THEN -- ELSIF map_type = 'denominated' THEN
sql = format('SELECT numer.%I / NULLIF((SELECT denom.%I FROM observatory.%I denom WHERE denom.%I = numer.%I LIMIT 1), 0) -- sql = format('SELECT numer.%I / NULLIF((SELECT denom.%I FROM observatory.%I denom WHERE denom.%I = numer.%I LIMIT 1), 0)
FROM observatory.%I numer -- FROM observatory.%I numer
WHERE numer.%I = -- WHERE numer.%I =
(SELECT geom.%I -- (SELECT geom.%I
FROM observatory.%I geom -- FROM observatory.%I geom
WHERE ST_Within($1, geom.%I) LIMIT 1)', -- WHERE ST_Within($1, geom.%I) LIMIT 1)',
numer_colname, denom_colname, denom_tablename, -- numer_colname, denom_colname, denom_tablename,
denom_geomref_colname, numer_geomref_colname, -- denom_geomref_colname, numer_geomref_colname,
numer_tablename, numer_geomref_colname, -- numer_tablename, numer_geomref_colname,
geom_geomref_colname, geom_tablename, geom_colname); -- geom_geomref_colname, geom_tablename, geom_colname);
ELSIF map_type = 'predenominated' THEN -- ELSIF map_type = 'predenominated' THEN
sql = format('SELECT numer.%I -- sql = format('SELECT numer.%I
FROM observatory.%I numer -- FROM observatory.%I numer
WHERE numer.%I = -- WHERE numer.%I =
(SELECT geom.%I -- (SELECT geom.%I
FROM observatory.%I geom -- FROM observatory.%I geom
WHERE ST_Within($1, geom.%I) LIMIT 1)', -- WHERE ST_Within($1, geom.%I) LIMIT 1)',
numer_colname, numer_tablename, numer_geomref_colname, -- numer_colname, numer_tablename, numer_geomref_colname,
geom_geomref_colname, geom_tablename, geom_colname); -- geom_geomref_colname, geom_tablename, geom_colname);
END IF; -- END IF;
ELSIF geom_type = 'polygon' THEN -- ELSIF geom_type = 'polygon' THEN
IF map_type = 'areaNormalized' THEN -- IF map_type = 'areaNormalized' THEN
sql = format('WITH _subdivided AS ( -- sql = format('WITH _subdivided AS (
SELECT ST_Subdivide($1) AS geom -- SELECT ST_Subdivide($1) AS geom
), _geom AS (SELECT SUM(ST_Area(ST_Intersection(s.geom, geom.%I))) -- ), _geom AS (SELECT SUM(ST_Area(ST_Intersection(s.geom, geom.%I)))
/ ST_Area(cdb_observatory.FIRST(geom.%I)) overlap, geom.%I geom_ref -- / ST_Area(cdb_observatory.FIRST(geom.%I)) overlap, geom.%I geom_ref
FROM observatory.%I geom, _subdivided s -- FROM observatory.%I geom, _subdivided s
WHERE ST_Intersects(s.geom, geom.%I) -- WHERE ST_Intersects(s.geom, geom.%I)
GROUP BY geom.%I) -- GROUP BY geom.%I)
SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / -- SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) /
(ST_Area($1::Geography) / 1000000) -- (ST_Area($1::Geography) / 1000000)
FROM observatory.%I numer -- FROM observatory.%I numer
WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])', -- WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])',
geom_colname, geom_colname, geom_geomref_colname, geom_tablename, -- geom_colname, geom_colname, geom_geomref_colname, geom_tablename,
geom_colname, geom_geomref_colname, numer_colname, -- geom_colname, geom_geomref_colname, numer_colname,
numer_geomref_colname, numer_tablename, numer_geomref_colname); -- numer_geomref_colname, numer_tablename, numer_geomref_colname);
ELSIF map_type = 'denominated' THEN -- ELSIF map_type = 'denominated' THEN
sql = format('WITH _subdivided AS ( -- sql = format('WITH _subdivided AS (
SELECT ST_Subdivide($1) AS geom -- SELECT ST_Subdivide($1) AS geom
), _geom AS (SELECT SUM(ST_Area(ST_Intersection(s.geom, geom.%I))) -- ), _geom AS (SELECT SUM(ST_Area(ST_Intersection(s.geom, geom.%I)))
/ ST_Area(cdb_observatory.FIRST(geom.%I)) overlap, geom.%I geom_ref -- / ST_Area(cdb_observatory.FIRST(geom.%I)) overlap, geom.%I geom_ref
FROM observatory.%I geom, _subdivided s -- FROM observatory.%I geom, _subdivided s
WHERE ST_Intersects(s.geom, geom.%I) -- WHERE ST_Intersects(s.geom, geom.%I)
GROUP BY geom.%I), -- GROUP BY geom.%I),
_denom AS (SELECT denom.%I, denom.%I geom_ref -- _denom AS (SELECT denom.%I, denom.%I geom_ref
FROM observatory.%I denom -- FROM observatory.%I denom
WHERE denom.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])) -- WHERE denom.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[]))
SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / -- SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) /
NullIf(SUM((SELECT _denom.%I * (SELECT _geom.overlap -- NullIf(SUM((SELECT _denom.%I * (SELECT _geom.overlap
FROM _geom -- FROM _geom
WHERE _geom.geom_ref = _denom.geom_ref) -- WHERE _geom.geom_ref = _denom.geom_ref)
FROM _denom WHERE _denom.geom_ref = numer.%I)), 0) -- FROM _denom WHERE _denom.geom_ref = numer.%I)), 0)
FROM observatory.%I numer -- FROM observatory.%I numer
WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])', -- WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])',
geom_colname, geom_colname, geom_geomref_colname, -- geom_colname, geom_colname, geom_geomref_colname,
geom_tablename, geom_colname, geom_geomref_colname, -- geom_tablename, geom_colname, geom_geomref_colname,
denom_colname, denom_geomref_colname, denom_tablename, -- denom_colname, denom_geomref_colname, denom_tablename,
denom_geomref_colname, numer_colname, numer_geomref_colname, -- denom_geomref_colname, numer_colname, numer_geomref_colname,
denom_colname, numer_geomref_colname, -- denom_colname, numer_geomref_colname,
numer_tablename, numer_geomref_colname); -- numer_tablename, numer_geomref_colname);
ELSIF map_type = 'predenominated' THEN -- ELSIF map_type = 'predenominated' THEN
IF numer_aggregate NOT ILIKE 'sum' THEN -- IF numer_aggregate NOT ILIKE 'sum' THEN
RAISE EXCEPTION 'Cannot calculate "%" (%) for custom area as it cannot be summed, use ST_PointOnSurface instead', -- RAISE EXCEPTION 'Cannot calculate "%" (%) for custom area as it cannot be summed, use ST_PointOnSurface instead',
numer_name, measure_id; -- numer_name, measure_id;
ELSE -- ELSE
sql = format('WITH _subdivided AS ( -- sql = format('WITH _subdivided AS (
SELECT ST_Subdivide($1) AS geom -- SELECT ST_Subdivide($1) AS geom
), _geom AS (SELECT SUM(ST_Area(ST_Intersection(s.geom, geom.%I))) -- ), _geom AS (SELECT SUM(ST_Area(ST_Intersection(s.geom, geom.%I)))
/ ST_Area(cdb_observatory.FIRST(geom.%I)) overlap, -- / ST_Area(cdb_observatory.FIRST(geom.%I)) overlap,
geom.%I geom_ref -- geom.%I geom_ref
FROM observatory.%I geom, _subdivided s -- FROM observatory.%I geom, _subdivided s
WHERE ST_Intersects(s.geom, geom.%I) -- WHERE ST_Intersects(s.geom, geom.%I)
GROUP BY geom.%I -- GROUP BY geom.%I
) -- )
SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) -- SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I))
FROM observatory.%I numer -- FROM observatory.%I numer
WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])', -- WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])',
geom_colname, geom_colname, geom_geomref_colname, -- geom_colname, geom_colname, geom_geomref_colname,
geom_tablename, geom_colname, geom_geomref_colname, -- geom_tablename, geom_colname, geom_geomref_colname,
numer_colname, numer_geomref_colname, numer_tablename, -- numer_colname, numer_geomref_colname, numer_tablename,
numer_geomref_colname); -- numer_geomref_colname);
END IF; -- END IF;
END IF; -- END IF;
END IF; -- END IF;
--
EXECUTE sql INTO result USING geom; -- EXECUTE SELECT cdb_observatory.OBS_GetMeasureDataMulti(
RETURN result; -- array[($1, 1)::geomval, $2
-- )
END; --
$$ LANGUAGE plpgsql IMMUTABLE; -- INTO result USING geom, params;
-- RETURN result;
--
--END;
--$$ LANGUAGE plpgsql IMMUTABLE;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure( CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure(
geom geometry(Geometry, 4326), geom geometry(Geometry, 4326),
@ -591,6 +596,7 @@ DECLARE
denom_name TEXT; denom_name TEXT;
geom_name TEXT; geom_name TEXT;
denom_id TEXT; denom_id TEXT;
map_type TEXT;
BEGIN BEGIN
IF geom IS NULL THEN IF geom IS NULL THEN
RETURN NULL; RETURN NULL;
@ -611,32 +617,42 @@ BEGIN
ST_GeometryType(geom); ST_GeometryType(geom);
END IF; END IF;
IF normalize ILIKE 'area%' THEN --AND numer_aggregate ILIKE 'sum' THEN
map_type := 'areaNormalized';
ELSIF normalize ILIKE 'denom%' THEN
map_type := 'denominated';
ELSE
-- defaults: area normalization for point if it's possible and none for
-- polygon or non-summable point
IF geom_type = 'point' THEN --AND numer_aggregate ILIKE 'sum' THEN
map_type := 'areaNormalized';
ELSE
map_type := 'predenominated';
END IF;
END IF;
RAISE NOTICE 'map_type: %, geom_type: %', map_type, geom_type;
params := (SELECT cdb_observatory.OBS_GetMeasureMetaMulti( params := (SELECT cdb_observatory.OBS_GetMeasureMetaMulti(
geom, JSON_Build_Array(JSON_Build_Object('numer_id', measure_id, geom, JSON_Build_Array(JSON_Build_Object('numer_id', measure_id,
'geom_id', boundary_id, 'geom_id', boundary_id,
'timespan', time_span 'timespan', time_span
)), 500))->>0; )), 1, 1, 500));
IF params->>'geom_id' IS NULL THEN --IF normalize IS NOT NULL THEN
params := JSON_Build_Array(JSONB_Set((params::JSONB)->0, '{normalization}', to_jsonb(map_type))::JSON);
--END IF;
RAISE NOTICE '%', params;
IF params->0->>'geom_id' IS NULL THEN
RAISE NOTICE 'No boundary found for geom'; RAISE NOTICE 'No boundary found for geom';
RETURN NULL; RETURN NULL;
ELSE ELSE
RAISE NOTICE 'Using boundary %', params->>'geom_id'; RAISE NOTICE 'Using boundary %', params->0->>'geom_id';
END IF; END IF;
SELECT cdb_observatory.OBS_GetMeasureData(geom, SELECT measure FROM
geom_type, cdb_observatory.OBS_GetMeasureDataMulti(ARRAY[(geom, 1)::geomval], params)
normalize, AS (id INT, measure NUMERIC)
params->>'numer_aggregate',
params->>'numer_colname',
params->>'numer_geomref_colname',
params->>'numer_tablename',
params->>'denom_colname',
params->>'denom_geomref_colname',
params->>'denom_tablename',
params->>'geom_colname',
params->>'geom_geomref_colname',
params->>'geom_tablename')
INTO result; INTO result;
RETURN result; RETURN result;
@ -693,8 +709,9 @@ $$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureDataMulti( CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureDataMulti(
--geoms Geometry(Geometry, 4326)[], params json) geomvals geomval[],
geomvals geomval[], params JSON) params JSON
)
RETURNS SETOF RECORD RETURNS SETOF RECORD
AS $$ AS $$
DECLARE DECLARE
@ -730,11 +747,13 @@ BEGIN
(unnest($1))->>'geom_geomref_colname' geom_geomref_colname, (unnest($1))->>'geom_geomref_colname' geom_geomref_colname,
(unnest($1))->>'geom_tablename' geom_tablename, (unnest($1))->>'geom_tablename' geom_tablename,
(unnest($1))->>'geom_type' geom_type, (unnest($1))->>'geom_type' geom_type,
(unnest($1))->>'timespan' timespan (unnest($1))->>'timespan' timespan,
(unnest($1))->>'normalization' normalization
) )
SELECT String_Agg(CASE SELECT String_Agg(CASE
-- denominated -- denominated
WHEN denom_id IS NOT NULL THEN ' CASE ' || WHEN LOWER(normalization) LIKE 'denom%' OR (normalization IS NULL AND denom_id IS NOT NULL)
THEN ' CASE ' ||
-- denominated point-in-poly or user polygon is same as OBS polygon -- denominated point-in-poly or user polygon is same as OBS polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' || ' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' || ' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
@ -744,7 +763,8 @@ BEGIN
-- SUM ((numer / denom) * (% user geom in OBS geom)) -- SUM ((numer / denom) * (% user geom in OBS geom))
' ELSE ' || ' ELSE ' ||
--' NULL END ' --' NULL END '
' SUM((' || numer_tablename || '.' || numer_colname || '/NullIf(' || denom_tablename || '.' || denom_colname || ', 0)) ' || ' SUM((' || numer_tablename || '.' || numer_colname || '/' ||
' NullIf(' || denom_tablename || '.' || denom_colname || ', 0)) ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' || ' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) THEN ' || ' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) THEN ' ||
' ST_Area(' || geom_tablename || '.' || geom_colname || ') ' || ' ST_Area(' || geom_tablename || '.' || geom_colname || ') ' ||
@ -753,7 +773,8 @@ BEGIN
' / ST_Area(_geoms.geom))' || ' / ST_Area(_geoms.geom))' ||
' END) END ' ' END) END '
-- areaNormalized -- areaNormalized
WHEN numer_aggregate ILIKE 'sum' THEN ' CASE ' || WHEN LOWER(normalization) LIKE 'area%' OR (normalization IS NULL AND numer_aggregate ILIKE 'sum')
THEN ' CASE ' ||
-- areaNormalized point-in-poly or user polygon is the same as OBS polygon -- areaNormalized point-in-poly or user polygon is the same as OBS polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' || ' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' || ' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
@ -763,24 +784,6 @@ BEGIN
-- SUM (numer * (% OBS geom in user geom)) / area of big geom -- SUM (numer * (% OBS geom in user geom)) / area of big geom
' ELSE ' || ' ELSE ' ||
--' NULL END ' --' NULL END '
' SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') ' ||
' THEN ST_Area(_geoms.geom) / ST_Area(' || geom_tablename || '.' || geom_colname || ') ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) ' ||
' THEN 1 ' ||
' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / ST_Area(' || geom_tablename || '.' || geom_colname || '))' ||
' END) END '
-- prenormalized
ELSE ' CASE ' ||
-- predenominated point-in-poly or user polygon is the same as OBS- polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' ||
' ELSE ' ||
-- predenominated polygon interpolation
-- TODO should weight by universe instead of area
-- SUM (numer * (% user geom in OBS geom))
' SUM((' || numer_tablename || '.' || numer_colname || ') ' || ' SUM((' || numer_tablename || '.' || numer_colname || ') ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' || ' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) THEN ' || ' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) THEN ' ||
@ -789,6 +792,24 @@ BEGIN
' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' || ' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / ST_Area(_geoms.geom))' || ' / ST_Area(_geoms.geom))' ||
' END) END ' ' END) END '
-- prenormalized
ELSE ' CASE ' ||
-- predenominated point-in-poly or user polygon is the same as OBS- polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
--' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' ||
' ELSE ' ||
-- predenominated polygon interpolation
-- TODO should weight by universe instead of area
-- SUM (numer * (% user geom in OBS geom))
' SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') ' ||
' THEN ST_Area(_geoms.geom) / ST_Area(' || geom_tablename || '.' || geom_colname || ') ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) ' ||
' THEN 1 ' ||
' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / ST_Area(' || geom_tablename || '.' || geom_colname || '))' ||
' END) END '
END || ':: ' || numer_type || ' AS ' || numer_colname, ', ') AS colspecs, END || ':: ' || numer_type || ' AS ' || numer_colname, ', ') AS colspecs,
(SELECT String_Agg(tablename, ', ') FROM (SELECT JSONB_Object_Keys(JSONB_Object( (SELECT String_Agg(tablename, ', ') FROM (SELECT JSONB_Object_Keys(JSONB_Object(

12
src/pg/sql/42_observatory_exploration.sql
View File

@ -298,7 +298,7 @@ BEGIN
) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms, ) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms,
percentfill, estnumgeoms, meanmediansize percentfill, estnumgeoms, meanmediansize
FROM available_geoms, scores FROM available_geoms, scores
WHERE available_geoms.geom_id = scores.geom_id WHERE available_geoms.geom_id = scores.column_id
$string$, geom_clause) $string$, geom_clause)
USING numer_id, denom_id, timespan, filter_tags, bounds; USING numer_id, denom_id, timespan, filter_tags, bounds;
RETURN; RETURN;
@ -420,7 +420,8 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
) RETURNS TABLE ( ) RETURNS TABLE (
score NUMERIC, score NUMERIC,
numtiles BIGINT, numtiles BIGINT,
geom_id TEXT, table_id TEXT,
column_id TEXT,
notnull_percent NUMERIC, notnull_percent NUMERIC,
numgeoms NUMERIC, numgeoms NUMERIC,
percentfill NUMERIC, percentfill NUMERIC,
@ -454,20 +455,20 @@ BEGIN
FROM clipped_geom FROM clipped_geom
GROUP BY column_id, table_id GROUP BY column_id, table_id
), clipped_geom_reagg AS ( ), clipped_geom_reagg AS (
SELECT COUNT(*)::BIGINT cnt, a.column_id, SELECT COUNT(*)::BIGINT cnt, a.column_id, a.table_id,
cdb_observatory.FIRST(pixels) first_pixel, cdb_observatory.FIRST(pixels) first_pixel,
cdb_observatory.FIRST(notnull_pixels) first_notnull_pixel, cdb_observatory.FIRST(notnull_pixels) first_notnull_pixel,
cdb_observatory.FIRST(tile) first_tile, cdb_observatory.FIRST(tile) first_tile,
(ST_SummaryStatsAgg(clipped_tile, 1, True)).sum::Numeric sum_geoms, -- ND (ST_SummaryStatsAgg(clipped_tile, 1, True)).sum::Numeric sum_geoms, -- ND
(ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND (ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND
--(ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND
FROM clipped_geom_countagg a, clipped_geom b FROM clipped_geom_countagg a, clipped_geom b
WHERE a.table_id = b.table_id WHERE a.table_id = b.table_id
AND a.column_id = b.column_id AND a.column_id = b.column_id
GROUP BY a.column_id, a.table_id GROUP BY a.column_id, a.table_id
), final AS ( ), final AS (
SELECT SELECT
cnt, column_id cnt, table_id, column_id
, (CASE WHEN first_notnull_pixel > 0 , (CASE WHEN first_notnull_pixel > 0
THEN first_notnull_pixel / first_pixel THEN first_notnull_pixel / first_pixel
ELSE 1 ELSE 1
@ -480,7 +481,6 @@ BEGIN
* first_pixel) -- -20 * first_pixel) -- -20
END)::Numeric END)::Numeric
AS numgeoms AS numgeoms
, (CASE WHEN first_notnull_pixel > 0 , (CASE WHEN first_notnull_pixel > 0
THEN mean_fill THEN mean_fill
ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2 ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2

2
src/pg/sql/44_observatory_geometries.sql
View File

@ -566,6 +566,8 @@ BEGIN
geom_ct.column_id = geom_c.id AND geom_ct.column_id = geom_c.id AND
geom_c.type ILIKE 'geometry' AND geom_c.type ILIKE 'geometry' AND
geom_c.id = '%s' geom_c.id = '%s'
ORDER BY timespan DESC
LIMIT 1
$string$, boundary_id, boundary_id); $string$, boundary_id, boundary_id);
RETURN; RETURN;
-- AND geom_t.timespan = '%s' <-- put in requested year -- AND geom_t.timespan = '%s' <-- put in requested year

11
src/pg/test/fixtures/drop_fixtures.sql vendored
View File

@ -15,16 +15,17 @@ DROP TABLE IF EXISTS observatory.obs_meta_timespan;
DROP TABLE IF EXISTS observatory.obs_column_table_tile; DROP TABLE IF EXISTS observatory.obs_column_table_tile;
DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple; DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple;
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963; DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74; DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
DROP TABLE IF EXISTS observatory.obs_0310c639744a2014bb1af82709228f05b59e7d3d;
DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103; DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103;
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13; DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;
DROP TABLE IF EXISTS observatory.obs_1babf5a26a1ecda5fb74963e88408f71d0364b81; DROP TABLE IF EXISTS observatory.obs_b393b5b88c6adda634b2071a8005b03c551b609a;
DROP TABLE IF EXISTS observatory.obs_78fb6c1d6ff6505225175922c2c389ce48d7632c;
DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d; DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d;
DROP TABLE IF EXISTS observatory.obs_144e8b4f906885b2e057ac4842644a553ae49c6e; DROP TABLE IF EXISTS observatory.obs_a01cd5d8ccaa6531cef715071e9307e6b1987ec3;
DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7; DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7;
DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd; DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd;
DROP TABLE IF EXISTS observatory.obs_1ea93bbc109c87c676b3270789dacf7a1430db6c; DROP TABLE IF EXISTS observatory.obs_1ea93bbc109c87c676b3270789dacf7a1430db6c;
DROP TABLE IF EXISTS observatory.obs_b393b5b88c6adda634b2071a8005b03c551b609a; DROP TABLE IF EXISTS observatory.obs_87a814e485deabe3b12545a537f693d16ca702c2;
DROP TABLE IF EXISTS observatory.obs_c4411eba732408d47d73281772dbf03d60645dec;
DROP TABLE IF EXISTS observatory.obs_1746e37b7cd28cb131971ea4187d42d71f09c5f3; DROP TABLE IF EXISTS observatory.obs_1746e37b7cd28cb131971ea4187d42d71f09c5f3;
DROP TABLE IF EXISTS observatory.obs_a01cd5d8ccaa6531cef715071e9307e6b1987ec3;

10321
src/pg/test/fixtures/load_fixtures.sql vendored
View File

File diff suppressed because one or more lines are too long

63
src/pg/test/sql/41_observatory_augmentation_test.sql
View File

@ -33,67 +33,6 @@ WITH result as (
) select expected = 0 as OBS_Get_median_income_at_null_island ) select expected = 0 as OBS_Get_median_income_at_null_island
from result; from result;
-- OBS_GetPoints
-- obs_getpoints
-- --------------------
-- {4809.33511352425}
-- SELECT
-- (cdb_observatory._OBS_GetPoints(
-- cdb_observatory._TestPoint(),
-- 'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- block groups (see _obs_geomtable)
-- (Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
-- ))[1]::text = '{"value":10923.093200390833950,"name":"Total Population","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'
-- as OBS_GetPoints_for_test_point;
WITH cte As (
SELECT
(cdb_observatory._OBS_GetPoints(
cdb_observatory._TestPoint(),
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- block groups (see _obs_geomtable)
(Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
))[1]
as OBS_GetPoints_for_test_point)
SELECT
(abs((OBS_GetPoints_for_test_point ->> 'value')::numeric - 10923.093200390833950) / 10923.093200390833950) < 0.001 As OBS_GetPoints_for_test_point_value,
(OBS_GetPoints_for_test_point ->> 'name') = 'Total Population' As OBS_GetPoints_for_test_point_name,
(OBS_GetPoints_for_test_point ->> 'tablename') = 'obs_1a098da56badf5f32e336002b0a81708c40d29cd' As OBS_GetPoints_for_test_point_tablename,
(OBS_GetPoints_for_test_point ->> 'aggregate') = 'sum' As OBS_GetPoints_for_test_point_aggregate,
(OBS_GetPoints_for_test_point ->> 'type') = 'Numeric' As OBS_GetPoints_for_test_point_type,
(OBS_GetPoints_for_test_point ->> 'description') = 'The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.' As OBS_GetPoints_for_test_point_description
FROM cte;
-- what happens at null island
SELECT
(cdb_observatory._OBS_GetPoints(
ST_SetSRID(ST_Point(0, 0), 4326),
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- see example in obs_geomtable
(Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
))[1]::text is null
as OBS_GetPoints_for_null_island;
-- OBS_GetPolygons
-- obs_getpolygons
-- --------------------
-- {12996.8172420752}
SELECT
(cdb_observatory._OBS_GetPolygons(
cdb_observatory._TestArea(),
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- see example in obs_geomtable
Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json]
))[1]::text = '{"value":12327.3133495107,"name":"Total Population","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'
as OBS_GetPolygons_for_test_point;
-- see what happens around null island
SELECT
((cdb_observatory._OBS_GetPolygons(
ST_Buffer(ST_SetSRID(ST_Point(0, 0), 4326)::geography, 500)::geometry,
'obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308'::text, -- see example in obs_geomtable
Array['{"colname":"total_pop","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json])
)[1]->>'value') is null
as OBS_GetPolygons_for_null_island;
SELECT cdb_observatory.OBS_GetSegmentSnapshot( SELECT cdb_observatory.OBS_GetSegmentSnapshot(
cdb_observatory._TestPoint(), cdb_observatory._TestPoint(),
'us.census.tiger.census_tract' 'us.census.tiger.census_tract'
@ -196,7 +135,7 @@ SELECT (abs(cdb_observatory.OBS_GetMeasure(
-- Poly-based OBS_GetMeasure with denominator normalization -- Poly-based OBS_GetMeasure with denominator normalization
SELECT abs(cdb_observatory.OBS_GetMeasure( SELECT abs(cdb_observatory.OBS_GetMeasure(
cdb_observatory._TestArea(), cdb_observatory._TestArea(),
'us.census.acs.B01001002', 'denominator') - 0.49026340444793965457) / 0.49026340444793965457 < 0.001 As OBS_GetMeasure_total_male_poly_denominator; 'us.census.acs.B01001002', 'denominator', null, '2010 - 2014') - 0.49026340444793965457) / 0.49026340444793965457 < 0.001 As OBS_GetMeasure_total_male_poly_denominator;
-- Poly-based OBS_GetMeasure with one very bad geom -- Poly-based OBS_GetMeasure with one very bad geom
SELECT abs(cdb_observatory.OBS_GetMeasure( SELECT abs(cdb_observatory.OBS_GetMeasure(

28
src/pg/test/sql/42_observatory_exploration_test.sql
View File

@ -350,7 +350,7 @@ AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014;
-- _OBS_GetGeometryScores tests -- _OBS_GetGeometryScores tests
-- --
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5'] 'us.census.tiger.county', 'us.census.tiger.zcta5']
AS _obs_geometryscores_500m_buffer AS _obs_geometryscores_500m_buffer
@ -359,7 +359,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5']); 'us.census.tiger.county', 'us.census.tiger.zcta5']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5'] 'us.census.tiger.county', 'us.census.tiger.zcta5']
AS _obs_geometryscores_5km_buffer AS _obs_geometryscores_5km_buffer
@ -368,7 +368,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5']); 'us.census.tiger.county', 'us.census.tiger.zcta5']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.block_group', ARRAY['us.census.tiger.census_tract', 'us.census.tiger.block_group',
'us.census.tiger.zcta5', 'us.census.tiger.county'] 'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_50km_buffer AS _obs_geometryscores_50km_buffer
@ -377,7 +377,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY[ 'us.census.tiger.zcta5', 'us.census.tiger.census_tract', ARRAY[ 'us.census.tiger.zcta5', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.block_group'] 'us.census.tiger.county', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer AS _obs_geometryscores_500km_buffer
@ -386,7 +386,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5', ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group'] 'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_2500km_buffer AS _obs_geometryscores_2500km_buffer
@ -395,7 +395,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text
= '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }' = '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }'
AS _obs_geometryscores_numgeoms_500m_buffer AS _obs_geometryscores_numgeoms_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
@ -403,7 +403,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }' '{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
AS _obs_geometryscores_numgeoms_5km_buffer AS _obs_geometryscores_numgeoms_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
@ -411,7 +411,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }' '{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }'
AS _obs_geometryscores_numgeoms_50km_buffer AS _obs_geometryscores_numgeoms_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
@ -419,7 +419,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 48420, "us.census.tiger.census_tract" : 15774, "us.census.tiger.zcta5" : 6533, "us.census.tiger.county" : 304 }' '{ "us.census.tiger.block_group" : 48420, "us.census.tiger.census_tract" : 15774, "us.census.tiger.zcta5" : 6533, "us.census.tiger.county" : 304 }'
AS _obs_geometryscores_numgeoms_500km_buffer AS _obs_geometryscores_numgeoms_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
@ -427,7 +427,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 165475, "us.census.tiger.census_tract" : 55128, "us.census.tiger.zcta5" : 26499, "us.census.tiger.county" : 2622 }' '{ "us.census.tiger.block_group" : 165475, "us.census.tiger.census_tract" : 55128, "us.census.tiger.zcta5" : 26499, "us.census.tiger.county" : 2622 }'
AS _obs_geometryscores_numgeoms_2500km_buffer AS _obs_geometryscores_numgeoms_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
@ -435,7 +435,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']); 'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5', ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group'] 'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_50_geoms AS _obs_geometryscores_500km_buffer_50_geoms
@ -444,7 +444,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 50); 'us.census.tiger.zcta5', 'us.census.tiger.county'], 50);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) SELECT ARRAY_AGG(column_id ORDER BY score DESC)
= ARRAY['us.census.tiger.zcta5', 'us.census.tiger.census_tract', = ARRAY['us.census.tiger.zcta5', 'us.census.tiger.census_tract',
'us.census.tiger.block_group', 'us.census.tiger.county'] 'us.census.tiger.block_group', 'us.census.tiger.county']
AS _obs_geometryscores_500km_buffer_500_geoms AS _obs_geometryscores_500km_buffer_500_geoms
@ -453,7 +453,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC)
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 500); 'us.census.tiger.zcta5', 'us.census.tiger.county'], 500);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.block_group', ARRAY['us.census.tiger.census_tract', 'us.census.tiger.block_group',
'us.census.tiger.zcta5', 'us.census.tiger.county'] 'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_500km_buffer_2500_geoms AS _obs_geometryscores_500km_buffer_2500_geoms
@ -462,7 +462,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500); 'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'] 'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_500km_buffer_25000_geoms AS _obs_geometryscores_500km_buffer_25000_geoms

4
src/pg/test/sql/44_observatory_geometries_test.sql
View File

@ -73,7 +73,7 @@ SELECT cdb_observatory.OBS_GetBoundaryId(
SELECT cdb_observatory.OBS_GetBoundaryId( SELECT cdb_observatory.OBS_GetBoundaryId(
cdb_observatory._TestPoint(), cdb_observatory._TestPoint(),
'us.census.tiger.county', 'us.census.tiger.county',
'2014' '2015'
) = '36047'::text As OBS_GetBoundaryId_cartodb_county_with_year; ) = '36047'::text As OBS_GetBoundaryId_cartodb_county_with_year;
-- should give back null since there is not a census tract at null island -- should give back null since there is not a census tract at null island
@ -335,7 +335,7 @@ FROM (
SELECT SELECT
geoid_colname = 'geoid' As geoid_name_matches, geoid_colname = 'geoid' As geoid_name_matches,
target_table = 'obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7' As table_name_matches, target_table = 'obs_87a814e485deabe3b12545a537f693d16ca702c2' As table_name_matches,
geom_colname = 'the_geom' As geom_name_matches geom_colname = 'the_geom' As geom_name_matches
FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract') FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract')
As m(geoid_colname, target_table, geom_colname); As m(geoid_colname, target_table, geom_colname);

322
src/python/test/perftest.py
View File

@ -23,11 +23,11 @@ for q in (
ST_Translate(the_geom, -0.1, 0.1) offset_geom, ST_Translate(the_geom, -0.1, 0.1) offset_geom,
geom_refs AS name geom_refs AS name
FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry( FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
st_makeenvelope(-74.05437469482422,40.66319159533881, st_makeenvelope(-74.1, 40.5,
-73.81885528564453,40.745696344339564, 4326), -73.8, 40.9, 4326),
'us.census.tiger.census_tract_clipped')) foo 'us.census.tiger.census_tract_clipped')) foo
ORDER BY ST_NPoints(the_geom) ASC ORDER BY ST_NPoints(the_geom) ASC
LIMIT 500''', LIMIT 1000''',
'DROP TABLE IF EXISTS obs_perftest_complex', 'DROP TABLE IF EXISTS obs_perftest_complex',
'''CREATE TABLE obs_perftest_complex (cartodb_id SERIAL PRIMARY KEY, '''CREATE TABLE obs_perftest_complex (cartodb_id SERIAL PRIMARY KEY,
point GEOMETRY, point GEOMETRY,
@ -101,159 +101,159 @@ def record(params, results):
} }
json.dump(tests, fhandle) json.dump(tests, fhandle)
@parameterized([ #@parameterized([
('simple', '_OBS_GetGeometryScores', 'NULL', 1), # ('simple', '_OBS_GetGeometryScores', 'NULL', 1),
('simple', '_OBS_GetGeometryScores', 'NULL', 500), # ('simple', '_OBS_GetGeometryScores', 'NULL', 500),
('simple', '_OBS_GetGeometryScores', 'NULL', 3000), # ('simple', '_OBS_GetGeometryScores', 'NULL', 3000),
#
('complex', '_OBS_GetGeometryScores', 'NULL', 1), # ('complex', '_OBS_GetGeometryScores', 'NULL', 1),
('complex', '_OBS_GetGeometryScores', 'NULL', 500), # ('complex', '_OBS_GetGeometryScores', 'NULL', 500),
('complex', '_OBS_GetGeometryScores', 'NULL', 3000), # ('complex', '_OBS_GetGeometryScores', 'NULL', 3000),
#
('country_simple', '_OBS_GetGeometryScores', 'NULL', 1), # ('country_simple', '_OBS_GetGeometryScores', 'NULL', 1),
('country_simple', '_OBS_GetGeometryScores', 'NULL', 500), # ('country_simple', '_OBS_GetGeometryScores', 'NULL', 500),
('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000), # ('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000),
#
('country_complex', '_OBS_GetGeometryScores', 'NULL', 1), # ('country_complex', '_OBS_GetGeometryScores', 'NULL', 1),
('country_complex', '_OBS_GetGeometryScores', 'NULL', 500), # ('country_complex', '_OBS_GetGeometryScores', 'NULL', 500),
('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000), # ('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000),
]) #])
def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms): #def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms):
print api_method, geom_complexity, filters, target_geoms # print api_method, geom_complexity, filters, target_geoms
#
rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,) # rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,)
results = [] # results = []
for rows in rownums: # for rows in rownums:
stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms}) # stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms})
FROM obs_perftest_{complexity} # FROM obs_perftest_{complexity}
WHERE cartodb_id <= {n}'''.format( # WHERE cartodb_id <= {n}'''.format(
complexity=geom_complexity, # complexity=geom_complexity,
schema='cdb_observatory.' if USE_SCHEMA else '', # schema='cdb_observatory.' if USE_SCHEMA else '',
api_method=api_method, # api_method=api_method,
filters=filters, # filters=filters,
target_geoms=target_geoms, # target_geoms=target_geoms,
n=rows) # n=rows)
start = time() # start = time()
query(stmt) # query(stmt)
end = time() # end = time()
qps = (rows / (end - start)) # qps = (rows / (end - start))
results.append({ # results.append({
'rows': rows, # 'rows': rows,
'qps': qps, # 'qps': qps,
'stmt': stmt # 'stmt': stmt
}) # })
print rows, ': ', qps, ' QPS' # print rows, ': ', qps, ' QPS'
#
if 'OBS_RECORD_TEST' in os.environ: # if 'OBS_RECORD_TEST' in os.environ:
record({ # record({
'geom_complexity': geom_complexity, # 'geom_complexity': geom_complexity,
'api_method': api_method, # 'api_method': api_method,
'filters': filters, # 'filters': filters,
'target_geoms': target_geoms # 'target_geoms': target_geoms
}, results) # }, results)
#
@parameterized([ #@parameterized([
('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None), # ('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None),
('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None), # ('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None),
#
('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'), # ('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'), # ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'), # ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'), # ('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'), # ('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'), # ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'), # ('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'), # ('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'), # ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
('simple', 'OBS_GetCategory', None, 'point', 'NULL'), # ('simple', 'OBS_GetCategory', None, 'point', 'NULL'),
('simple', 'OBS_GetCategory', None, 'geom', 'NULL'), # ('simple', 'OBS_GetCategory', None, 'geom', 'NULL'),
('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'), # ('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
#
('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"), # ('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
#
('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'), # ('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'), # ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'), # ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'), # ('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'), # ('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'), # ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'), # ('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'), # ('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'), # ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
('complex', 'OBS_GetCategory', None, 'point', 'NULL'), # ('complex', 'OBS_GetCategory', None, 'point', 'NULL'),
('complex', 'OBS_GetCategory', None, 'geom', 'NULL'), # ('complex', 'OBS_GetCategory', None, 'geom', 'NULL'),
('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'), # ('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
#
('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"),
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"), # ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"),
('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"), # ('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"), # ('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"), # ('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
]) #])
def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary): #def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary):
print api_method, geom_complexity, normalization, geom, boundary # print api_method, geom_complexity, normalization, geom, boundary
col = 'measure' if 'measure' in api_method.lower() else 'category' # col = 'measure' if 'measure' in api_method.lower() else 'category'
results = [] # results = []
#
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, ) # rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, )
for rows in rownums: # for rows in rownums:
stmt = '''UPDATE obs_perftest_{complexity} # stmt = '''UPDATE obs_perftest_{complexity}
SET {col} = {schema}{api_method}({args}) # SET {col} = {schema}{api_method}({args})
WHERE cartodb_id <= {n}'''.format( # WHERE cartodb_id <= {n}'''.format(
col=col, # col=col,
complexity=geom_complexity, # complexity=geom_complexity,
schema='cdb_observatory.' if USE_SCHEMA else '', # schema='cdb_observatory.' if USE_SCHEMA else '',
api_method=api_method, # api_method=api_method,
args=ARGS[api_method, normalization].format(geom, boundary), # args=ARGS[api_method, normalization].format(geom, boundary),
n=rows) # n=rows)
start = time() # start = time()
query(stmt) # query(stmt)
end = time() # end = time()
qps = (rows / (end - start)) # qps = (rows / (end - start))
results.append({ # results.append({
'rows': rows, # 'rows': rows,
'qps': qps, # 'qps': qps,
'stmt': stmt # 'stmt': stmt
}) # })
print rows, ': ', qps, ' QPS' # print rows, ': ', qps, ' QPS'
#
if 'OBS_RECORD_TEST' in os.environ: # if 'OBS_RECORD_TEST' in os.environ:
record({ # record({
'geom_complexity': geom_complexity, # 'geom_complexity': geom_complexity,
'api_method': api_method, # 'api_method': api_method,
'normalization': normalization, # 'normalization': normalization,
'geom': geom # 'geom': geom
}, results) # }, results)
@parameterized([ @parameterized([
('simple', 'predenominated', 'point', 'NULL'), ('simple', 'predenominated', 'point', 'null'),
('simple', 'predenominated', 'geom', 'NULL'), ('simple', 'predenominated', 'geom', 'null'),
('simple', 'predenominated', 'offset_geom', 'NULL'), ('simple', 'predenominated', 'offset_geom', 'null'),
('simple', 'area', 'point', 'NULL'), ('simple', 'area', 'point', 'null'),
('simple', 'area', 'geom', 'NULL'), ('simple', 'area', 'geom', 'null'),
('simple', 'area', 'offset_geom', 'NULL'), ('simple', 'area', 'offset_geom', 'null'),
('simple', 'denominator', 'point', 'NULL'), ('simple', 'denominator', 'point', 'null'),
('simple', 'denominator', 'geom', 'NULL'), ('simple', 'denominator', 'geom', 'null'),
('simple', 'denominator', 'offset_geom', 'NULL'), ('simple', 'denominator', 'offset_geom', 'null'),
('simple', 'predenominated', 'point', "'us.census.tiger.census_tract'"), ('simple', 'predenominated', 'point', "'us.census.tiger.census_tract'"),
('simple', 'predenominated', 'geom', "'us.census.tiger.census_tract'"), ('simple', 'predenominated', 'geom', "'us.census.tiger.census_tract'"),
@ -289,17 +289,23 @@ def test_getmeasure_split_performance(geom_complexity, normalization, geom, boun
print geom_complexity, normalization, geom, boundary print geom_complexity, normalization, geom, boundary
results = [] results = []
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, ) rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (100, 500, 1000)
for rows in rownums: for rows in rownums:
stmt = ''' stmt = '''
with data as ( with data as (
SELECT * FROM {schema}{api_method}datamulti( SELECT * FROM {schema}{api_method}datamulti(
(SELECT array_agg((geom, cartodb_id)::geomval) (SELECT array_agg(({geom}, cartodb_id)::geomval)
FROM obs_perftest_{complexity} FROM obs_perftest_{complexity}
WHERE cartodb_id <= {n}), WHERE cartodb_id <= {n}),
(SELECT {schema}{api_method}metamulti( (SELECT {schema}{api_method}metamulti(
(SELECT st_setsrid(st_extent(geom),4326) from obs_perftest_{complexity}), (SELECT st_setsrid(st_extent({geom}), 4326)
json_build_array(json_build_object('numer_id', 'us.census.acs.B01001002')) FROM obs_perftest_{complexity}
WHERE cartodb_id <= {n}),
'[{{
"numer_id": "us.census.acs.B01001002",
"normalization": "{normalization}",
"geom_id": {boundary}
}}]'::JSON
)) ))
) )
AS x(cartodb_id INTEGER, measure Numeric)) AS x(cartodb_id INTEGER, measure Numeric))
@ -315,7 +321,7 @@ WHERE obs_perftest_{complexity}.cartodb_id = data.cartodb_id
api_method='obs_getmeasure', api_method='obs_getmeasure',
normalization=normalization, normalization=normalization,
geom=geom, geom=geom,
boundary=boundary, boundary=boundary.replace("'", '"'),
n=rows) n=rows)
start = time() start = time()
query(stmt) query(stmt)