cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add perf tests for different geometry complexities as well as all code branches for getmeasure

Browse Source
This commit is contained in:
John Krauss 2016-12-05 18:51:58 +00:00
parent cbe7b6dd15
commit 463db99222
1 changed files with 74 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
src/python/test/perftest.py
View File

@ -8,14 +8,41 @@ from time import time
USE_SCHEMA = True USE_SCHEMA = True
for q in ( for q in (
'DROP TABLE IF EXISTS obs_censustest', 'DROP TABLE IF EXISTS obs_perftest_simple',
'''CREATE TABLE obs_censustest (cartodb_id SERIAL PRIMARY KEY, '''CREATE TABLE obs_perftest_simple (cartodb_id SERIAL PRIMARY KEY,
the_geom GEOMETRY, name TEXT, measure NUMERIC, category TEXT)''', point GEOMETRY,
'''INSERT INTO obs_censustest (the_geom, name) geom GEOMETRY,
SELECT * FROM {schema}OBS_GetBoundariesByGeometry( offset_geom GEOMETRY,
st_makeenvelope(-74.05437469482422,40.66319159533881, name TEXT, measure NUMERIC, category TEXT)''',
-73.81885528564453,40.745696344339564, 4326), '''INSERT INTO obs_perftest_simple (point, geom, offset_geom, name)
'us.census.tiger.block_group_clipped') As m(the_geom, geoid)''' SELECT ST_PointOnSurface(the_geom) point,
the_geom geom,
ST_Translate(the_geom, -0.1, 0.1) offset_geom,
geom_refs AS name
FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
st_makeenvelope(-74.05437469482422,40.66319159533881,
-73.81885528564453,40.745696344339564, 4326),
'us.census.tiger.census_tract_clipped')) foo
ORDER BY ST_NPoints(the_geom) ASC
LIMIT 50''',
'DROP TABLE IF EXISTS obs_perftest_complex',
'''CREATE TABLE obs_perftest_complex (cartodb_id SERIAL PRIMARY KEY,
point GEOMETRY,
geom GEOMETRY,
offset_geom GEOMETRY,
name TEXT, measure NUMERIC, category TEXT)''',
'''INSERT INTO obs_perftest_complex (point, geom, offset_geom, name)
SELECT ST_PointOnSurface(the_geom) point,
the_geom geom,
ST_Translate(the_geom, -0.1, 0.1) offset_geom,
geom_refs AS name
FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
st_makeenvelope(-75.05437469482422,40.66319159533881,
-73.81885528564453,41.745696344339564, 4326),
'us.census.tiger.county_clipped')) foo
ORDER BY ST_NPoints(the_geom) DESC
LIMIT 50;''',
'''SET statement_timeout = 5000;'''
): ):
query(q.format( query(q.format(
schema='cdb_observatory.' if USE_SCHEMA else '', schema='cdb_observatory.' if USE_SCHEMA else '',
@ -24,37 +51,53 @@ for q in (
ARGS = { ARGS = {
'OBS_GetMeasureByID': "name, 'us.census.acs.B01001002', '{}'", ('OBS_GetMeasureByID', None): "name, 'us.census.acs.B01001002', '{}'",
'OBS_GetMeasure': "{}, 'us.census.acs.B01001002'", ('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001'",
'OBS_GetCategory': "{}, 'us.census.spielman_singleton_segments.X10'", ('OBS_GetMeasure', 'area'): "{}, 'us.census.acs.B01001002', 'area'",
('OBS_GetMeasure', 'denominator'): "{}, 'us.census.acs.B01001002', 'denominator'",
('OBS_GetCategory', None): "{}, 'us.census.spielman_singleton_segments.X10'",
} }
GEOMS = {
'point': 'ST_PointOnSurface(the_geom)',
'polygon_match': 'the_geom',
'polygon_buffered': 'ST_Buffer(the_geom::GEOGRAPHY, 1000)::GEOMETRY(GEOMETRY, 4326)',
}
@parameterized([ @parameterized([
('OBS_GetMeasureByID', 'us.census.tiger.block_group_clipped'), ('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract'),
('OBS_GetMeasureByID', 'us.census.tiger.county'), ('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county'),
('OBS_GetMeasure', GEOMS['point']),
('OBS_GetMeasure', GEOMS['polygon_match']), ('simple', 'OBS_GetMeasure', 'predenominated', 'point'),
('OBS_GetMeasure', GEOMS['polygon_buffered']), ('simple', 'OBS_GetMeasure', 'predenominated', 'geom'),
('OBS_GetCategory', GEOMS['point']), ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom'),
('OBS_GetCategory', GEOMS['polygon_match']), ('simple', 'OBS_GetMeasure', 'area', 'point'),
('OBS_GetCategory', GEOMS['polygon_buffered']), ('simple', 'OBS_GetMeasure', 'area', 'geom'),
('simple', 'OBS_GetMeasure', 'area', 'offset_geom'),
('simple', 'OBS_GetMeasure', 'denominator', 'point'),
('simple', 'OBS_GetMeasure', 'denominator', 'geom'),
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom'),
('simple', 'OBS_GetCategory', None, 'point'),
('simple', 'OBS_GetCategory', None, 'geom'),
('simple', 'OBS_GetCategory', None, 'offset_geom'),
('complex', 'OBS_GetMeasure', 'predenominated', 'point'),
('complex', 'OBS_GetMeasure', 'predenominated', 'geom'),
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom'),
('complex', 'OBS_GetMeasure', 'area', 'point'),
('complex', 'OBS_GetMeasure', 'area', 'geom'),
('complex', 'OBS_GetMeasure', 'area', 'offset_geom'),
('complex', 'OBS_GetMeasure', 'denominator', 'point'),
('complex', 'OBS_GetMeasure', 'denominator', 'geom'),
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom'),
('complex', 'OBS_GetCategory', None, 'point'),
('complex', 'OBS_GetCategory', None, 'geom'),
('complex', 'OBS_GetCategory', None, 'offset_geom'),
]) ])
def test_performance(api_method, arg): def test_performance(geom_complexity, api_method, normalization, geom):
print api_method, arg print api_method, geom_complexity, normalization, geom
col = 'measure' if 'measure' in api_method.lower() else 'category' col = 'measure' if 'measure' in api_method.lower() else 'category'
for rows in (1, 10, 50, 100): for rows in (1, 5, 10, ):
q = 'UPDATE obs_censustest SET {col} = {schema}{api_method}({args}) WHERE cartodb_id < {n}'.format( q = 'UPDATE obs_perftest_{complexity} SET {col} = {schema}{api_method}({args}) WHERE cartodb_id < {n}'.format(
col=col, col=col,
complexity=geom_complexity,
schema='cdb_observatory.' if USE_SCHEMA else '', schema='cdb_observatory.' if USE_SCHEMA else '',
api_method=api_method, api_method=api_method,
args=ARGS[api_method].format(arg), args=ARGS[api_method, normalization].format(geom),
n=rows+1) n=rows+1)
start = time() start = time()
query(q) query(q)