Merge pull request #303 from CartoDB/precheck_metadata
OBS_MetadataValidation
This commit is contained in:
commit
6066ef028d
42
.travis.yml
Normal file
42
.travis.yml
Normal file
@ -0,0 +1,42 @@
|
||||
language: c
|
||||
|
||||
env:
|
||||
global:
|
||||
- PAGER=cat
|
||||
|
||||
before_install:
|
||||
- sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
|
||||
- sudo add-apt-repository -y ppa:cartodb/gis
|
||||
- sudo add-apt-repository -y ppa:cartodb/gis-testing
|
||||
- sudo apt-get update
|
||||
|
||||
# Install postgres db and build deps
|
||||
- sudo /etc/init.d/postgresql stop # stop travis default instance
|
||||
- sudo apt-get -y remove --purge postgresql-9.1
|
||||
- sudo apt-get -y remove --purge postgresql-9.2
|
||||
- sudo apt-get -y remove --purge postgresql-9.3
|
||||
- sudo apt-get -y remove --purge postgresql-9.4
|
||||
- sudo apt-get -y remove --purge postgresql-9.5
|
||||
- sudo rm -rf /var/lib/postgresql/
|
||||
- sudo rm -rf /var/log/postgresql/
|
||||
- sudo rm -rf /etc/postgresql/
|
||||
- sudo apt-get -y remove --purge postgis-2.2
|
||||
- sudo apt-get -y autoremove
|
||||
|
||||
- sudo apt-get -y install postgresql-9.5=9.5.2-3cdb3
|
||||
- sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb3
|
||||
- sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb3
|
||||
- sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
|
||||
- sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
|
||||
|
||||
# configure it to accept local connections from postgres
|
||||
- echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \
|
||||
| sudo tee /etc/postgresql/9.5/main/pg_hba.conf
|
||||
- sudo /etc/init.d/postgresql restart 9.5
|
||||
|
||||
install:
|
||||
- sudo make install
|
||||
|
||||
script:
|
||||
- cd src/pg
|
||||
- make test || { cat src/pg/test/regression.diffs; false; }
|
@ -52,8 +52,8 @@ def get_tablename_query(column_id, boundary_id, timespan):
|
||||
METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
|
||||
'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta',
|
||||
'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom',
|
||||
'obs_meta_timespan', 'obs_column_table_tile',
|
||||
'obs_column_table_tile_simple']
|
||||
'obs_meta_timespan', 'obs_meta_geom_numer_timespan',
|
||||
'obs_column_table_tile', 'obs_column_table_tile_simple']
|
||||
|
||||
FIXTURES = [
|
||||
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
|
||||
|
@ -1,3 +1,4 @@
|
||||
requests
|
||||
nose
|
||||
nose_parameterized
|
||||
psycopg2
|
||||
|
@ -1076,3 +1076,46 @@ BEGIN
|
||||
RETURN result;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql STABLE;
|
||||
|
||||
-- MetadataValidation checks the metadata parameters and the geometry type
|
||||
-- of the data in order to find possible wrong cases
|
||||
CREATE OR REPLACE FUNCTION cdb_observatory.obs_metadatavalidation(
|
||||
geometry_extent geometry(Geometry, 4326),
|
||||
geometry_type text,
|
||||
params JSON,
|
||||
target_geoms INTEGER DEFAULT NULL
|
||||
)
|
||||
RETURNS TABLE(valid boolean, errors text[]) AS $$
|
||||
DECLARE
|
||||
meta json;
|
||||
errors text[];
|
||||
BEGIN
|
||||
errors := (ARRAY[])::TEXT[];
|
||||
IF geometry_type IN ('ST_Polygon', 'ST_MultiPolygon') THEN
|
||||
FOR meta IN EXECUTE 'SELECT json_array_elements(cdb_observatory.OBS_GetMeta($1, $2, 1, 1, $3))' USING geometry_extent, params, target_geoms
|
||||
LOOP
|
||||
IF (meta->>'normalization' = 'denominated' AND meta->>'denom_id' is NULL) THEN
|
||||
errors := array_append(errors, 'Normalizated measure should have a numerator and a denominator. Please review the provided options.');
|
||||
END IF;
|
||||
IF (meta->>'numer_aggregate' IS NULL) THEN
|
||||
errors := array_append(errors, 'For polygon geometries, aggregation is mandatory. Please review the provided options');
|
||||
END IF;
|
||||
IF (meta->>'numer_aggregate' IN ('median', 'average') AND meta->>'denom_id' IS NULL) THEN
|
||||
errors := array_append(errors, 'Median or average aggregation for polygons requires a denominator to provide weights. Please review the provided options');
|
||||
END IF;
|
||||
IF (meta->>'numer_aggregate' IN ('median', 'average') AND meta->>'normalization' NOT LIKE 'pre%') THEN
|
||||
errors := array_append(errors, format('Median or average aggregation only supports prenormalized normalization, %s passed. Please review the provided options', meta->>'normalization'));
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
IF CARDINALITY(errors) > 0 THEN
|
||||
RETURN QUERY EXECUTE 'SELECT FALSE, $1' USING errors;
|
||||
ELSE
|
||||
RETURN QUERY SELECT TRUE, ARRAY[]::TEXT[];
|
||||
END IF;
|
||||
ELSE
|
||||
RETURN QUERY SELECT TRUE, ARRAY[]::TEXT[];
|
||||
END IF;
|
||||
RETURN;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql STABLE;
|
||||
|
@ -304,3 +304,12 @@ tract_sample|tract_max_error|tract_avg_error|tract_min_error
|
||||
no_bg_point_error
|
||||
t
|
||||
(1 row)
|
||||
valid|errors
|
||||
t|{}
|
||||
(1 row)
|
||||
valid|errors
|
||||
f|{"Median or average aggregation only supports prenormalized normalization, denominated passed. Please review the provided options"}
|
||||
(1 row)
|
||||
valid|errors
|
||||
f|{"Normalizated measure should have a numerator and a denominator. Please review the provided options."}
|
||||
(1 row)
|
||||
|
1
src/pg/test/fixtures/drop_fixtures.sql
vendored
1
src/pg/test/fixtures/drop_fixtures.sql
vendored
@ -12,6 +12,7 @@ DROP TABLE IF EXISTS observatory.obs_meta_numer;
|
||||
DROP TABLE IF EXISTS observatory.obs_meta_denom;
|
||||
DROP TABLE IF EXISTS observatory.obs_meta_geom;
|
||||
DROP TABLE IF EXISTS observatory.obs_meta_timespan;
|
||||
DROP TABLE IF EXISTS observatory.obs_meta_geom_numer_timespan;
|
||||
DROP TABLE IF EXISTS observatory.obs_column_table_tile;
|
||||
DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple;
|
||||
DROP TABLE IF EXISTS observatory.obs_78fb6c1d6ff6505225175922c2c389ce48d7632c;
|
||||
|
175993
src/pg/test/fixtures/load_fixtures.sql
vendored
175993
src/pg/test/fixtures/load_fixtures.sql
vendored
File diff suppressed because one or more lines are too long
@ -973,3 +973,9 @@ WITH _geoms AS (
|
||||
FROM geoms, results
|
||||
WHERE cartodb_id = id
|
||||
;
|
||||
|
||||
-- OBS_MetadataValidation
|
||||
|
||||
SELECT * FROM cdb_observatory.OBS_MetadataValidation(NULL, 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001","denom_id": null,"normalization": "prenormalized","geom_id": null,"numer_timespan": "2010 - 2014"}]'::json, 500);
|
||||
SELECT * FROM cdb_observatory.OBS_MetadataValidation(NULL, 'ST_Polygon', '[{"numer_id": "us.census.acs.B25058001","denom_id": null,"normalization": "denominated","geom_id": null,"numer_timespan": "2010 - 2014"}]'::json, 500);
|
||||
SELECT * FROM cdb_observatory.OBS_MetadataValidation(NULL, 'ST_Polygon', '[{"numer_id": "us.census.acs.B15003001","denom_id": null,"normalization": "denominated","geom_id": null,"numer_timespan": "2010 - 2014"}]'::json, 500);
|
||||
|
Loading…
Reference in New Issue
Block a user