Merge pull request #191 from CartoDB/develop

Release 1.0.6 with table level framework improvements

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Request for a new Data observatory extension deploy
+
+I'd like to request a new data observatory extension deploy: dump + extension
+
+## Dump database id to be deployed
+
+Please put here the dump id to be deployed: <dump_id>
+
+## Data Observatory extension PRs included.
+
+*Please update the NEWS.md*
+
+Add down here the PR links to be added and deployed:
+
+  - 
+
+// @CartoDB/dataservices

Makefile

@@ -18,7 +18,7 @@ test:   ## Run the tests for the development version of the extension
 	$(MAKE) -C $(EXT_DIR) test
 
 # Generate a new release into release
-release: ## Generate a new release of the extension. Only for telease manager
+release: ## Generate a new release of the extension. Only for release manager
 	$(MAKE) -C $(EXT_DIR) release
 
 # Install the current release.

NEWS.md

@@ -1,3 +1,10 @@
+1.0.6 (2016-09-08)
+
+__Improvements__
+
+* New function structure for Table-level functions which allows to separate the
+  framework logic from the observatory measure functions.
+
 1.0.5 (2016-08-12)
 
 __Improvements__

release/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.0.5'
+default_version = '1.0.6'
 requires = 'postgis, postgres_fdw'
 superuser = true
 schema = cdb_observatory

src/pg/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.0.5'
+default_version = '1.0.6'
 requires = 'postgis, postgres_fdw'
 superuser = true
 schema = cdb_observatory

src/pg/sql/50_table_level_framework.sql

@@ -24,9 +24,12 @@ BEGIN
 EXCEPTION
   WHEN others THEN
     -- Disconnect user imported table. Delete schema and FDW server.
-    EXECUTE 'DROP FOREIGN TABLE IF EXISTS ' || fdw_import_schema || '.' || table_name;
+    EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".' || table_name;
-    EXECUTE 'DROP SCHEMA IF EXISTS ' || fdw_import_schema || ' CASCADE';
+    EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".cdb_tablemetadata';
-    EXECUTE 'DROP SERVER IF EXISTS ' || fdw_server || ' CASCADE;';
+    EXECUTE 'DROP SCHEMA IF EXISTS "' || fdw_import_schema || '"';
+    EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || fdw_server || '"';
+    EXECUTE 'DROP SERVER IF EXISTS "' || fdw_server || '"';
+
     RETURN (null, null, null);
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
@@ -37,27 +40,9 @@ AS $$
 DECLARE
   colnames text[];
   coltypes text[];
-  requested_measures text[];
-  measure text;
 BEGIN
-
+  EXECUTE FORMAT('SELECT r.colnames::text[], r.coltypes::text[] FROM cdb_observatory._%sResultMetadata(%L::json) r', function_name, params::text)
-  -- Simple mock, there should be real logic in here.
+  INTO colnames, coltypes;
-
-  IF $3 NOT ILIKE 'GetMeasure' OR $3 IS NULL THEN
-    RAISE 'This function is not supported yet: %', $3;
-  END IF;
-
-  SELECT translate($4::json->>'tag_name','[]', '{}')::text[] INTO requested_measures;
-
-  FOREACH measure IN ARRAY requested_measures
-  LOOP
-    IF NOT measure ILIKE ANY (Array['total_pop', 'pop_16_over']::text[]) THEN
-      RAISE 'This measure is not supported yet: %', measure;
-    END IF;
-  SELECT array_append(colnames, measure) INTO colnames;
-  SELECT array_append(coltypes, 'double precision'::text) INTO coltypes;
-
-  END LOOP;
 
   RETURN (colnames::text[], coltypes::text[]);
 END;
@@ -68,41 +53,17 @@ RETURNS SETOF record
 AS $$
 DECLARE
   data_query text;
-  tag_name text[];
-  tag text;
-  tags_list text;
-  tags_query text;
   rec RECORD;
 BEGIN
-    SELECT translate($6::json->>'tag_name','[]', '{}')::text[] INTO tag_name;
-    SELECT array_to_string(tag_name, ',') INTO tags_list;
-    tags_query := '';
-
-    FOREACH tag IN ARRAY tag_name
-    LOOP
-      SELECT tags_query || ' sum(' || tag || '/fraction)::double precision as ' || tag || ', ' INTO tags_query;
-
-    END LOOP;
-
-    -- Simple mock, there should be real logic in here.
-    data_query := '(WITH _areas AS(SELECT ST_Area(a.the_geom::geography)'
-        || '/ (1000 * 1000) as fraction, a.geoid, b.cartodb_id FROM '
-        || 'observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308 as a, '
-        || table_schema || '.' || table_name || ' AS b '
-        || 'WHERE b.the_geom && a.the_geom ), values AS (SELECT geoid, '
-        || tags_list
-        || ' FROM observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd ) '
-        || 'SELECT '
-        || tags_query
-        || ' cartodb_id::int FROM _areas, values '
-        || 'WHERE values.geoid = _areas.geoid GROUP BY cartodb_id);';
 
+    EXECUTE FORMAT('SELECT cdb_observatory._%sQuery(%L, %L, %L::json)', function_name, table_schema, table_name, params::text)
+    INTO data_query;
 
     FOR rec IN EXECUTE data_query
     LOOP
         RETURN NEXT rec;
     END LOOP;
-    RETURN;
+  RETURN;
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
 
@@ -112,8 +73,10 @@ RETURNS boolean
 AS $$
 BEGIN
     EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".' || table_name;
-    EXECUTE 'DROP SCHEMA IF EXISTS ' || table_schema || ' CASCADE';
+    EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".cdb_tablemetadata';
-    EXECUTE 'DROP SERVER IF EXISTS ' || servername || ' CASCADE;';
+    EXECUTE 'DROP SCHEMA IF EXISTS "' || table_schema || '"';
+    EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || servername || '"';
+    EXECUTE 'DROP SERVER IF EXISTS "' || servername || '"';
     RETURN true;
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;

src/pg/sql/51_table_level_functions.sql

@@ -0,0 +1,79 @@
+--
+--
+-- OBS_GetMeasure
+--
+--
+
+CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureResultMetadata(params json)
+RETURNS cdb_observatory.ds_return_metadata
+AS $$
+DECLARE
+  colnames text[]; -- Array to store the name of the measures to be returned
+  coltypes text[]; -- Array to store the type of the measures to be returned
+  requested_measures text[];
+  measure_id text;
+BEGIN
+  -- By definition, all the measure results for the OBS_GetMeasure API are numeric values
+  SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO requested_measures;
+
+  FOREACH measure_id IN ARRAY requested_measures
+  LOOP
+    SELECT array_append(colnames, measure_id) INTO colnames;
+    SELECT array_append(coltypes, 'numeric'::text) INTO coltypes;
+  END LOOP;
+
+  RETURN (colnames::text[], coltypes::text[]);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureQuery(table_schema text, table_name text, params json)
+RETURNS text
+AS $$
+DECLARE
+  data_query text;
+  measure_ids_arr text[];
+  measure_id text;
+  measures_list text;
+  measures_query text;
+  normalize text;
+  boundary_id text;
+  time_span text;
+  geom_table_name text;
+  data_table_name text;
+BEGIN
+    measures_query := '';
+    -- SELECT table_name from obs_meta WHERE boundary_id = {bound} AND [...] INTO geom_table_name
+    geom_table_name := 'observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308';
+    -- SELECT table_name from obs_meta WHERE time_span = {time} AND [...] INTO data_table_name
+    data_table_name := 'observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd';
+
+    -- Get measure_ids array from JSON
+    SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO measure_ids_arr;
+
+    -- Get a comma-separated list of measures ("total_pop, over_16_pop") to be used in SELECTs
+    SELECT array_to_string(measure_ids_arr, ',') INTO measures_list;
+
+    FOREACH measure_id IN ARRAY measure_ids_arr
+    LOOP
+      -- Build query to compute each value and normalize
+      -- Assumes the default normalization method, the normalize parameter given in the JSON
+      -- should be checked in order to build the final query
+      SELECT measures_query || ' sum(' || measure_id || '/fraction)::numeric as ' || measure_id || ', ' INTO measures_query;
+    END LOOP;
+
+    -- Data query should select the measures and the cartodb_id of the user table, in that order.
+    data_query := '(WITH _areas AS(SELECT ST_Area(a.the_geom::geography)'
+        || '/ (1000 * 1000) as fraction, a.geoid, b.cartodb_id FROM '
+        || geom_table_name || ' as a, '
+        || table_schema || '.' || table_name || ' AS b '
+        || 'WHERE b.the_geom && a.the_geom ), values AS (SELECT geoid, '
+        || measures_list
+        || ' FROM ' || data_table_name || ' ) '
+        || 'SELECT '
+        || measures_query
+        || ' cartodb_id::int FROM _areas, values '
+        || 'WHERE values.geoid = _areas.geoid GROUP BY cartodb_id);';
+    RETURN data_query;
+END;
+$$ LANGUAGE plpgsql;
+