Remove DO Python code and SQL tests

NEWS.md

@@ -1,7 +1,8 @@
 Mar 4th, 2021
 =============
 * Version `0.39.4` of the server extension
-    * Remove Data Observatory v1
+* Version `0.23.6` of the Python library
+    * Remove Data Observatory
 
 Jan 26th, 2021
 ==============

client/renderer/interface.yaml

@@ -311,390 +311,6 @@
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
     - { name: units, type: "text", default: "'kilometers'"}
 
-- name: obs_get_demographic_snapshot
-  return_type: json
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: time_span,  type: "text", default: "'2009 - 2013'::text" }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_get_segment_snapshot
-  return_type: json
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_getdemographicsnapshot
-  return_type: SETOF JSON
-  multi_row: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: time_span,  type: "text", default: 'NULL' }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_getsegmentsnapshot
-  return_type: SETOF JSON
-  multi_row: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_getboundary
-  return_type: public.Geometry
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getboundaryid
-  return_type: text
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getboundarybyid
-  return_type: public.Geometry
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geometry_id,  type: text }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getboundariesbygeometry
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getboundariesbypointandradius
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: radius, type: numeric }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getpointsbygeometry
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getpointsbypointandradius
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "public.geometry(Geometry, 4326)" }
-    - { name: radius, type: numeric }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getmeasure
-  return_type: numeric
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: public.Geometry }
-    - { name: measure_id, type: text }
-    - { name: normalize, type: text, default: 'NULL'}
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getmeasurebyid
-  return_type: numeric
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom_ref,  type: text }
-    - { name: measure_id, type: text }
-    - { name: boundary_id, type: text}
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getdata
-  return_type: TABLE(id int, data json)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: id, type: integer }
-    - { name: data, type: json }
-  params:
-    - { name: geomvals,  type: "geomval[]" }
-    - { name: params, type: json }
-    - { name: merge, type: boolean, default: true }
-
-- name: obs_getdata
-  return_type: TABLE(id text, data json)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: id, type: text }
-    - { name: data, type: json }
-  params:
-    - { name: geomrefs,  type: "text[]" }
-    - { name: params, type: json }
-
-- name: obs_getmeta
-  return_type: json
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom_ref,  type: "public.Geometry(Geometry, 4326)" }
-    - { name: params, type: json }
-    - { name: max_timespan_rank, type: integer, default: 'NULL' }
-    - { name: max_score_rank, type: integer, default: 'NULL' }
-    - { name: target_geoms, type: integer, default: 'NULL' }
-
-- name: obs_metadatavalidation
-  return_type: TABLE(valid boolean, errors text[])
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom_extent,  type: "public.Geometry(Geometry, 4326)" }
-    - { name: geom_type,  type: text }
-    - { name: params, type: json }
-    - { name: target_geoms, type: integer, default: 'NULL' }
-
-- name: obs_getcategory
-  return_type: text
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: public.Geometry }
-    - { name: category_id, type: text }
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getuscensusmeasure
-  return_type: numeric
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: public.Geometry }
-    - { name: name, type: text }
-    - { name: normalize, type: text, default: 'NULL'}
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getuscensuscategory
-  return_type: text
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: public.Geometry }
-    - { name: name, type: text }
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getpopulation
-  return_type: numeric
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: geom,  type: public.Geometry }
-    - { name: normalize, type: text, default: 'NULL'}
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_search
-  return_type: TABLE(id text, description text, name text, aggregate text, source text)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: id, type: text }
-    - { name: description, type: text }
-    - { name: name, type: text }
-    - { name: aggregate, type: text }
-    - { name: source, type: text }
-  params:
-    - { name: search_term, type: text }
-    - { name: relevant_boundary, type: text, default: 'NULL' }
-
-- name: obs_getavailableboundaries
-  return_type: TABLE(boundary_id text, description text, time_span text, tablename text)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  table_fields:
-    - { name: boundary_id, type: text }
-    - { name: description, type: text }
-    - { name: time_span, type: text }
-    - { name: tablename, type: text }
-  params:
-    - { name: geom,  type: public.Geometry }
-    - { name: timespan, type: text, default: 'NULL'}
-
-- name: obs_dumpversion
-  return_type: text
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - {}
-
-- name: obs_getavailablenumerators
-  return_type: SETOF cdb_dataservices_client.obs_meta_numerator
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: bounds, type: "public.geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: denom_id, type: text, default: 'NULL' }
-    - { name: geom_id, type: text, default: 'NULL' }
-    - { name: timespan, type: text, default: 'NULL'}
-
-- name: _obs_getnumerators
-  return_type: SETOF cdb_dataservices_client.obs_meta_numerator
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: bounds, type: "public.geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: section_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: subsection_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: other_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: ids, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: name, type: text, default: 'NULL' }
-    - { name: denom_id, type: text, default: "''" }
-    - { name: geom_id, type: text, default: "''" }
-    - { name: timespan, type: text, default: "''"}
-
-- name: obs_getavailabledenominators
-  return_type: SETOF cdb_dataservices_client.obs_meta_denominator
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: bounds, type: "public.geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: numer_id, type: text, default: 'NULL' }
-    - { name: geom_id, type: text, default: 'NULL' }
-    - { name: timespan, type: text, default: 'NULL'}
-
-- name: obs_getavailablegeometries
-  return_type: SETOF cdb_dataservices_client.obs_meta_geometry
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: bounds, type: "public.geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: numer_id, type: text, default: 'NULL' }
-    - { name: denom_id, type: text, default: 'NULL' }
-    - { name: timespan, type: text, default: 'NULL'}
-    - { name: number_geometries, type: integer, default: 'NULL'}
-
-- name: obs_getavailabletimespans
-  return_type: SETOF cdb_dataservices_client.obs_meta_timespan
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: bounds, type: "public.geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: numer_id, type: text, default: 'NULL' }
-    - { name: denom_id, type: text, default: 'NULL' }
-    - { name: geom_id, type: text, default: 'NULL'}
-
-- name: obs_legacybuildermetadata
-  return_type: TABLE(name text, subsection json)
-  multi_row: true
-  multi_field: true
-  requires_permission: true
-  permission_name: observatory
-  permission_error: Data Observatory permission denied
-  params:
-    - { name: aggregate_type, type: text, default: 'NULL' }
-
 - name: cdb_service_quota_info
   return_type: SETOF service_quota_info
   multi_row: true

client/sql/16_custom_types.sql

@@ -16,22 +16,11 @@ CREATE TYPE cdb_dataservices_client.simple_route AS (
     duration integer
 );
 
--- For the OBS_Meta functions
-CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean);
-
-CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean);
-
-CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
-
-CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
-
-
 -- For quotas and services configuration
 CREATE TYPE cdb_dataservices_client.service_type AS ENUM (
     'isolines',
     'hires_geocoder',
-    'routing',
+    'routing'
-    'observatory'
 );
 
 CREATE TYPE cdb_dataservices_client.service_quota_info AS (

client/sql/20_table_augmentation.sql

@@ -1,279 +0,0 @@
-CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
-CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-DECLARE
-  username text;
-  user_db_role text;
-  orgname text;
-  user_schema text;
-  result boolean;
-BEGIN
-  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
-    RAISE EXCEPTION 'The api_key must be provided';
-  END IF;
-
-  SELECT session_user INTO user_db_role;
-
-  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
-  -- JSON value stored "" is taken as literal
-  IF username IS NULL OR username = '' OR username = '""' THEN
-    RAISE EXCEPTION 'Username is a mandatory argument';
-  END IF;
-
-  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
-    user_schema := 'public';
-  ELSE
-    user_schema := username;
-  END IF;
-
-  SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
-      username,
-      orgname,
-      user_db_role,
-      user_schema,
-      output_table_name,
-      params
-  ) INTO result;
-
-  RETURN result;
-END;
-$$  LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE
-    SET search_path = pg_temp;
-
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(
-    table_name text,
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-DECLARE
-  username text;
-  user_db_role text;
-  orgname text;
-  dbname text;
-  user_schema text;
-  result boolean;
-BEGIN
-  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
-    RAISE EXCEPTION 'The api_key must be provided';
-  END IF;
-
-  SELECT session_user INTO user_db_role;
-
-  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
-  -- JSON value stored "" is taken as literal
-  IF username IS NULL OR username = '' OR username = '""' THEN
-    RAISE EXCEPTION 'Username is a mandatory argument';
-  END IF;
-
-  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
-    user_schema := 'public';
-  ELSE
-    user_schema := username;
-  END IF;
-
-  SELECT current_database() INTO dbname;
-
-  SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
-      username,
-      orgname,
-      user_db_role,
-      user_schema,
-      dbname,
-      table_name,
-      output_table_name,
-      params
-  ) INTO result;
-
-  RETURN result;
-END;
-$$  LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE
-    SET search_path = pg_temp;
-
-
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
-    username text,
-    orgname text,
-    user_db_role text,
-    user_schema text,
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-    function_name = 'OBS_GetMeasure'
-    # Obtain return types for augmentation procedure
-    ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
-        "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
-        .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            function_name=plpy.quote_literal(function_name),
-            params=plpy.quote_literal(params)
-            )
-        )
-    if ds_return_metadata[0]["colnames"]:
-        colnames_arr = ds_return_metadata[0]["colnames"]
-        coltypes_arr = ds_return_metadata[0]["coltypes"]
-    else:
-        raise Exception('Error retrieving OBS_GetMeasure metadata')
-
-
-    # Prepare column and type strings required in the SQL queries
-    columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
-    columns_with_types = ','.join(columns_with_types_arr)
-
-    # Create a new table with the required columns
-    plpy.execute('CREATE TABLE "{schema}".{table_name} ( '
-        'cartodb_id int, the_geom public.geometry, {columns_with_types} '
-        ');'
-        .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types)
-        )
-
-    plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
-        .format(schema=user_schema, table_name=output_table_name, user=user_db_role)
-        )
-
-    return True
-$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
-    username text,
-    orgname text,
-    user_db_role text,
-    user_schema text,
-    dbname text,
-    table_name text,
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-    function_name = 'OBS_GetMeasure'
-    # Obtain return types for augmentation procedure
-    ds_return_metadata = plpy.execute(
-        "SELECT colnames, coltypes "
-        "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            function_name=plpy.quote_literal(function_name),
-            params=plpy.quote_literal(params)))
-
-    if ds_return_metadata[0]["colnames"]:
-        colnames_arr = ds_return_metadata[0]["colnames"]
-        coltypes_arr = ds_return_metadata[0]["coltypes"]
-    else:
-        raise Exception('Error retrieving OBS_GetMeasure metadata')
-
-    # Prepare column and type strings required in the SQL queries
-    columns_with_types_arr = [
-        colnames_arr[i] +
-        ' ' +
-        coltypes_arr[i] for i in range(
-            0,
-            len(colnames_arr))]
-    columns_with_types = ','.join(columns_with_types_arr)
-    aliased_colname_list = ','.join(
-        ['result.' + name for name in colnames_arr])
-
-    # Instruct the OBS server side to establish a FDW
-    # The metadata is obtained as well in order to:
-    #   - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
-    #   - (b) be able to tell OBS to free resources when done.
-    ds_fdw_metadata = plpy.execute(
-        "SELECT schemaname, tabname, servername "
-        "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, "
-        "{schema}::text, {dbname}::text, {table_name}::text);" .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            user_db_role=plpy.quote_literal(user_db_role),
-            schema=plpy.quote_literal(user_schema),
-            dbname=plpy.quote_literal(dbname),
-            table_name=plpy.quote_literal(table_name)))
-
-    if ds_fdw_metadata[0]["schemaname"]:
-        server_schema = ds_fdw_metadata[0]["schemaname"]
-        server_table_name = ds_fdw_metadata[0]["tabname"]
-        server_name = ds_fdw_metadata[0]["servername"]
-    else:
-        raise Exception('Error connecting dataset via FDW')
-
-    # Create a new table with the required columns
-    plpy.execute(
-        'INSERT INTO "{schema}".{analysis_table_name} '
-        'SELECT ut.cartodb_id, ut.the_geom, {colname_list} '
-        'FROM "{schema}".{table_name} ut '
-        'LEFT JOIN cdb_dataservices_client._DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, '
-        '{function_name}::text, {params}::json) '
-        'AS result ({columns_with_types}, cartodb_id int)  '
-        'ON result.cartodb_id = ut.cartodb_id;' .format(
-            schema=user_schema,
-            analysis_table_name=output_table_name,
-            colname_list=aliased_colname_list,
-            table_name=table_name,
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            server_schema=plpy.quote_literal(server_schema),
-            server_table_name=plpy.quote_literal(server_table_name),
-            function_name=plpy.quote_literal(function_name),
-            params=plpy.quote_literal(params),
-            columns_with_types=columns_with_types))
-
-    # Wipe user FDW data from the server
-    wiped = plpy.execute(
-        "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, "
-        "{server_table_name}::text, {fdw_server}::text)" .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            server_schema=plpy.quote_literal(server_schema),
-            server_table_name=plpy.quote_literal(server_table_name),
-            fdw_server=plpy.quote_literal(server_name)))
-
-    return True
-$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
-    username text,
-    orgname text,
-    user_db_role text,
-    user_schema text,
-    dbname text,
-    table_name text
-)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_ConnectUserTable;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata(
-    username text,
-    orgname text,
-    function_name text,
-    params json
-) RETURNS cdb_dataservices_client.ds_return_metadata AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_GetReturnMetadata;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData(
-    username text,
-    orgname text,
-    table_schema text,
-    table_name text,
-    function_name text,
-    params json
-) RETURNS SETOF record AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable(
-    username text,
-    orgname text,
-    table_schema text,
-    table_name text,
-    server_name text
-) RETURNS boolean AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_DisconnectUserTable;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;

client/sql/35_precheck_functions.sql

@@ -1,54 +0,0 @@
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck(
-    source_query text,
-    parameters json
-) RETURNS boolean AS $$
-DECLARE
-    errors text[];
-    validator_errors text[];
-    validator_error text;
-    valid boolean;
-    geoms record;
-BEGIN
-    errors := (ARRAY[])::TEXT[];
-    FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type,
-                                 bool_and(st_isvalid(the_geom)) as valid,
-                                 avg(st_npoints(the_geom)) as avg_vertex,
-                                 ST_SetSRID(ST_Extent(the_geom), 4326) as extent,
-                                 count(*)::INT as numgeoms
-                                 FROM (%s) as _source
-                                 GROUP BY ST_GeometryType(the_geom)', source_query)
-    LOOP
-        IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN
-            errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type));
-        END IF;
-
-        IF geoms.valid IS FALSE THEN
-            errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them');
-        END IF;
-
-        -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here
-        -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813
-        IF geoms.avg_vertex > 1000 THEN
-            errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them');
-        END IF;
-
-        -- OBS specific part
-        EXECUTE 'SELECT valid, errors
-        FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)'
-        INTO valid, validator_errors
-        USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms;
-        IF valid is FALSE THEN
-            FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors
-            LOOP
-                errors := array_append(errors, validator_error);
-            END LOOP;
-        END IF;
-    END LOOP;
-
-    IF CARDINALITY(errors) > 0 THEN
-        RAISE EXCEPTION '%', errors;
-    END IF;
-
-    RETURN TRUE;
-END;
-$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE;

client/sql/95_grant_execute_manual.sql

@@ -1,6 +1,3 @@
-GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser;
-GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser;
-GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser;
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser;
 
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser;

client/test/expected/95_data_observatory_tables_test.out

@@ -1,77 +0,0 @@
--- Add to the search path the schema
-SET search_path TO public,cartodb,cdb_dataservices_client;
-CREATE TABLE my_table(cartodb_id int);
-INSERT INTO my_table (cartodb_id) VALUES (1);
--- Mock the server functions
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
-RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
-BEGIN
-  RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
-END;
-$$ LANGUAGE 'plpgsql';
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
-RETURNS cdb_dataservices_client.ds_return_metadata AS $$
-BEGIN
-  RETURN (Array['total_pop'], Array['double precision']);
-END;
-$$ LANGUAGE 'plpgsql';
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
-RETURNS RECORD AS $$
-BEGIN
-  RETURN (23.4::double precision, 1::int);
-END;
-$$ LANGUAGE 'plpgsql';
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
-RETURNS boolean AS $$
-BEGIN
-  RETURN true;
-END;
-$$ LANGUAGE 'plpgsql';
-SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}');
- cdb_conf_setconf 
-------------------
- 
-(1 row)
-
--- Create a sample user table
-CREATE TABLE user_table (cartodb_id int, the_geom geometry);
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (2, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (3, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
--- Prepare a table with the total_pop column
-SELECT cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure('my_table_dst', '{"dummy":"dummy"}'::json);
- _dst_preparetableobs_getmeasure 
----------------------------------
- t
-(1 row)
-
--- The table should now exist and be empty
-SELECT * FROM my_table_dst;
- cartodb_id | the_geom | total_pop 
-------------+----------+-----------
-(0 rows)
-
--- Populate the table with measurement data
-SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'my_table_dst', '{"dummy":"dummy"}'::json);
- _dst_populatetableobs_getmeasure 
-----------------------------------
- t
-(1 row)
-
--- The table should now show the results
-SELECT * FROM my_table_dst;
- cartodb_id |                      the_geom                      | total_pop 
-------------+----------------------------------------------------+-----------
-          1 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |      23.4
-          2 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |          
-          3 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |          
-(3 rows)
-
--- Clean tables
-DROP TABLE my_table_dst;
-SELECT CDB_Conf_RemoveConf('api_keys_postgres');
- cdb_conf_removeconf 
----------------------
- 
-(1 row)
-

client/test/sql/95_data_observatory_tables_test.sql

@@ -1,60 +0,0 @@
--- Add to the search path the schema
-SET search_path TO public,cartodb,cdb_dataservices_client;
-
-CREATE TABLE my_table(cartodb_id int);
-
-INSERT INTO my_table (cartodb_id) VALUES (1);
-
--- Mock the server functions
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
-RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
-BEGIN
-  RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
-END;
-$$ LANGUAGE 'plpgsql';
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
-RETURNS cdb_dataservices_client.ds_return_metadata AS $$
-BEGIN
-  RETURN (Array['total_pop'], Array['double precision']);
-END;
-$$ LANGUAGE 'plpgsql';
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
-RETURNS RECORD AS $$
-BEGIN
-  RETURN (23.4::double precision, 1::int);
-END;
-$$ LANGUAGE 'plpgsql';
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
-RETURNS boolean AS $$
-BEGIN
-  RETURN true;
-END;
-$$ LANGUAGE 'plpgsql';
-
-SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}');
-
--- Create a sample user table
-CREATE TABLE user_table (cartodb_id int, the_geom geometry);
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (2, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (3, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-
--- Prepare a table with the total_pop column
-SELECT cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure('my_table_dst', '{"dummy":"dummy"}'::json);
-
--- The table should now exist and be empty
-SELECT * FROM my_table_dst;
-
--- Populate the table with measurement data
-SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'my_table_dst', '{"dummy":"dummy"}'::json);
-
--- The table should now show the results
-SELECT * FROM my_table_dst;
-
--- Clean tables
-DROP TABLE my_table_dst;
-
-SELECT CDB_Conf_RemoveConf('api_keys_postgres');

docs/developer-center/reference/08-demographic-functions.md

@@ -1,5 +0,0 @@
-### Demographic Functions
-
-<div class="is-caption is-caption--mobile header-warning is-semibold u-vspace--16">
-  <p>We will no longer support these demographic functions from the 1st of January 2021. If you want to collect demographic reports around a point location, we recommend using Data Observatory from our Python library <a href="https://carto.com/developers/cartoframes/">CARTOframes</a> instead.</p>
-</div>

docs/developer-center/reference/09-segmentation-functions.md

@@ -1,5 +0,0 @@
-## Segmentation Functions
-
-<div class="is-caption is-caption--mobile header-warning is-semibold u-vspace--16">
-  <p>We will no longer support these segmentations functions from the 1st of January 2021. For a similar functionality, we recommend using Data Observatory from our Python library <a href="https://carto.com/developers/cartoframes/">CARTOframes</a> instead.</p>
-</div>

docs/developer-center/support/03-quota-information.md

@@ -39,7 +39,6 @@ Service Types:
 * `'isolines'` [Isoline/Isochrones (isochrone/isodistance lines) service]({{site.dataservicesapi_docs}}/reference/#isoline_functions/)
 * `'hires_geocoder'` [Street level geocoding]({{site.dataservicesapi_docs}}/reference/#street-level-geocoder)
 * `'routing'` [Routing functions]({{site.dataservicesapi_docs}}/reference/#routing_functions/)
-* `'observatory'` Data Observatory services ([demographic]({{site.dataservicesapi_docs}}/reference/#demographic_functions/) and [segmentation]({{site.dataservicesapi_docs}}/reference/#segmentation_functions/) functions)
 
 **Notes**
 
@@ -62,12 +61,11 @@ Result:
  isolines       |           100 |          0 | f          | tomtom
  hires_geocoder |           100 |          0 | f          | tomtom
  routing        |            50 |          0 | f          | tomtom
- observatory    |             0 |          0 | f          | data observatory
 (4 rows)
 
 ```
 
-In this case, notice that the user has no access to the observatory services. All quotas are *hard-limited* (no soft limits), and no quota has been used in the present period.
+All quotas are *hard-limited* (no soft limits), and no quota has been used in the present period.
 
 ### cdb_enough_quota(service text ,input_size numeric)
 

docs/internal/quota_management.md

@@ -59,6 +59,6 @@ This operation could be done through the rails console:
 
 ### How is the quota spent?###
 
-Almost in all the services: geocoding, data observatory snapshot and general the number of spent credits is calculated per request made (either successful or empty request).
+Almost in all the services: geocoding and general the number of spent credits is calculated per request made (either successful or empty request).
 
 In the case of the isolines service, the number of credits is calculated based on the number of isolines generated by the request. Ie. If your query generates 3 isolines for the request, you've spent 3 isolines credits.

server/extension/carto-package.json

@@ -7,8 +7,7 @@
       "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
-      "dataservices-api-server-python-lib": "^0.23.1",
+      "dataservices-api-server-python-lib": "^0.23.1"
-      "observatory-server-extension": "^1.9.0"
     }
   }
 }

server/extension/sql/200_quotas.sql

@@ -6,8 +6,7 @@ BEGIN
     CREATE TYPE cdb_dataservices_server.service_type AS ENUM (
       'isolines',
       'hires_geocoder',
-      'routing',
+      'routing'
-      'observatory'
     );
   END IF;
 END $$;

server/extension/test/expected/00_install_test.out

@@ -53,12 +53,6 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null
   
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
- cdb_conf_setconf 
-------------------
- 
-(1 row)
-
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)
 CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$

server/extension/test/sql/00_install_test.sql

@@ -15,7 +15,6 @@ SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routi
 SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
-SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
 
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,4 +1,4 @@
-from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
+from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException
 from cartodb_services.metrics.quota import QuotaService
 from cartodb_services.metrics.user import UserMetricsService
 from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -57,57 +57,6 @@ class ServiceConfig(object):
             return None
 
 
-class DataObservatoryConfig(ServiceConfig):
-
-    METRICS_LOG_KEY = 'do_log_path'
-
-    def __init__(self, redis_connection, db_conn, username, orgname=None):
-        super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
-                                            username, orgname)
-
-    @property
-    def monthly_quota(self):
-        return self._monthly_quota
-
-    @property
-    def period_end_date(self):
-        return self._period_end_date
-
-    @property
-    def soft_limit(self):
-        return self._soft_limit
-
-    @property
-    def connection_str(self):
-        return self._connection_str
-
-    @property
-    def provider(self):
-        return 'data observatory'
-
-
-class ObservatoryConfig(DataObservatoryConfig):
-
-    SOFT_LIMIT_KEY = 'soft_obs_general_limit'
-    QUOTA_KEY = 'obs_general_quota'
-    PERIOD_END_DATE = 'period_end_date'
-
-    def __init__(self, redis_connection, db_conn, username, orgname=None):
-        super(ObservatoryConfig, self).__init__(redis_connection, db_conn,
-                                            username, orgname)
-        self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
-        if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
-            self._soft_limit = True
-        else:
-            self._soft_limit = False
-        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
-        self._connection_str = self._db_config.data_observatory_connection_str
-
-    @property
-    def service_type(self):
-        return 'obs_general'
-
-
 class RoutingConfig(ServiceConfig):
 
     PERIOD_END_DATE = 'period_end_date'
@@ -624,7 +573,6 @@ class ServicesDBConfig:
         self._get_mapbox_config()
         self._get_tomtom_config()
         self._get_geocodio_config()
-        self._get_data_observatory_config()
 
     def _get_server_config(self):
         server_config_json = self._get_conf('server_conf')
@@ -716,19 +664,6 @@ class ServicesDBConfig:
             self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
             self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
 
-    def _get_data_observatory_config(self):
-        do_conf_json = self._get_conf('data_observatory_conf')
-        if not do_conf_json:
-            raise ConfigException('Data Observatory configuration missing')
-        else:
-            do_conf = json.loads(do_conf_json)
-            if self._orgname and self._orgname in do_conf['connection']['whitelist']:
-                self._data_observatory_connection_str = do_conf['connection']['staging']
-            elif self._username in do_conf['connection']['whitelist']:
-                self._data_observatory_connection_str = do_conf['connection']['staging']
-            else:
-                self._data_observatory_connection_str = do_conf['connection']['production']
-
     def _get_conf(self, key):
         try:
             sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
@@ -897,10 +832,6 @@ class ServicesDBConfig:
     def geocodio_geocoder_service_params(self):
         return self._geocodio_geocoder_service_params
 
-    @property
-    def data_observatory_connection_str(self):
-        return self._data_observatory_connection_str
-
     @property
     def logger_config(self):
         logger_conf_json = self._get_conf('logger_conf')
@@ -917,7 +848,6 @@ class ServicesRedisConfig:
     QUOTA_KEY = 'geocoding_quota'
     ISOLINES_QUOTA_KEY = 'here_isolines_quota'
     ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
-    OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
     PERIOD_END_DATE = 'period_end_date'
     GEOCODER_PROVIDER_KEY = 'geocoder_provider'
     ISOLINES_PROVIDER_KEY = 'isolines_provider'
@@ -960,8 +890,6 @@ class ServicesRedisConfig:
                 user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
             if self.ROUTING_QUOTA_KEY in org_config:
                 user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
-            if self.OBS_GENERAL_QUOTA_KEY in org_config:
-                user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
             if self.PERIOD_END_DATE in org_config:
                 user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
             if self.GOOGLE_GEOCODER_CLIENT_ID in org_config:

server/lib/python/cartodb_services/cartodb_services/metrics/log.py

@@ -127,8 +127,6 @@ class MetricsServiceLoggerFactory:
             return MetricsGenericLogger(service_config, logger)
         elif re.search('_isolines$', service_config.service_type):
             return MetricsIsolinesLogger(service_config, logger)
-        elif re.search('^obs_*', service_config.service_type):
-            return MetricsGenericLogger(service_config, logger)
         else:
             return None
 

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -78,9 +78,6 @@ class QuotaChecker:
         elif re.match('^routing_',
                       self._user_service_config.service_type) is not None:
             return self.__check_routing_quota()
-        elif re.match('^obs_',
-                      self._user_service_config.service_type) is not None:
-            return self.__check_data_observatory_quota()
         else:
             return False
 
@@ -123,18 +120,3 @@ class QuotaChecker:
             return True
         else:
             return False
-
-    def __check_data_observatory_quota(self):
-        user_quota = self._user_service_config.monthly_quota
-        soft_limit = self._user_service_config.soft_limit
-        today = date.today()
-        service_type = self._user_service_config.service_type
-        current_used = self._user_service.used_quota(service_type, today)
-
-        # Quick workaround so we don't take into account numer of credits
-        # spent for users that have defined the quota.
-        # See https://github.com/CartoDB/bigmetadata/issues/215
-        if soft_limit or (user_quota > 0):
-            return True
-        else:
-            return False

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -26,7 +26,6 @@ class UserMetricsService:
     SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
     SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
     SERVICE_TOMTOM_ROUTING = 'routing_tomtom'
-    SERVICE_OBSERVATORY = 'obs_general'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
@@ -45,8 +44,6 @@ class UserMetricsService:
                               self.SERVICE_MAPBOX_ROUTING,
                               self.SERVICE_TOMTOM_ROUTING]:
             return self.__used_routing_quota(service_type, date)
-        elif service_type == self.SERVICE_OBSERVATORY:
-            return self.__used_observatory_quota(service_type, date)
         else:
             return self.__used_geocoding_quota(service_type, date)
 
@@ -97,19 +94,6 @@ class UserMetricsService:
 
         return current_use
 
-    def __used_observatory_quota(self, service_type, date):
-        date_from, date_to = self.__current_billing_cycle()
-        current_use = 0
-        success_responses = self.get_metrics(service_type,
-                                             'success_responses', date_from,
-                                             date_to)
-        empty_responses = self.get_metrics(service_type,
-                                           'empty_responses', date_from,
-                                           date_to)
-        current_use += (success_responses + empty_responses)
-
-        return current_use
-
     def increment_service_use(self, service_type, metric, date=date.today(),
                               amount=1):
         """ Increment the services uses in monthly and daily basis"""

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.23.5',
+    version='0.23.6',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -344,102 +344,6 @@ class TestRoutingConfig(TestCase):
         assert config.soft_limit == True
 
 
-class TestDataObservatoryUserConfig(TestCase):
-
-    def setUp(self):
-        self.redis_conn = MockRedis()
-        plpy_mock_config()
-
-    def test_should_return_config_for_obs_config(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100, end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.monthly_quota == 100
-        assert do_config.soft_limit is False
-        assert do_config.period_end_date.date() == yesterday.date()
-
-    def test_should_return_0_if_quota_is_0_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                              'test_user')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_0_if_quota_is_empty_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota='', end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                              'test_user')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_true_if_soft_limit_is_true_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, soft_limit=True, end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.soft_limit is True
-
-    def test_should_return_true_if_soft_limit_is_empty_string_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, soft_limit='', end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.soft_limit is False
-
-class TestDataObservatoryOrgConfig(TestCase):
-
-    def setUp(self):
-        self.redis_conn = MockRedis()
-        plpy_mock_config()
-
-    def test_should_return_organization_config(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100, end_date=yesterday)
-        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
-                                quota=200, end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                      'test_user', 'test_org')
-        assert do_config.monthly_quota == 200
-        assert do_config.period_end_date.date() == yesterday.date()
-
-    def test_should_return_quota_0_for_0_in_org_quota_config(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100)
-        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
-                                quota=0, end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                      'test_user', 'test_org')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_quota_0_for_empty_in_org_quota_config(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100)
-        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
-                                quota='', end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                      'test_user', 'test_org')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_user_config_when_org_quota_is_not_defined(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100)
-        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
-                                quota=None, end_date=yesterday)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                      'test_user', 'test_org')
-        assert do_config.monthly_quota == 100
-
-
 class TestServicesRedisConfig(TestCase):
     def test_it_picks_mapzen_routing_quota_from_redis(self):
         redis_conn = MockRedis()

server/lib/python/cartodb_services/test/test_helper.py

@@ -27,9 +27,6 @@ def build_redis_user_config(redis_conn, username, service, quota=100,
         redis_conn.hset(user_redis_name, 'routing_provider', provider)
         redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota))
         redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower())
-    elif service is 'data_observatory':
-        redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota))
-        redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower())
 
     redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
     redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
@@ -53,9 +50,6 @@ def build_redis_org_config(redis_conn, orgname, service, quota=100,
         redis_conn.hset(org_redis_name, 'routing_provider', provider)
         if quota is not None:
             redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota))
-    elif service is 'data_observatory':
-        if quota is not None:
-            redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota))
 
     redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
     redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
@@ -80,6 +74,5 @@ def plpy_mock_config():
     plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
-    plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
     plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}])

server/lib/python/cartodb_services/test/test_quota_service.py

@@ -178,17 +178,6 @@ class TestQuotaService(TestCase):
         qs.increment_isolines_service_use(amount=1500000)
         assert qs.check_user_quota() is False
 
-    # Quick workaround so we don't take into account numer of credits
-    # spent for users that have defined the quota.
-    # See https://github.com/CartoDB/bigmetadata/issues/215
-    def test_should_check_org_obs_quota_correctly(self):
-        qs = self.__build_obs_quota_service('test_user',
-                                            orgname='testorg')
-        qs.increment_success_service_use()
-        assert qs.check_user_quota() is True
-        qs.increment_success_service_use(amount=100000)
-        assert qs.check_user_quota() is True
-
     def __prepare_quota_service(self, username, service, quota, provider,
                                 orgname, soft_limit, end_date):
         build_redis_user_config(self.redis_conn, username, service,
@@ -226,14 +215,3 @@ class TestQuotaService(TestCase):
         isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
                                                username, orgname)
         return QuotaService(isolines_config, redis_connection=self.redis_conn)
-
-    def __build_obs_quota_service(self, username, quota=100,
-                                  provider='obs_general',
-                                  orgname=None,
-                                  soft_limit=False,
-                                  end_date=datetime.today()):
-        self.__prepare_quota_service(username, 'data_observatory', quota,
-                                     None, orgname, soft_limit, end_date)
-        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
-                                          username, orgname)
-        return QuotaService(do_config, redis_connection=self.redis_conn)

test/README.md

@@ -23,7 +23,6 @@ This suite of tests test the following parts of the geocoding API through the SQ
 - Street address functions (This will call Heremaps or Google so it will cost you 2 credits)
 - Routing functions
 - Isolines functions
-- Data Observatory functions
 
 
 ### How to debug the tests
@@ -46,7 +45,7 @@ GEOCODER_API_TEST_USERNAME=your_username \
 GEOCODER_API_TEST_API_KEY=your_api_key \
 GEOCODER_API_TEST_TABLE_NAME=your_test_table \
 GEOCODER_API_TEST_HOST=your_target_test_host \
-nosetests --where=integration/ test_data_observatory_functions.py:TestDataObservatoryFunctions.test_if_obs_search_is_ok
+nosetests --where=integration/ test_admin1_functions.py:TestAdmin1Functions.test_if_select_with_admin1_without_country_is_ok
 ```
 
 (replace the environment variables, test file, class and function according to your needs)

test/integration/test_data_observatory_functions.py

@@ -1,329 +0,0 @@
-from unittest import TestCase
-from nose.tools import assert_raises
-from nose.tools import assert_not_equal, assert_in
-from ..helpers.integration_test_helper import IntegrationTestHelper
-
-
-class TestDataObservatoryFunctions(TestCase):
-
-    def setUp(self):
-        self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
-            self.env_variables['schema'],
-            self.env_variables['username'],
-            self.env_variables['host'],
-        )
-
-    def test_if_get_demographic_snapshot_is_ok(self):
-        query = "SELECT obs_GetDemographicSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['snapshot'], None)
-
-    def test_if_get_demographic_snapshot_without_api_key_raise_error(self):
-        query = "SELECT obs_GetDemographicSnapshot(CDB_LatLng(40.704512, -73.936669));"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getdemographicsnapshot(geometry) does not exist"])
-
-    def test_if_get_segment_snapshot_is_ok(self):
-        query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['snapshot'], None)
-
-    def test_if_get_segment_snapshot_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669));"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getsegmentsnapshot(geometry) does not exist"])
-
-    def test_if_get_measure_with_point_is_ok(self):
-        query = "SELECT OBS_GetMeasure(CDB_LatLng(40.704512, -73.936669), 'us.census.acs.B01003001') as measure;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['measure'], None)
-
-    def test_if_get_measure_with_area_is_ok(self):
-        query = "SELECT OBS_GetMeasure('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'us.census.acs.B01003001') as measure;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['measure'], None)
-
-    def test_if_get_measure_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetMeasure(CDB_LatLng(40.704512, -73.936669), 'us.census.acs.B01003001');"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeasure(geometry, unknown) does not exist"])
-
-    def test_if_get_measure_by_id_ok(self):
-        query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['measure'], None)
-
-    def test_if_get_measure_by_id_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeasurebyid(unknown, unknown, unknown, unknown) does not exist"])
-
-    def test_if_get_category_is_ok(self):
-        query = "SELECT OBS_GetCategory(CDB_LatLng(40.704512, -73.936669), 'us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014') as category;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['category'], None)
-
-    def test_if_get_category_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetCategory(CDB_LatLng(40.704512, -73.936669), 'us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014');"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getcategory(geometry, unknown, unknown, unknown) does not exist"])
-
-    def test_if_get_us_census_measure_with_point_is_ok(self):
-        query = "SELECT OBS_GetUSCensusMeasure(CDB_LatLng(40.704512, -73.936669), 'male population') as measure;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['measure'], None)
-
-    def test_if_get_us_census_measure_with_area_is_ok(self):
-        query = "SELECT OBS_GetUSCensusMeasure('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'male population') as measure;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['measure'], None)
-
-    def test_if_get_us_census_measure_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetUSCensusMeasure('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'male population');"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getuscensusmeasure(geometry, unknown) does not exist"])
-
-    def test_if_get_us_census_category_is_ok(self):
-        query = "SELECT OBS_GetUSCensusCategory('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'Spielman-Singleton Segments: 10 Clusters') as category;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['category'], None)
-
-    def test_if_get_us_census_category_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetUSCensusCategory('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'Spielman-Singleton Segments: 10 Clusters');"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getuscensuscategory(geometry, unknown) does not exist"])
-
-    def test_if_get_population_is_ok(self):
-        query = "SELECT OBS_GetPopulation(CDB_LatLng(40.704512, -73.936669)) as population;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['population'], None)
-
-    def test_if_get_population_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetPopulation(CDB_LatLng(40.704512, -73.936669));"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getpopulation(geometry) does not exist"])
-
-    def test_if_obs_search_is_ok(self):
-        sql = "SELECT id FROM OBS_Search('total_pop') WHERE id LIKE 'es.ine%' LIMIT 1;"
-        import urllib
-        query = "{0}&api_key={1}".format(urllib.quote(sql), self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['id'], None)
-
-    def test_if_obs_search_without_api_key_raise_error(self):
-        query = "SELECT id FROM OBS_Search('total_pop') LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_search(unknown) does not exist"])
-
-    def test_if_obs_get_available_boundaries_is_ok(self):
-        query = "SELECT boundary_id FROM OBS_GetAvailableBoundaries(CDB_LatLng(40.704512, -73.936669)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['boundary_id'], None)
-
-    def test_if_obs_get_available_boundaries_without_api_key_raise_error(self):
-        query = "SELECT boundary_id FROM OBS_GetAvailableBoundaries(CDB_LatLng(40.704512, -73.936669)) LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailableboundaries(geometry) does not exist"])
-
-    def test_if_obs_get_boundary_is_ok(self):
-        query = "SELECT OBS_GetBoundary(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract') as boundary;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['boundary'], None)
-
-    def test_if_obs_get_boundary_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetBoundary(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract') as boundary;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundary(geometry, unknown) does not exist"])
-
-    def test_if_obs_get_boundary_id_is_ok(self):
-        query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['boundary_id'], None)
-
-    def test_if_obs_get_boundary_id_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundaryid(geometry, unknown, unknown) does not exist"])
-
-    def test_if_obs_get_boundary_by_id_is_ok(self):
-        query = "SELECT OBS_GetBoundaryById('36047', 'us.census.tiger.county', '2014') as boundary;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['boundary'], None)
-
-    def test_if_obs_get_boundary_by_id_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetBoundaryById('36047', 'us.census.tiger.county', '2014') as boundary_id;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundarybyid(unknown, unknown, unknown) does not exist"])
-
-    def test_if_obs_get_boundaries_by_geometry_is_ok(self):
-        query = "SELECT geom_refs FROM OBS_GetBoundariesByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['geom_refs'], None)
-
-    def test_if_obs_get_boundaries_by_geometry_without_api_key_raise_error(self):
-        query = "SELECT geom_refs FROM OBS_GetBoundariesByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundariesbygeometry(geometry, unknown) does not exist"])
-
-    def test_if_obs_get_boundaries_by_point_and_radius_is_ok(self):
-        query = "SELECT geom_refs FROM OBS_GetBoundariesByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['geom_refs'], None)
-
-    def test_if_obs_get_boundaries_by_point_and_radius_without_api_key_raise_error(self):
-        query = "SELECT geom_refs FROM OBS_GetBoundariesByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundariesbypointandradius(geometry, integer, unknown) does not exist"])
-
-    def test_if_obs_get_points_by_geometry_is_ok(self):
-        query = "SELECT geom_refs FROM OBS_GetPointsByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['geom_refs'], None)
-
-    def test_if_obs_get_points_by_geometry_without_api_key_raise_error(self):
-        query = "SELECT geom_refs FROM OBS_GetPointsByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getpointsbygeometry(geometry, unknown) does not exist"])
-
-    def test_if_obs_get_points_by_point_and_radius_is_ok(self):
-        query = "SELECT geom_refs FROM OBS_GetPointsByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract', '2014') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['geom_refs'], None)
-
-    def test_if_obs_get_points_by_point_and_radius_without_api_key_raise_error(self):
-        query = "SELECT geom_refs FROM OBS_GetPointsByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract', '2014') ORDER BY geom_refs ASC LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getpointsbypointandradius(geometry, integer, unknown, unknown) does not exist"])
-
-    def test_if_obs_get_legacy_builder_metadata_is_ok(self):
-        query = "SELECT name FROM OBS_LegacyBuilderMetadata() LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['name'], None)
-
-    def test_if_legacy_builder_metadata_without_api_key_raise_error(self):
-        query = "SELECT name FROM OBS_LegacyBuilderMetadata() LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_legacybuildermetadata() does not exist"])
-
-    def test_if_obs_get_available_numerators_is_ok(self):
-        query = "SELECT numer_id FROM OBS_GetAvailableNumerators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['numer_id'], None)
-
-    def test_if_obs_get_available_numerators_without_api_key_raise_error(self):
-        query = "SELECT numer_id FROM OBS_GetAvailableNumerators() LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailablenumerators() does not exist"])
-
-    def test_if_obs_get_available_denominators_is_ok(self):
-        query = "SELECT denom_id FROM OBS_GetAvailableDenominators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['denom_id'], None)
-
-    def test_if_obs_get_available_denominators_without_api_key_raise_error(self):
-        query = "SELECT denom_id FROM OBS_GetAvailableDenominators() LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailabledenominators() does not exist"])
-
-    def test_if_obs_get_available_geometries_is_ok(self):
-        query = "SELECT geom_id FROM OBS_GetAvailableGeometries() LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['geom_id'], None)
-
-    def test_if_obs_get_available_geometries_without_api_key_raise_error(self):
-        query = "SELECT geom_id FROM OBS_GetAvailableGeometries() LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailablegeometries() does not exist"])
-
-    def test_if_obs_get_available_timespans_is_ok(self):
-        query = "SELECT timespan_id FROM OBS_GetAvailableTimespans() LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['timespan_id'], None)
-
-    def test_if_obs_get_available_timespans_without_api_key_raise_error(self):
-        query = "SELECT timespan_id FROM OBS_GetAvailableTimespans() LIMIT 1;"
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailabletimespans() does not exist"])
-
-    def test_if_obs_get_meta_is_ok(self):
-        params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
-        query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), {0}, 1, 1, 1000) as metadata LIMIT 1;&api_key={1}".format(params, self.env_variables['api_key'])
-        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        assert_not_equal(result['metadata'], None)
-
-    def test_if_obs_get_meta_without_api_key_raise_error(self):
-        params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
-        query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), {0}, 1, 1, 1000) LIMIT 1;".format(params)
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"])
-
-    def test_if_obs_get_data_is_ok(self):
-        params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\''
-        params_2 = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
-        query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0}, 1, 1, 1000)) LIMIT 1;&api_key={1}".format(params_1, self.env_variables['api_key'])
-        query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0})) LIMIT 1;&api_key={1}".format(params_2,self.env_variables['api_key'])
-        result_1 = IntegrationTestHelper.execute_query(self.sql_api_url, query_1)
-        assert_not_equal(result_1['data_id'], None)
-        result_2 = IntegrationTestHelper.execute_query(self.sql_api_url, query_2)
-        assert_not_equal(result_2['data_id'], None)
-
-    def test_if_obs_get_data_without_api_key_raise_error(self):
-        params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\'';
-        params_2 = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
-        query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0}, 1, 1, 1000)) LIMIT 1;".format(params_1)
-        query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0})) LIMIT 1;".format(params_2)
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query_1)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"])
-        try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query_2)
-        except Exception as e:
-            assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeta(geometry, unknown) does not exist"])