Green/Blue deploy system for the observatory functions

server/extension/Makefile

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
 DATA =  $(NEW_EXTENSION_ARTIFACT) \
   $(OLD_VERSIONS) \
-  cdb_dataservices_server--0.7.0--0.7.1.sql \
+  cdb_dataservices_server--0.7.1--0.7.2.sql \
-  cdb_dataservices_server--0.7.1--0.7.0.sql
+  cdb_dataservices_server--0.7.2--0.7.1.sql
 
 REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
 TEST_DIR = test/

server/extension/cdb_dataservices_server--0.7.1--0.7.2.sql

@@ -0,0 +1,115 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.2'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_demographic_snapshot(text, text, geometry(Geometry, 4326), text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_segment_snapshot(text, text, geometry(Geometry, 4326), text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
+  username TEXT,
+  orgname TEXT
+)
+RETURNS text AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  return user_obs_snapshot_config.connection_str
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT '2009 - 2013',
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_dataservices_server._obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT '2009 - 2013',
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
+      result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_dataservices_server._obs_get_segment_snapshot (username, orgname, geom, geometry_level);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
+      result = plpy.execute(obs_plan, [geom, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server--0.7.2--0.7.1.sql

@@ -0,0 +1,84 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.1'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_server._obs_server_conn_str(text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_demographic_snapshot(text, text, geometry(Geometry, 4326), text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_segment_snapshot(text, text, geometry(Geometry, 4326), text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._obs_get_demographic_snapshot(text, text, geometry(Geometry, 4326), text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._obs_get_segment_snapshot(text, text, geometry(Geometry, 4326), text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT '2009 - 2013',
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
+      result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
+      result = plpy.execute(obs_plan, [geom, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.7.1'
+default_version = '0.7.2'
-requires = 'plpythonu, postgis, cdb_geocoder'
+requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/110_data_observatory.sql

@@ -1,9 +1,38 @@
+--
+-- Observatory connection config
+--
+-- The purpose of this function is provide to the PL/Proxy functions
+-- the connection string needed to connect with the current production database
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
+  username TEXT,
+  orgname TEXT)
+RETURNS text AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  return user_obs_snapshot_config.connection_str
+$$ LANGUAGE plpythonu;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
   username TEXT,
   orgname TEXT,
   geom geometry(Geometry, 4326),
   time_span TEXT DEFAULT '2009 - 2013',
   geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_dataservices_server._obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT '2009 - 2013',
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
 RETURNS json AS $$
   from cartodb_services.metrics import QuotaService
   import json
@@ -42,6 +71,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
   orgname TEXT,
   geom geometry(Geometry, 4326),
   geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_dataservices_server._obs_get_segment_snapshot (username, orgname, geom, geometry_level);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
 RETURNS json AS $$
   from cartodb_services.metrics import QuotaService
   import json
@@ -73,4 +112,4 @@ RETURNS json AS $$
       plpy.error(error_msg)
   finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu;
+$$ LANGUAGE plpythonu;

server/extension/test/expected/00_install_test.out

@@ -2,6 +2,7 @@
 CREATE EXTENSION postgis;
 CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
+CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
 CREATE EXTENSION observatory VERSION 'dev';
@@ -38,7 +39,7 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null
  
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
  cdb_conf_setconf 
 ------------------
  

server/extension/test/sql/00_install_test.sql

@@ -2,6 +2,7 @@
 CREATE EXTENSION postgis;
 CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
+CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
 CREATE EXTENSION observatory VERSION 'dev';
@@ -15,7 +16,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
-SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
 
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -14,9 +14,10 @@ class ServiceConfig(object):
         self._redis_connection = redis_connection
         self._username = username
         self._orgname = orgname
-        self._db_config = ServicesDBConfig(db_conn)
+        self._db_config = ServicesDBConfig(db_conn, username, orgname)
         if redis_connection:
-            self._redis_config = ServicesRedisConfig(redis_connection).build(username, orgname)
+            self._redis_config = ServicesRedisConfig(redis_connection).build(
+                username, orgname)
         else:
             self._redis_config = None
 
@@ -53,6 +54,7 @@ class ObservatorySnapshotConfig(ServiceConfig):
             self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
         else:
             self._monthly_quota = float(self._db_config.data_observatory_monthly_quota)
+        self._connection_str = self._db_config.data_observatory_connection_str
 
     @property
     def service_type(self):
@@ -70,6 +72,9 @@ class ObservatorySnapshotConfig(ServiceConfig):
     def soft_limit(self):
         return self._soft_limit
 
+    @property
+    def connection_str(self):
+        return self._connection_str
 
 class RoutingConfig(ServiceConfig):
 
@@ -323,8 +328,10 @@ class GeocoderConfig(ServiceConfig):
 
 class ServicesDBConfig:
 
-    def __init__(self, db_conn):
+    def __init__(self, db_conn, username, orgname):
         self._db_conn = db_conn
+        self._username = username
+        self._orgname = orgname
         return self._build()
 
     def _build(self):
@@ -364,6 +371,12 @@ class ServicesDBConfig:
         else:
             do_conf = json.loads(do_conf_json)
             self._data_observatory_monthly_quota = do_conf['monthly_quota']
+            if self._orgname and self._orgname in do_conf['connection']['whitelist']:
+                self._data_observatory_connection_str = do_conf['connection']['staging']
+            elif self._username in do_conf['connection']['whitelist']:
+                self._data_observatory_connection_str = do_conf['connection']['staging']
+            else:
+                self._data_observatory_connection_str = do_conf['connection']['production']
 
     def _get_logger_config(self):
         logger_conf_json = self._get_conf('logger_conf')
@@ -425,6 +438,10 @@ class ServicesDBConfig:
     def data_observatory_monthly_quota(self):
         return self._data_observatory_monthly_quota
 
+    @property
+    def data_observatory_connection_str(self):
+        return self._data_observatory_connection_str
+
 
 class ServicesRedisConfig:
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.5.1',
+    version='0.5.2',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_helper.py

@@ -61,4 +61,4 @@ def _plpy_execute_side_effect(*args, **kwargs):
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
         return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
-        return [{'conf': '{"monthly_quota": 100000}'}]
+        return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'}]