cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Green/Blue deploy system for the observatory functions

Browse Source
This commit is contained in:
Mario de Frutos 2016-05-03 13:32:44 +02:00
parent 97b415080d
commit e81dadaf2e
13 changed files with 269 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/extension/Makefile
View File

@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS) \
cdb_dataservices_server--0.7.0--0.7.1.sql \
cdb_dataservices_server--0.7.1--0.7.0.sql
cdb_dataservices_server--0.7.1--0.7.2.sql \
cdb_dataservices_server--0.7.2--0.7.1.sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
TEST_DIR = test/

115
server/extension/cdb_dataservices_server--0.7.1--0.7.2.sql Normal file
View File

@ -0,0 +1,115 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.2'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_demographic_snapshot(text, text, geometry(Geometry, 4326), text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_segment_snapshot(text, text, geometry(Geometry, 4326), text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
username TEXT,
orgname TEXT
)
RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
return user_obs_snapshot_config.connection_str
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_dataservices_server._obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_dataservices_server._obs_get_segment_snapshot (username, orgname, geom, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

84
server/extension/cdb_dataservices_server--0.7.2--0.7.1.sql Normal file
View File

@ -0,0 +1,84 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.1'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_server._obs_server_conn_str(text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_demographic_snapshot(text, text, geometry(Geometry, 4326), text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server.obs_get_segment_snapshot(text, text, geometry(Geometry, 4326), text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._obs_get_demographic_snapshot(text, text, geometry(Geometry, 4326), text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._obs_get_segment_snapshot(text, text, geometry(Geometry, 4326), text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

4
server/extension/cdb_dataservices_server.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.7.1'
requires = 'plpythonu, postgis, cdb_geocoder'
default_version = '0.7.2'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql → server/extension/old_versions/cdb_dataservices_server--0.7.0--0.7.1.sql
View File

0
server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql → server/extension/old_versions/cdb_dataservices_server--0.7.1--0.7.0.sql
View File

0
server/extension/cdb_dataservices_server--0.7.1.sql → server/extension/old_versions/cdb_dataservices_server--0.7.1.sql
View File

41
server/extension/sql/110_data_observatory.sql
View File

@ -1,9 +1,38 @@
--
-- Observatory connection config
--
-- The purpose of this function is provide to the PL/Proxy functions
-- the connection string needed to connect with the current production database
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
username TEXT,
orgname TEXT)
RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
return user_obs_snapshot_config.connection_str
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_dataservices_server._obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
@ -42,6 +71,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_dataservices_server._obs_get_segment_snapshot (username, orgname, geom, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
@ -73,4 +112,4 @@ RETURNS json AS $$
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
$$ LANGUAGE plpythonu;

3
server/extension/test/expected/00_install_test.out
View File

@ -2,6 +2,7 @@
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
CREATE EXTENSION observatory VERSION 'dev';
@ -38,7 +39,7 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null
(1 row)
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
cdb_conf_setconf
------------------

3
server/extension/test/sql/00_install_test.sql
View File

@ -2,6 +2,7 @@
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
CREATE EXTENSION observatory VERSION 'dev';
@ -15,7 +16,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)

23
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -14,9 +14,10 @@ class ServiceConfig(object):
self._redis_connection = redis_connection
self._username = username
self._orgname = orgname
self._db_config = ServicesDBConfig(db_conn)
self._db_config = ServicesDBConfig(db_conn, username, orgname)
if redis_connection:
self._redis_config = ServicesRedisConfig(redis_connection).build(username, orgname)
self._redis_config = ServicesRedisConfig(redis_connection).build(
username, orgname)
else:
self._redis_config = None
@ -53,6 +54,7 @@ class ObservatorySnapshotConfig(ServiceConfig):
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
else:
self._monthly_quota = float(self._db_config.data_observatory_monthly_quota)
self._connection_str = self._db_config.data_observatory_connection_str
@property
def service_type(self):
@ -70,6 +72,9 @@ class ObservatorySnapshotConfig(ServiceConfig):
def soft_limit(self):
return self._soft_limit
@property
def connection_str(self):
return self._connection_str
class RoutingConfig(ServiceConfig):
@ -323,8 +328,10 @@ class GeocoderConfig(ServiceConfig):
class ServicesDBConfig:
def __init__(self, db_conn):
def __init__(self, db_conn, username, orgname):
self._db_conn = db_conn
self._username = username
self._orgname = orgname
return self._build()
def _build(self):
@ -364,6 +371,12 @@ class ServicesDBConfig:
else:
do_conf = json.loads(do_conf_json)
self._data_observatory_monthly_quota = do_conf['monthly_quota']
if self._orgname and self._orgname in do_conf['connection']['whitelist']:
self._data_observatory_connection_str = do_conf['connection']['staging']
elif self._username in do_conf['connection']['whitelist']:
self._data_observatory_connection_str = do_conf['connection']['staging']
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
@ -425,6 +438,10 @@ class ServicesDBConfig:
def data_observatory_monthly_quota(self):
return self._data_observatory_monthly_quota
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str
class ServicesRedisConfig:

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.5.1',
version='0.5.2',
description='CartoDB Services API Python Library',

2
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -61,4 +61,4 @@ def _plpy_execute_side_effect(*args, **kwargs):
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"monthly_quota": 100000}'}]
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'}]