From 945c6cd685cba64658d065991406619b0e98fc20 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 16 Mar 2017 19:12:39 +0100 Subject: [PATCH] WIP: support for storing rate limits configuration --- server/extension/sql/15_config_helper.sql | 18 +++++++++ .../refactor/config/rate_limits.py | 37 +++++++++++++++++++ .../refactor/storage/redis_config.py | 6 +++ .../refactor/storage/server_config.py | 10 +++++ .../cartodb_services/tools/service_manager.py | 37 +++++++++++++++---- 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 9d83ebd..95eccf8 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -16,6 +16,24 @@ RETURNS JSON AS $$ SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; $$ LANGUAGE SQL STABLE SECURITY DEFINER; +CREATE OR REPLACE +FUNCTION cartodb.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cartodb.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL VOLATILE; + +CREATE OR REPLACE +FUNCTION cartodb.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL VOLATILE; + + CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) RETURNS boolean AS $$ cache_key = "user_geocoder_config_{0}".format(username) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py index 6f6d0df..4079d0d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py @@ -65,3 +65,40 @@ class RateLimitsConfigBuilder(object): self._username, rate_limit.get('limit', None), rate_limit.get('period', None)) + + +class RateLimitsConfigSetter(object): + + def __init__(self, service, username, orgname=None): + self._service = service + self._service_config = ServerConfiguration(service, username, orgname) + + def set_user_rate_limits(self, rate_limits_config): + # Note we allow copying a config from another user/service, so we + # ignore rate_limits:config.service and rate_limits:config.username + rate_limit_key = "{0}_rate_limit".format(service) + if rate_limits_config.is_limited(): + rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period} + self.service_config.user.set(rate_limit_key, rate_limit) + else + self.service_config.user.remove(rate_limit_key) + + def set_org_rate_limits(self, rate_limits_config): + rate_limit_key = "{0}_rate_limit".format(service) + if rate_limits_config.is_limited(): + rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period} + self.service_config.org.set(rate_limit_key, rate_limit) + else + self.service_config.org.remove(rate_limit_key) + + def set_server_rate_limits(self, rate_limits_config): + rate_limits = self.service_config.server.get('rate_limits', {}) + if rate_limits_config.is_limited(): + rate_limits[self._service] = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period} + else + rate_limits.pop(self._service, None) + if rate_limits: + self.service_config.server.set('rate_limits', rate_limits) + else + self.service_config.server.remove('rate_limits') + diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index 1e1f33c..cc35c24 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -17,6 +17,12 @@ class RedisConfigStorage(ConfigBackendInterface): else: return self._data.get(key, default) + def set(self, key, value): + self._connection.hset(self._config_key, key, value) + + def remove(self, key): + self._connection.hdel(self._config_key, key) + class RedisUserConfigStorageBuilder(object): def __init__(self, redis_connection, username): self._redis_connection = redis_connection diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 1d3603c..7ee3fe5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -19,3 +19,13 @@ class InDbServerConfigStorage(ConfigBackendInterface): raise KeyError else: return default + + def set(self, key, config): + json_config = json.dumps(config) + quoted_config = cartodb_services.plpy.quote_nullable(json_config) + sql = "SELECT cdb_dataservices_server.cdb_conf_setconf('{0}', {0})".format(key, quoted_config) + cartodb_services.plpy.execute(sql) + + def remove(self, key): + sql = "SELECT cdb_dataservices_server.cdb_conf_removeconf('{0}')".format(key) + cartodb_services.plpy.execute(sql) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py index 06005d7..a6d40ac 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py @@ -53,6 +53,30 @@ class ServiceManagerBase: def logger(self): return self.logger + +class ServiceConfiguration: + def __init__(self, service, username, orgname): + self._server_config_backend = ServerConfigBackendFactory().get() + self._environment = ServerEnvironmentBuilder(server_config_backend).get() + self._user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + self._org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + @property + def environment(self): + return self._environment + + @property + def server(self): + return self._server_config_backend + + @property + def user(self): + return self._user_config_backend + + @property + def org(self): + return self._org_config_backend + class ServiceManager(ServiceManagerBase): """ This service manager delegates the configuration parameter details, @@ -61,18 +85,15 @@ class ServiceManager(ServiceManagerBase): """ def __init__(self, service, config_builder, username, orgname): - server_config_backend = ServerConfigBackendFactory().get() - environment = ServerEnvironmentBuilder(server_config_backend).get() - user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() - org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + service_config = ServerConfiguration(service, username, orgname) - logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get() self.logger = Logger(logger_config) - self.config = config_builder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() - rate_limit_config = RateLimitsConfigBuilder(server_config_backend, user_config_backend, org_config_backend, service=service, user=username, org=orgname).get() + self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get() + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, user=username, org=orgname).get() - redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get() self.rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection) self.quota_service = QuotaService(self.config, redis_metrics_connection)