Merge pull request #355 from CartoDB/346-user-rate-limits
Service rate limits
This commit is contained in:
Javier Goizueta
GitHub
commit
f0b0a9e7f2
@ -43,7 +43,6 @@ Steps to deploy a new Data Services API version :
|
||||
|
||||
```
|
||||
# in dataservices-api repo root path:
|
||||
cd dataservices-api
|
||||
cd client && sudo make install
|
||||
cd -
|
||||
cd server/extension && sudo make install
|
||||
@ -53,7 +52,7 @@ Steps to deploy a new Data Services API version :
|
||||
|
||||
```
|
||||
# in dataservices-api repo root path:
|
||||
cd server/lib/python/cartodb_services && pip install -r requirements.txt —upgrade
|
||||
cd server/lib/python/cartodb_services && pip install -r requirements.txt && sudo pip install . --upgrade
|
||||
```
|
||||
|
||||
- install extensions in user database
|
||||
@ -66,7 +65,6 @@ Steps to deploy a new Data Services API version :
|
||||
create extension cdb_dataservices_client;
|
||||
```
|
||||
|
||||
|
||||
### Server configuration
|
||||
|
||||
Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`.
|
||||
|
||||
@ -421,3 +421,29 @@
|
||||
params:
|
||||
- { name: service, type: TEXT }
|
||||
- { name: input_size, type: NUMERIC }
|
||||
|
||||
- name: cdb_service_get_rate_limit
|
||||
return_type: json
|
||||
params:
|
||||
- { name: service, type: "text" }
|
||||
|
||||
- name: cdb_service_set_user_rate_limit
|
||||
superuser: true
|
||||
return_type: void
|
||||
params:
|
||||
- { name: service, type: "text" }
|
||||
- { name: rate_limit, type: json }
|
||||
|
||||
- name: cdb_service_set_org_rate_limit
|
||||
superuser: true
|
||||
return_type: void
|
||||
params:
|
||||
- { name: service, type: "text" }
|
||||
- { name: rate_limit, type: json }
|
||||
|
||||
- name: cdb_service_set_server_rate_limit
|
||||
superuser: true
|
||||
return_type: void
|
||||
params:
|
||||
- { name: service, type: "text" }
|
||||
- { name: rate_limit, type: json }
|
||||
|
||||
@ -44,16 +44,29 @@ class SqlTemplateRenderer
|
||||
@function_signature['geocoder_config_key']
|
||||
end
|
||||
|
||||
def params
|
||||
@function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}
|
||||
def parameters_info(with_user_org)
|
||||
parameters = []
|
||||
if with_user_org
|
||||
parameters << { 'name' => 'username', 'type' => 'text' }
|
||||
parameters << { 'name' => 'orgname', 'type' => 'text' }
|
||||
end
|
||||
parameters + @function_signature['params'].reject(&:empty?)
|
||||
end
|
||||
|
||||
def params_with_type
|
||||
@function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }
|
||||
def user_org_declaration()
|
||||
"username text;\n orgname text;" unless superuser_function?
|
||||
end
|
||||
|
||||
def params_with_type_and_default
|
||||
parameters = @function_signature['params'].reject(&:empty?).map do |p|
|
||||
def params(with_user_org = superuser_function?)
|
||||
parameters_info(with_user_org).map { |p| p['name'].to_s }
|
||||
end
|
||||
|
||||
def params_with_type(with_user_org = superuser_function?)
|
||||
parameters_info(with_user_org).map { |p| "#{p['name']} #{p['type']}" }
|
||||
end
|
||||
|
||||
def params_with_type_and_default(with_user_org = superuser_function?)
|
||||
parameters = parameters_info(with_user_org).map do |p|
|
||||
if not p['default'].nil?
|
||||
"#{p['name']} #{p['type']} DEFAULT #{p['default']}"
|
||||
else
|
||||
@ -62,6 +75,49 @@ class SqlTemplateRenderer
|
||||
end
|
||||
return parameters
|
||||
end
|
||||
|
||||
def superuser_function?
|
||||
!!@function_signature['superuser']
|
||||
end
|
||||
|
||||
def void_return_type?
|
||||
return_type.downcase == 'void'
|
||||
end
|
||||
|
||||
def return_declaration
|
||||
"ret #{return_type};" unless void_return_type? || multi_row
|
||||
end
|
||||
|
||||
def return_statement(&block)
|
||||
if block
|
||||
erb_out = block.binding.eval('_erbout')
|
||||
|
||||
if multi_row
|
||||
erb_out << 'RETURN QUERY SELECT * FROM '
|
||||
elsif multi_field
|
||||
erb_out << 'SELECT * FROM '
|
||||
elsif void_return_type?
|
||||
erb_out << 'PERFORM '
|
||||
else
|
||||
erb_out << 'SELECT '
|
||||
end
|
||||
yield
|
||||
if multi_row || void_return_type?
|
||||
erb_out << ';'
|
||||
else
|
||||
erb_out << ' INTO ret;'
|
||||
end
|
||||
if !multi_row && !void_return_type?
|
||||
erb_out << ' RETURN ret;'
|
||||
end
|
||||
else
|
||||
if !multi_row && !void_return_type?
|
||||
' RETURN ret;'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
end
|
||||
|
||||
|
||||
|
||||
@ -7,9 +7,8 @@
|
||||
CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default.join(' ,') %>)
|
||||
RETURNS <%= return_type %> AS $$
|
||||
DECLARE
|
||||
<% if not multi_row %>ret <%= return_type %>;<% end %>
|
||||
username text;
|
||||
orgname text;
|
||||
<%= return_declaration if not multi_row %>
|
||||
<%= user_org_declaration %>
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
@ -19,15 +18,7 @@ BEGIN
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
<% if multi_row %>
|
||||
RETURN QUERY
|
||||
SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
|
||||
<% elsif multi_field %>
|
||||
SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
|
||||
RETURN ret;
|
||||
<% else %>
|
||||
SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
|
||||
RETURN ret;
|
||||
<% end %>
|
||||
|
||||
<% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %>
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
|
||||
|
||||
@ -5,9 +5,8 @@
|
||||
CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe (<%= params_with_type_and_default.join(' ,') %>)
|
||||
RETURNS <%= return_type %> AS $$
|
||||
DECLARE
|
||||
<% if not multi_row %>ret <%= return_type %>;<% end %>
|
||||
username text;
|
||||
orgname text;
|
||||
<%= return_declaration %>
|
||||
<%= user_org_declaration %>
|
||||
_returned_sqlstate TEXT;
|
||||
_message_text TEXT;
|
||||
_pg_exception_context TEXT;
|
||||
@ -21,41 +20,16 @@ BEGIN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
<% if multi_row %>
|
||||
BEGIN
|
||||
RETURN QUERY
|
||||
SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
|
||||
BEGIN
|
||||
<% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %>
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
END;
|
||||
<% elsif multi_field %>
|
||||
BEGIN
|
||||
SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
|
||||
RETURN ret;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
RETURN ret;
|
||||
END;
|
||||
<% else %>
|
||||
BEGIN
|
||||
SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
|
||||
RETURN ret;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
RETURN ret;
|
||||
END;
|
||||
<% end %>
|
||||
<%= return_statement %>
|
||||
END;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= ['username text', 'organization_name text'].concat(params_with_type_and_default).join(', ') %>)
|
||||
CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= params_with_type_and_default(_with_user_org=true).join(', ') %>)
|
||||
RETURNS <%= return_type %> AS $$
|
||||
CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str();
|
||||
<% if multi_field %>
|
||||
SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>);
|
||||
SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= params(_with_user_org=true).join(', ') %>);
|
||||
<% else %>
|
||||
SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>);
|
||||
SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= params(_with_user_org=true).join(', ') %>);
|
||||
<% end %>
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
@ -1,2 +1,4 @@
|
||||
<% unless superuser_function? %>
|
||||
GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %>) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %> ) TO publicuser;
|
||||
<% end %>
|
||||
195
doc/rate_limits.md
Normal file
195
doc/rate_limits.md
Normal file
@ -0,0 +1,195 @@
|
||||
# Rate limits
|
||||
|
||||
Services can be rate-limited. (currently only gecoding is limited)
|
||||
|
||||
The rate limits configuration can be established at server, organization or user levels, the latter having precedence over the earlier.
|
||||
|
||||
The default configuration (a null or empty configuration) doesn't impose any limits.
|
||||
|
||||
The configuration consist of a JSON object with two attributes:
|
||||
|
||||
* `period`: the rate-limiting period, in seconds.
|
||||
* `limit`: the maximum number of request in the established period.
|
||||
|
||||
If a service request exceeds the configured rate limits
|
||||
(i.e. if more than `limit` calls are performe in a fixed interval of
|
||||
duration `period` seconds) the call will fail with an "Rate limit exceeded" error.
|
||||
|
||||
## Server-side interface
|
||||
|
||||
There's a server-side SQL interface to query or change the configuration.
|
||||
|
||||
### cdb_dataservices_server.cdb_service_get_rate_limit(username, orgname, service)
|
||||
|
||||
This function returns the rate limit configuration for a given user and service.
|
||||
|
||||
#### Returns
|
||||
|
||||
The result is a JSON object with the configuration (`period` and `limit` attributes as explained above).
|
||||
|
||||
### cdb_dataservices_server.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
|
||||
|
||||
This function sets the rate limit configuration for the user. This overrides any other configuration.
|
||||
|
||||
The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`.
|
||||
|
||||
#### Returns
|
||||
|
||||
This functions doesn't return any value.
|
||||
|
||||
### cdb_dataservices_server.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
|
||||
|
||||
This function sets the rate limit configuration for the organization.
|
||||
This overrides server level configuration and is overriden by user configuration if present.
|
||||
|
||||
The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
|
||||
|
||||
#### Returns
|
||||
|
||||
This functions doesn't return any value.
|
||||
|
||||
### cdb_dataservices_server.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
|
||||
|
||||
This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration.
|
||||
|
||||
The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
|
||||
|
||||
#### Returns
|
||||
|
||||
This functions doesn't return any value.
|
||||
|
||||
## Client-side interface
|
||||
|
||||
For convenience there's also a client-side interface (in the client dataservices-api extension), consisting
|
||||
of public functions to get the current configuration and privileged functions to change it.
|
||||
|
||||
### Public functions
|
||||
|
||||
These functions are accesible to non-privileged roles, and should only be executed
|
||||
using the role corresponding to a CARTO user, since that will determine the
|
||||
user and organization to which the rate limits configuration applies.
|
||||
|
||||
### cdb_dataservices_client.cdb_service_get_rate_limit(service)
|
||||
|
||||
This function returns the rate limit configuration in effect for the specified service
|
||||
and the user corresponding to the role which makes the calls. The effective configuration
|
||||
may come from any of the configuration levels (server/organization/user); only the
|
||||
existing configuration with most precedence is returned.
|
||||
|
||||
#### Returns
|
||||
|
||||
The result is a JSON object with the configuration (`period` and `limit` attributes as explained above).
|
||||
|
||||
#### Example:
|
||||
|
||||
```
|
||||
SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoding');
|
||||
|
||||
cdb_service_get_rate_limit
|
||||
---------------------------------
|
||||
{"limit": 1000, "period": 86400}
|
||||
(1 row)
|
||||
```
|
||||
|
||||
|
||||
### Privileged (superuser) functions
|
||||
|
||||
Thes functions are not accessible by regular user roles, and the user and organization names must be provided as parameters.
|
||||
|
||||
### cdb_dataservices_client.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
|
||||
|
||||
This function sets the rate limit configuration for the user. This overrides any other configuration.
|
||||
|
||||
The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`.
|
||||
|
||||
#### Returns
|
||||
|
||||
This functions doesn't return any value.
|
||||
|
||||
#### Example
|
||||
|
||||
This will configure the geocoding service rate limit for user `myusername`, a non-organization user.
|
||||
The limit will be set at 1000 requests per day. Since the user doesn't belong to any organization,
|
||||
`NULL` will be passed to the organization argument; otherwise the name of the user's organization should
|
||||
be provided.
|
||||
|
||||
```
|
||||
SELECT cdb_dataservices_client.cdb_service_set_user_rate_limit(
|
||||
'myusername',
|
||||
NULL,
|
||||
'geocoding',
|
||||
'{"limit":1000,"period":86400}'
|
||||
);
|
||||
|
||||
cdb_service_set_user_rate_limit
|
||||
---------------------------------
|
||||
|
||||
(1 row)
|
||||
```
|
||||
|
||||
### cdb_dataservices_client.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
|
||||
|
||||
This function sets the rate limit configuration for the organization.
|
||||
This overrides server level configuration and is overriden by user configuration if present.
|
||||
|
||||
The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
|
||||
|
||||
#### Returns
|
||||
|
||||
This functions doesn't return any value.
|
||||
|
||||
#### Example
|
||||
|
||||
This will configure the geocoding service rate limit for the `myorg` organization.
|
||||
The limit will be set at 100 requests per hour.
|
||||
Note that even we're setting the default configuration for the whole organization,
|
||||
the name of a user of the organization must be provided for technical reasons.
|
||||
|
||||
```
|
||||
SELECT cdb_dataservices_client.cdb_service_set_org_rate_limit(
|
||||
'myorgadmin',
|
||||
'myorg',
|
||||
'geocoding',
|
||||
'{"limit":100,"period":3600}'
|
||||
);
|
||||
|
||||
|
||||
cdb_service_set_org_rate_limit
|
||||
---------------------------------
|
||||
|
||||
(1 row)
|
||||
```
|
||||
|
||||
### cdb_dataservices_client.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
|
||||
|
||||
This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration.
|
||||
|
||||
The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
|
||||
|
||||
#### Returns
|
||||
|
||||
This functions doesn't return any value.
|
||||
|
||||
#### Example
|
||||
|
||||
This will configure the default geocoding service rate limit for all users
|
||||
accesing the data-services server.
|
||||
The limit will be set at 10000 requests per month.
|
||||
Note that even we're setting the default configuration for the server,
|
||||
the name of a user and the name of the corresponding organization (or NULL)
|
||||
must be provided for technical reasons.
|
||||
|
||||
```
|
||||
SELECT cdb_dataservices_client.cdb_service_set_server_rate_limit(
|
||||
'myorgadmin',
|
||||
'myorg',
|
||||
'geocoding',
|
||||
'{"limit":10000,"period":108000}'
|
||||
);
|
||||
|
||||
|
||||
cdb_service_set_server_rate_limit
|
||||
---------------------------------
|
||||
|
||||
(1 row)
|
||||
```
|
||||
@ -16,6 +16,24 @@ RETURNS JSON AS $$
|
||||
SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key;
|
||||
$$ LANGUAGE SQL STABLE SECURITY DEFINER;
|
||||
|
||||
CREATE OR REPLACE
|
||||
FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON)
|
||||
RETURNS void AS $$
|
||||
BEGIN
|
||||
PERFORM cartodb.CDB_Conf_RemoveConf(key);
|
||||
EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL VOLATILE;
|
||||
|
||||
CREATE OR REPLACE
|
||||
FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text)
|
||||
RETURNS void AS $$
|
||||
BEGIN
|
||||
EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL VOLATILE;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
|
||||
RETURNS boolean AS $$
|
||||
cache_key = "user_geocoder_config_{0}".format(username)
|
||||
|
||||
@ -72,23 +72,15 @@ $$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import LegacyServiceManager
|
||||
from cartodb_services.here import HereMapsGeocoder
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params)
|
||||
geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
@ -96,46 +88,41 @@ RETURNS Geometry AS $$
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using here maps')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import LegacyServiceManager
|
||||
from cartodb_services.google import GoogleMapsGeocoder
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||
service_manager.assert_within_limits(quota=False)
|
||||
|
||||
try:
|
||||
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger)
|
||||
geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using google maps')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
@ -143,38 +130,19 @@ $$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger
|
||||
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
|
||||
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
|
||||
|
||||
server_config_backend = ServerConfigBackendFactory().get()
|
||||
environment = ServerEnvironmentBuilder(server_config_backend).get()
|
||||
user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
|
||||
org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
|
||||
logger = Logger(logger_config)
|
||||
|
||||
mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
|
||||
|
||||
redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
|
||||
|
||||
quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params)
|
||||
geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
country_iso3 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
@ -182,18 +150,18 @@ RETURNS Geometry AS $$
|
||||
state_province=state_province,
|
||||
country=country_iso3, search_type='address')
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapzen')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
91
server/extension/sql/210_rates.sql
Normal file
91
server/extension/sql/210_rates.sql
Normal file
@ -0,0 +1,91 @@
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
service TEXT)
|
||||
RETURNS JSON AS $$
|
||||
import json
|
||||
from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_config = ServiceConfiguration(service, username, orgname)
|
||||
rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
|
||||
if rate_limit_config.is_limited():
|
||||
return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period})
|
||||
else:
|
||||
return None
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
service TEXT,
|
||||
rate_limit_json JSON)
|
||||
RETURNS VOID AS $$
|
||||
import json
|
||||
from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
|
||||
if rate_limit_json:
|
||||
rate_limit = json.loads(rate_limit_json)
|
||||
limit = rate_limit.get('limit', None)
|
||||
period = rate_limit.get('period', None)
|
||||
else:
|
||||
limit = None
|
||||
period = None
|
||||
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
|
||||
config_setter.set_user_rate_limits(config)
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
service TEXT,
|
||||
rate_limit_json JSON)
|
||||
RETURNS VOID AS $$
|
||||
import json
|
||||
from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
|
||||
if rate_limit_json:
|
||||
rate_limit = json.loads(rate_limit_json)
|
||||
limit = rate_limit.get('limit', None)
|
||||
period = rate_limit.get('period', None)
|
||||
else:
|
||||
limit = None
|
||||
period = None
|
||||
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
|
||||
config_setter.set_org_rate_limits(config)
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
service TEXT,
|
||||
rate_limit_json JSON)
|
||||
RETURNS VOID AS $$
|
||||
import json
|
||||
from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
|
||||
if rate_limit_json:
|
||||
rate_limit = json.loads(rate_limit_json)
|
||||
limit = rate_limit.get('limit', None)
|
||||
period = rate_limit.get('period', None)
|
||||
else:
|
||||
limit = None
|
||||
period = None
|
||||
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
|
||||
config_setter.set_server_rate_limits(config)
|
||||
$$ LANGUAGE plpythonu;
|
||||
44
server/extension/test/expected/210_rates_test.out
Normal file
44
server/extension/test/expected/210_rates_test.out
Normal file
@ -0,0 +1,44 @@
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_get_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text');
|
||||
exists
|
||||
--------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_set_user_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
|
||||
exists
|
||||
--------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_set_org_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
|
||||
exists
|
||||
--------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_set_server_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
|
||||
exists
|
||||
--------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
27
server/extension/test/sql/210_rates_test.sql
Normal file
27
server/extension/test/sql/210_rates_test.sql
Normal file
@ -0,0 +1,27 @@
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_get_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text');
|
||||
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_set_user_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
|
||||
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_set_org_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
|
||||
|
||||
SELECT exists(SELECT *
|
||||
FROM pg_proc p
|
||||
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
|
||||
WHERE ns.nspname = 'cdb_dataservices_server'
|
||||
AND proname = 'cdb_service_set_server_rate_limit'
|
||||
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
|
||||
@ -0,0 +1,3 @@
|
||||
from service_configuration import ServiceConfiguration
|
||||
from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
|
||||
from legacy_rate_limits import RateLimitsConfigLegacyBuilder
|
||||
@ -0,0 +1,46 @@
|
||||
import json
|
||||
from rate_limits import RateLimitsConfig
|
||||
|
||||
class RateLimitsConfigLegacyBuilder(object):
|
||||
"""
|
||||
Build a RateLimitsConfig object using the *legacy* configuration classes
|
||||
"""
|
||||
|
||||
def __init__(self, redis_connection, db_conn, service, username, orgname):
|
||||
self._service = service
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
self._redis_connection = redis_connection
|
||||
self._db_conn = db_conn
|
||||
|
||||
def get(self):
|
||||
rate_limit = self.__get_rate_limit()
|
||||
return RateLimitsConfig(self._service,
|
||||
self._username,
|
||||
rate_limit.get('limit', None),
|
||||
rate_limit.get('period', None))
|
||||
|
||||
def __get_rate_limit(self):
|
||||
rate_limit = {}
|
||||
rate_limit_key = "{0}_rate_limit".format(self._service)
|
||||
user_key = "rails:users:{0}".format(self._username)
|
||||
rate_limit_json = self.__get_redis_config(user_key, rate_limit_key)
|
||||
if not rate_limit_json and self._orgname:
|
||||
org_key = "rails:orgs:{0}".format(self._orgname)
|
||||
rate_limit_json = self.__get_redis_config(org_key, rate_limit_key)
|
||||
if rate_limit_json:
|
||||
rate_limit = json.loads(rate_limit_json)
|
||||
else:
|
||||
conf_key = 'rate_limits'
|
||||
sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(conf_key)
|
||||
try:
|
||||
conf = self._db_conn.execute(sql, 1)[0]['conf']
|
||||
except Exception:
|
||||
conf = None
|
||||
if conf:
|
||||
rate_limit = json.loads(conf).get(self._service)
|
||||
return rate_limit or {}
|
||||
|
||||
def __get_redis_config(self, basekey, param):
|
||||
config = self._redis_connection.hgetall(basekey)
|
||||
return config and config.get(param)
|
||||
@ -0,0 +1,113 @@
|
||||
import json
|
||||
|
||||
from service_configuration import ServiceConfiguration
|
||||
|
||||
class RateLimitsConfig(object):
|
||||
"""
|
||||
Value object that represents the configuration needed to rate-limit services
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
service,
|
||||
username,
|
||||
limit,
|
||||
period):
|
||||
self._service = service
|
||||
self._username = username
|
||||
self._limit = limit and int(limit)
|
||||
self._period = period and int(period)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.__dict__ == other.__dict__
|
||||
|
||||
# service this limit applies to
|
||||
@property
|
||||
def service(self):
|
||||
return self._service
|
||||
|
||||
# user this limit applies to
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
# rate period in seconds
|
||||
@property
|
||||
def period(self):
|
||||
return self._period
|
||||
|
||||
# rate limit in seconds
|
||||
@property
|
||||
def limit(self):
|
||||
return self._limit
|
||||
|
||||
def is_limited(self):
|
||||
return self._limit and self._limit > 0 and self._period and self._period > 0
|
||||
|
||||
class RateLimitsConfigBuilder(object):
|
||||
"""
|
||||
Build a rate limits configuration obtaining the parameters
|
||||
from the user/org/server configuration.
|
||||
"""
|
||||
|
||||
def __init__(self, server_conf, user_conf, org_conf, service, username, orgname):
|
||||
self._server_conf = server_conf
|
||||
self._user_conf = user_conf
|
||||
self._org_conf = org_conf
|
||||
self._service = service
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
|
||||
def get(self):
|
||||
# Order of precedence is user_conf, org_conf, server_conf
|
||||
|
||||
rate_limit_key = "{0}_rate_limit".format(self._service)
|
||||
|
||||
rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None)
|
||||
if (rate_limit_json):
|
||||
rate_limit = rate_limit_json and json.loads(rate_limit_json)
|
||||
else:
|
||||
rate_limit = self._server_conf.get('rate_limits', {}).get(self._service, {})
|
||||
|
||||
return RateLimitsConfig(self._service,
|
||||
self._username,
|
||||
rate_limit.get('limit', None),
|
||||
rate_limit.get('period', None))
|
||||
|
||||
|
||||
class RateLimitsConfigSetter(object):
|
||||
|
||||
def __init__(self, service, username, orgname=None):
|
||||
self._service = service
|
||||
self._service_config = ServiceConfiguration(service, username, orgname)
|
||||
|
||||
def set_user_rate_limits(self, rate_limits_config):
|
||||
# Note we allow copying a config from another user/service, so we
|
||||
# ignore rate_limits:config.service and rate_limits:config.username
|
||||
rate_limit_key = "{0}_rate_limit".format(self._service)
|
||||
if rate_limits_config.is_limited():
|
||||
rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
|
||||
rate_limit_json = json.dumps(rate_limit)
|
||||
self._service_config.user.set(rate_limit_key, rate_limit_json)
|
||||
else:
|
||||
self._service_config.user.remove(rate_limit_key)
|
||||
|
||||
def set_org_rate_limits(self, rate_limits_config):
|
||||
rate_limit_key = "{0}_rate_limit".format(self._service)
|
||||
if rate_limits_config.is_limited():
|
||||
rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
|
||||
rate_limit_json = json.dumps(rate_limit)
|
||||
self._service_config.org.set(rate_limit_key, rate_limit_json)
|
||||
else:
|
||||
self._service_config.org.remove(rate_limit_key)
|
||||
|
||||
def set_server_rate_limits(self, rate_limits_config):
|
||||
rate_limits = self._service_config.server.get('rate_limits', {})
|
||||
if rate_limits_config.is_limited():
|
||||
rate_limits[self._service] = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
|
||||
else:
|
||||
rate_limits.pop(self._service, None)
|
||||
if rate_limits:
|
||||
self._service_config.server.set('rate_limits', rate_limits)
|
||||
else:
|
||||
self._service_config.server.remove('rate_limits')
|
||||
|
||||
@ -0,0 +1,36 @@
|
||||
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
|
||||
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
|
||||
|
||||
class ServiceConfiguration(object):
|
||||
"""
|
||||
This class instantiates configuration backend objects for all the configuration levels of a service:
|
||||
* environment
|
||||
* server
|
||||
* organization
|
||||
* user
|
||||
The configuration backends allow retrieval and modification of configuration parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, service, username, orgname):
|
||||
self._server_config_backend = ServerConfigBackendFactory().get()
|
||||
self._environment = ServerEnvironmentBuilder(self._server_config_backend ).get()
|
||||
self._user_config_backend = UserConfigBackendFactory(username, self._environment, self._server_config_backend ).get()
|
||||
self._org_config_backend = OrgConfigBackendFactory(orgname, self._environment, self._server_config_backend ).get()
|
||||
|
||||
@property
|
||||
def environment(self):
|
||||
return self._environment
|
||||
|
||||
@property
|
||||
def server(self):
|
||||
return self._server_config_backend
|
||||
|
||||
@property
|
||||
def user(self):
|
||||
return self._user_config_backend
|
||||
|
||||
@property
|
||||
def org(self):
|
||||
return self._org_config_backend
|
||||
@ -6,6 +6,6 @@ class ConfigBackendInterface(object):
|
||||
__metaclass__ = abc.ABCMeta
|
||||
|
||||
@abc.abstractmethod
|
||||
def get(self, key):
|
||||
def get(self, key, default=None):
|
||||
"""Return a value based on the key supplied from some storage"""
|
||||
pass
|
||||
|
||||
@ -5,8 +5,8 @@ class InMemoryConfigStorage(ConfigBackendInterface):
|
||||
def __init__(self, config_hash={}):
|
||||
self._config_hash = config_hash
|
||||
|
||||
def get(self, key):
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
return self._config_hash[key]
|
||||
except KeyError:
|
||||
return None
|
||||
return default
|
||||
|
||||
@ -2,5 +2,5 @@ from ..core.interfaces import ConfigBackendInterface
|
||||
|
||||
class NullConfigStorage(ConfigBackendInterface):
|
||||
|
||||
def get(self, key):
|
||||
return None
|
||||
def get(self, key, default=None):
|
||||
return default
|
||||
|
||||
@ -9,11 +9,19 @@ class RedisConfigStorage(ConfigBackendInterface):
|
||||
self._config_key = config_key
|
||||
self._data = None
|
||||
|
||||
def get(self, key):
|
||||
def get(self, key, default=KeyError):
|
||||
if not self._data:
|
||||
self._data = self._connection.hgetall(self._config_key)
|
||||
return self._data[key]
|
||||
if (default == KeyError):
|
||||
return self._data[key]
|
||||
else:
|
||||
return self._data.get(key, default)
|
||||
|
||||
def set(self, key, value):
|
||||
self._connection.hset(self._config_key, key, value)
|
||||
|
||||
def remove(self, key):
|
||||
self._connection.hdel(self._config_key, key)
|
||||
|
||||
class RedisUserConfigStorageBuilder(object):
|
||||
def __init__(self, redis_connection, username):
|
||||
|
||||
@ -4,11 +4,28 @@ from ..core.interfaces import ConfigBackendInterface
|
||||
|
||||
class InDbServerConfigStorage(ConfigBackendInterface):
|
||||
|
||||
def get(self, key):
|
||||
def get(self, key, default=None):
|
||||
sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key)
|
||||
rows = cartodb_services.plpy.execute(sql, 1)
|
||||
json_output = rows[0]['conf']
|
||||
if json_output:
|
||||
json_output = None
|
||||
try:
|
||||
json_output = rows[0]['conf']
|
||||
except (IndexError, KeyError):
|
||||
pass
|
||||
if (json_output):
|
||||
return json.loads(json_output)
|
||||
else:
|
||||
return None
|
||||
if (default == KeyError):
|
||||
raise KeyError
|
||||
else:
|
||||
return default
|
||||
|
||||
def set(self, key, config):
|
||||
json_config = json.dumps(config)
|
||||
quoted_config = cartodb_services.plpy.quote_nullable(json_config)
|
||||
sql = "SELECT cdb_dataservices_server.cdb_conf_setconf('{0}', {1})".format(key, quoted_config)
|
||||
cartodb_services.plpy.execute(sql)
|
||||
|
||||
def remove(self, key):
|
||||
sql = "SELECT cdb_dataservices_server.cdb_conf_removeconf('{0}')".format(key)
|
||||
cartodb_services.plpy.execute(sql)
|
||||
|
||||
@ -2,3 +2,6 @@ from redis_tools import RedisConnection, RedisDBConfig
|
||||
from coordinates import Coordinate
|
||||
from polyline import PolyLine
|
||||
from log import Logger, LoggerConfig
|
||||
from rate_limiter import RateLimiter
|
||||
from service_manager import ServiceManager, RateLimitExceeded
|
||||
from legacy_service_manager import LegacyServiceManager
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
from cartodb_services.tools import RateLimiter
|
||||
from cartodb_services.config import RateLimitsConfigLegacyBuilder
|
||||
from cartodb_services.tools.service_manager import ServiceManagerBase
|
||||
import plpy
|
||||
|
||||
class LegacyServiceManager(ServiceManagerBase):
|
||||
"""
|
||||
This service manager relies on cached configuration (in gd) stored in *legacy* configuration objects
|
||||
It's intended for use by the *legacy* configuration objects (in use prior to the configuration refactor).
|
||||
"""
|
||||
|
||||
def __init__(self, service, username, orgname, gd):
|
||||
redis_conn = gd["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
self.config = gd["user_{0}_config_{1}".format(service, username)]
|
||||
logger_config = gd["logger_config"]
|
||||
self.logger = Logger(logger_config)
|
||||
|
||||
self.quota_service = QuotaService(self.config, redis_conn)
|
||||
|
||||
rate_limit_config = RateLimitsConfigLegacyBuilder(redis_conn, plpy, service=service, username=username, orgname=orgname).get()
|
||||
self.rate_limiter = RateLimiter(rate_limit_config, redis_conn)
|
||||
@ -0,0 +1,18 @@
|
||||
from rratelimit import Limiter
|
||||
|
||||
class RateLimiter:
|
||||
|
||||
def __init__(self, rate_limits_config, redis_connection):
|
||||
self._config = rate_limits_config
|
||||
self._limiter = None
|
||||
if (self._config.is_limited()):
|
||||
self._limiter = Limiter(redis_connection,
|
||||
action=self._config.service,
|
||||
limit=self._config.limit,
|
||||
period=self._config.period)
|
||||
|
||||
def check(self):
|
||||
ok = True
|
||||
if (self._limiter):
|
||||
ok = self._limiter.checked_insert(self._config.username)
|
||||
return ok
|
||||
@ -0,0 +1,70 @@
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger
|
||||
from cartodb_services.tools import RateLimiter
|
||||
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
|
||||
from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
|
||||
|
||||
class RateLimitExceeded(Exception):
|
||||
def __str__(self):
|
||||
return repr('Rate limit exceeded')
|
||||
|
||||
class ServiceManagerBase:
|
||||
"""
|
||||
A Service manager collects the configuration needed to use a service,
|
||||
including thir-party services parameters.
|
||||
|
||||
This abstract class serves as the base for concrete service manager classes;
|
||||
derived class must provide and initialize attributes for ``config``,
|
||||
``quota_service``, ``logger`` and ``rate_limiter`` (which can be None
|
||||
for no limits).
|
||||
|
||||
It provides an `assert_within_limits` method to check quota and rate limits
|
||||
which raises exceptions when limits are exceeded.
|
||||
|
||||
It exposes properties containing:
|
||||
|
||||
* ``config`` : a configuration object containing the configuration parameters for
|
||||
a given service and provider.
|
||||
* ``quota_service`` a QuotaService object to for quota accounting
|
||||
* ``logger``
|
||||
|
||||
"""
|
||||
|
||||
def assert_within_limits(self, quota=True, rate=True):
|
||||
if rate and not self.rate_limiter.check():
|
||||
raise RateLimitExceeded()
|
||||
if quota and not self.quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self.config
|
||||
|
||||
@property
|
||||
def quota_service(self):
|
||||
return self.quota_service
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
return self.logger
|
||||
class ServiceManager(ServiceManagerBase):
|
||||
"""
|
||||
This service manager delegates the configuration parameter details,
|
||||
and the policies about configuration precedence to a configuration-builder class.
|
||||
It uses the refactored configuration classes.
|
||||
"""
|
||||
|
||||
def __init__(self, service, config_builder, username, orgname):
|
||||
service_config = ServiceConfiguration(service, username, orgname)
|
||||
|
||||
logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get()
|
||||
self.logger = Logger(logger_config)
|
||||
|
||||
self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get()
|
||||
rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
|
||||
|
||||
redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get()
|
||||
|
||||
self.rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection)
|
||||
self.quota_service = QuotaService(self.config, redis_metrics_connection)
|
||||
@ -4,7 +4,8 @@ python-dateutil==2.2
|
||||
googlemaps==2.4.2
|
||||
rollbar==0.13.2
|
||||
# Dependency for googlemaps package
|
||||
requests<=2.9.1
|
||||
requests==2.9.1
|
||||
rratelimit==0.0.4
|
||||
|
||||
# Test
|
||||
mock==1.3.0
|
||||
|
||||
@ -16,7 +16,7 @@ class MockPlPy:
|
||||
def __init__(self):
|
||||
self._reset()
|
||||
|
||||
def _reset(self):
|
||||
def _reset(self, log_executed_queries=False):
|
||||
self.infos = []
|
||||
self.notices = []
|
||||
self.debugs = []
|
||||
@ -28,11 +28,30 @@ class MockPlPy:
|
||||
self.results = []
|
||||
self.prepares = []
|
||||
self.results = {}
|
||||
self._log_executed_queries = log_executed_queries
|
||||
self._logged_queries = []
|
||||
|
||||
def _define_result(self, query, result):
|
||||
pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
|
||||
self.results[pattern] = result
|
||||
|
||||
def _executed_queries(self):
|
||||
if self._log_executed_queries:
|
||||
return self._logged_queries
|
||||
else:
|
||||
raise Exception('Executed queries logging is not active')
|
||||
|
||||
def _has_executed_query(self, query):
|
||||
pattern = re.compile(re.escape(query))
|
||||
for executed_query in self._executed_queries():
|
||||
if pattern.search(executed_query):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _start_logging_executed_queries(self):
|
||||
self._logged_queries = []
|
||||
self._log_executed_queries = True
|
||||
|
||||
def notice(self, msg):
|
||||
self.notices.append(msg)
|
||||
|
||||
@ -47,7 +66,15 @@ class MockPlPy:
|
||||
return MockCursor(data)
|
||||
|
||||
def execute(self, query, rows=1):
|
||||
if self._log_executed_queries:
|
||||
self._logged_queries.append(query)
|
||||
for pattern, result in self.results.iteritems():
|
||||
if pattern.search(query):
|
||||
return result
|
||||
return []
|
||||
|
||||
def quote_nullable(self, value):
|
||||
if value is None:
|
||||
return 'NULL'
|
||||
else:
|
||||
return "'{0}'".format(value)
|
||||
|
||||
124
server/lib/python/cartodb_services/test/test_ratelimitsconfig.py
Normal file
124
server/lib/python/cartodb_services/test/test_ratelimitsconfig.py
Normal file
@ -0,0 +1,124 @@
|
||||
from test_helper import *
|
||||
from unittest import TestCase
|
||||
from mock import Mock, MagicMock, patch
|
||||
from nose.tools import assert_raises, assert_not_equal, assert_equal
|
||||
|
||||
from datetime import datetime, date
|
||||
from mockredis import MockRedis
|
||||
import cartodb_services
|
||||
|
||||
from cartodb_services.tools import ServiceManager, LegacyServiceManager
|
||||
|
||||
from cartodb_services.metrics import GeocoderConfig
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
|
||||
from cartodb_services.tools import RateLimitExceeded
|
||||
|
||||
from cartodb_services.refactor.storage.redis_config import *
|
||||
from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
|
||||
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
|
||||
from cartodb_services.config import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
|
||||
|
||||
class TestRateLimitsConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
plpy_mock_config()
|
||||
cartodb_services.init(plpy_mock, _GD={})
|
||||
self.username = 'test_user'
|
||||
self.orgname = 'test_org'
|
||||
self.redis_conn = MockRedis()
|
||||
build_redis_user_config(self.redis_conn, self.username, 'geocoding')
|
||||
build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
|
||||
self.environment = 'production'
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
|
||||
basic_server_conf = {"server_conf": {"environment": "testing"},
|
||||
"mapzen_conf":
|
||||
{"geocoder":
|
||||
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}}
|
||||
}, "logger_conf": {}}
|
||||
self.empty_server_config = InMemoryConfigStorage(basic_server_conf)
|
||||
self.empty_redis_config = InMemoryConfigStorage({})
|
||||
self.user_config = RedisUserConfigStorageBuilder(self.redis_conn, self.username).get()
|
||||
self.org_config = RedisOrgConfigStorageBuilder(self.redis_conn, self.orgname).get()
|
||||
self.server_config = ServerConfigBackendFactory().get()
|
||||
|
||||
def test_server_config(self):
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
|
||||
# Write server level configuration
|
||||
config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
|
||||
config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
|
||||
plpy_mock._start_logging_executed_queries()
|
||||
config_setter.set_server_rate_limits(config)
|
||||
assert plpy_mock._has_executed_query('cdb_conf_setconf(\'rate_limits\', \'{"geocoder": {"limit": 1234, "period": 86400}}\')')
|
||||
|
||||
# Re-read configuration
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
|
||||
read_config = RateLimitsConfigBuilder(
|
||||
server_conf=self.server_config,
|
||||
user_conf=self.empty_redis_config,
|
||||
org_conf=self.empty_redis_config,
|
||||
service='geocoder',
|
||||
username=self.username,
|
||||
orgname=self.orgname
|
||||
).get()
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
assert_equal(read_config, config)
|
||||
|
||||
def test_server_org_config(self):
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
|
||||
server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
|
||||
org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400)
|
||||
config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
|
||||
|
||||
# Write server level configuration
|
||||
config_setter.set_server_rate_limits(server_config)
|
||||
# Override with org level configuration
|
||||
config_setter.set_org_rate_limits(org_config)
|
||||
|
||||
# Re-read configuration
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
|
||||
read_config = RateLimitsConfigBuilder(
|
||||
server_conf=self.server_config,
|
||||
user_conf=self.empty_redis_config,
|
||||
org_conf=self.org_config,
|
||||
service='geocoder',
|
||||
username=self.username,
|
||||
orgname=self.orgname
|
||||
).get()
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
assert_equal(read_config, org_config)
|
||||
|
||||
def test_server_org_user_config(self):
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
|
||||
server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
|
||||
org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400)
|
||||
user_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1236, period=86400)
|
||||
config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
|
||||
|
||||
# Write server level configuration
|
||||
config_setter.set_server_rate_limits(server_config)
|
||||
# Override with org level configuration
|
||||
config_setter.set_org_rate_limits(org_config)
|
||||
# Override with user level configuration
|
||||
config_setter.set_user_rate_limits(user_config)
|
||||
|
||||
# Re-read configuration
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
|
||||
read_config = RateLimitsConfigBuilder(
|
||||
server_conf=self.server_config,
|
||||
user_conf=self.user_config,
|
||||
org_conf=self.org_config,
|
||||
service='geocoder',
|
||||
username=self.username,
|
||||
orgname=self.orgname
|
||||
).get()
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
assert_equal(read_config, user_config)
|
||||
218
server/lib/python/cartodb_services/test/test_servicemanager.py
Normal file
218
server/lib/python/cartodb_services/test/test_servicemanager.py
Normal file
@ -0,0 +1,218 @@
|
||||
from test_helper import *
|
||||
from unittest import TestCase
|
||||
from mock import Mock, MagicMock, patch
|
||||
from nose.tools import assert_raises, assert_not_equal, assert_equal
|
||||
|
||||
from datetime import datetime, date
|
||||
from cartodb_services.tools import ServiceManager, LegacyServiceManager
|
||||
from mockredis import MockRedis
|
||||
import cartodb_services
|
||||
from cartodb_services.metrics import GeocoderConfig
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
|
||||
from cartodb_services.tools import RateLimitExceeded
|
||||
|
||||
LUA_AVAILABLE_FOR_MOCKREDIS = False
|
||||
|
||||
class MockRedisWithVersionInfo(MockRedis):
|
||||
def info(self):
|
||||
return {'redis_version': '3.0.2'}
|
||||
|
||||
class TestServiceManager(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
plpy_mock_config()
|
||||
cartodb_services.init(plpy_mock, _GD={})
|
||||
self.username = 'test_user'
|
||||
self.orgname = 'test_org'
|
||||
self.redis_conn = MockRedisWithVersionInfo()
|
||||
build_redis_user_config(self.redis_conn, self.username, 'geocoding')
|
||||
build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
|
||||
self.environment = 'production'
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
|
||||
|
||||
def check_rate_limit(self, service_manager, n, active=True):
|
||||
if LUA_AVAILABLE_FOR_MOCKREDIS:
|
||||
for _ in xrange(n):
|
||||
service_manager.assert_within_limits()
|
||||
if active:
|
||||
with assert_raises(RateLimitExceeded):
|
||||
service_manager.assert_within_limits()
|
||||
else:
|
||||
service_manager.assert_within_limits()
|
||||
else:
|
||||
# rratelimit doesn't work with MockRedis because it needs Lua support
|
||||
# so, we'll simply perform some sanity check on the configuration of the rate limiter
|
||||
if active:
|
||||
assert_equal(service_manager.rate_limiter._config.is_limited(), True)
|
||||
assert_equal(service_manager.rate_limiter._config.limit, n)
|
||||
else:
|
||||
assert not service_manager.rate_limiter._config.is_limited()
|
||||
|
||||
def test_legacy_service_manager(self):
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
service_manager.assert_within_limits()
|
||||
assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
|
||||
|
||||
def test_service_manager(self):
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
service_manager.assert_within_limits()
|
||||
assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
|
||||
|
||||
def test_no_rate_limit_by_default(self):
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
self.check_rate_limit(service_manager, 3, False)
|
||||
|
||||
def test_no_legacy_rate_limit_by_default(self):
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
self.check_rate_limit(service_manager, 3, False)
|
||||
|
||||
def test_legacy_server_rate_limit(self):
|
||||
rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
|
||||
def test_server_rate_limit(self):
|
||||
rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
|
||||
def test_user_rate_limit(self):
|
||||
user_redis_name = "rails:users:{0}".format(self.username)
|
||||
rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
||||
|
||||
def test_legacy_user_rate_limit(self):
|
||||
user_redis_name = "rails:users:{0}".format(self.username)
|
||||
rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
||||
|
||||
def test_org_rate_limit(self):
|
||||
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
||||
rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
||||
|
||||
def test_legacy_org_rate_limit(self):
|
||||
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
||||
rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
||||
|
||||
def test_user_rate_limit_precedence_over_org(self):
|
||||
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
||||
org_rate_limits = '{"limit":"1000","period":3600}'
|
||||
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
||||
user_redis_name = "rails:users:{0}".format(self.username)
|
||||
user_rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
||||
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
||||
|
||||
def test_org_rate_limit_precedence_over_server(self):
|
||||
server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
|
||||
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
||||
org_rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
||||
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
||||
get_fn.return_value = self.redis_conn
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
|
||||
def test_legacy_user_rate_limit_precedence_over_org(self):
|
||||
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
||||
org_rate_limits = '{"limit":"1000","period":3600}'
|
||||
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
||||
user_redis_name = "rails:users:{0}".format(self.username)
|
||||
user_rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
||||
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
||||
|
||||
def test_legacy_org_rate_limit_precedence_over_server(self):
|
||||
server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
|
||||
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
||||
org_rate_limits = '{"limit":"3","period":3600}'
|
||||
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
||||
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
||||
config_cache = {
|
||||
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
||||
'user_geocoder_config_test_user' : config,
|
||||
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
||||
}
|
||||
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
||||
self.check_rate_limit(service_manager, 3)
|
||||
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
||||
Loading…
Reference in New Issue
Block a user