From 29caaf929783f6ce2374fc6ee0a80682a04183cc Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 12 Dec 2016 17:42:46 +0100 Subject: [PATCH 01/13] Update README.md Remove misleading paragraph about `requirements.txt` and `setup.py` dependencies. Refer to https://packaging.python.org/requirements/ for an authoritative discussion. --- server/lib/python/cartodb_services/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/lib/python/cartodb_services/README.md b/server/lib/python/cartodb_services/README.md index 197ccd4..aa3be41 100644 --- a/server/lib/python/cartodb_services/README.md +++ b/server/lib/python/cartodb_services/README.md @@ -45,6 +45,3 @@ See the [[../../../../test/README.md]]. Basically, move to the `/test` directory cd $(git rev-parse --show-toplevel)/test python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY ``` - -## TODO -- Move dependencies expressed in `requirements.txt` to `setup.py` From a00fca6d137f2fdb872485f9c62923760c0be416 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 12 Dec 2016 17:54:02 +0100 Subject: [PATCH 02/13] Update README.md Add a bit about versioning stuff --- server/lib/python/cartodb_services/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/lib/python/cartodb_services/README.md b/server/lib/python/cartodb_services/README.md index aa3be41..f873544 100644 --- a/server/lib/python/cartodb_services/README.md +++ b/server/lib/python/cartodb_services/README.md @@ -45,3 +45,8 @@ See the [[../../../../test/README.md]]. Basically, move to the `/test` directory cd $(git rev-parse --show-toplevel)/test python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY ``` + +## Versioning +Once you're satisfied with your changes, it is time to bump the version number in the `setup.py`. A couple of rules: +- **Backwards compatibility**: in general all changes shall be backwards compatible. Do not remove any code used from the server public `pl/python` functions or you'll run into problems when deploying. +- **Semantic versioning**: we try to stick to [Semantic Versioning 2.0.0](http://semver.org/spec/v2.0.0.html) From 6c71d73498cc8c2b41414bfd256f95475c015102 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 16 Dec 2016 15:24:34 +0100 Subject: [PATCH 03/13] Use the current quota for mapzen services --- .../cartodb_services/metrics/config.py | 42 +++++++++---------- .../service/mapzen_geocoder_config.py | 15 ++++--- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 7cdac6a..25cedd4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -43,12 +43,20 @@ class ServiceConfig(object): def metrics_log_path(self): return self._metrics_log_path + def _get_effective_monthly_quota(self, quota_key, default=0): + quota_from_redis = self._redis_config.get(quota_key, None) + if quota_from_redis and quota_from_redis <> '': + return int(quota_from_redis) + else: + return default + def __get_metrics_log_path(self): if self.METRICS_LOG_KEY: return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None) else: return None + class DataObservatoryConfig(ServiceConfig): METRICS_LOG_KEY = 'do_log_path' @@ -92,9 +100,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig): self._soft_limit = True else: self._soft_limit = False - self._monthly_quota = 0 - if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = int(self._redis_config[self.QUOTA_KEY]) + self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) self._connection_str = self._db_config.data_observatory_connection_str @property @@ -116,9 +122,7 @@ class ObservatoryConfig(DataObservatoryConfig): self._soft_limit = True else: self._soft_limit = False - self._monthly_quota = 0 - if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = int(self._redis_config[self.QUOTA_KEY]) + self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) self._connection_str = self._db_config.data_observatory_connection_str @property @@ -173,14 +177,7 @@ class RoutingConfig(ServiceConfig): return self._soft_limit def _set_monthly_quota(self): - self._monthly_quota = self._get_effective_monthly_quota() - - def _get_effective_monthly_quota(self): - quota_from_redis = self._redis_config.get(self.QUOTA_KEY) - if quota_from_redis and quota_from_redis <> '': - return int(quota_from_redis) - else: - return self._db_config.mapzen_routing_monthly_quota + self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) def _set_soft_limit(self): if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': @@ -217,18 +214,16 @@ class IsolinesRoutingConfig(ServiceConfig): self._isolines_provider = self.DEFAULT_PROVIDER self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower() self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + self._isolines_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) + if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_isolines_limit = True + else: + self._soft_isolines_limit = False if self._isolines_provider == self.HEREMAPS_PROVIDER: - self._isolines_quota = int(filtered_config[self.QUOTA_KEY]) self._heremaps_app_id = db_config.heremaps_isolines_app_id self._heremaps_app_code = db_config.heremaps_isolines_app_code - if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': - self._soft_isolines_limit = True - else: - self._soft_isolines_limit = False elif self._isolines_provider == self.MAPZEN_PROVIDER: self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key - self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota - self._soft_isolines_limit = False @property def service_type(self): @@ -361,8 +356,10 @@ class GeocoderConfig(ServiceConfig): self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower() else: self._geocoder_provider = self.DEFAULT_PROVIDER - self._geocoding_quota = int(filtered_config[self.QUOTA_KEY]) + + self._geocoding_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True else: @@ -377,7 +374,6 @@ class GeocoderConfig(ServiceConfig): self._cost_per_hit = 0 elif self._geocoder_provider == self.MAPZEN_GEOCODER: self._mapzen_api_key = db_config.mapzen_geocoder_api_key - self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota self._cost_per_hit = 0 @property diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index b37b483..ffe3906 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -67,6 +67,7 @@ class MapzenGeocoderConfig(object): @property def username(self): return self._username + @property def organization(self): return self._organization @@ -86,16 +87,13 @@ class MapzenGeocoderConfigBuilder(object): self._username = username self._orgname = orgname - def get(self): mapzen_server_conf = self._server_conf.get('mapzen_conf') - geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota'] mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] + geocoding_quota = int(self._get_quota(mapzen_server_conf)) soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' - - cost_per_hit=0 - + cost_per_hit = 0 period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') period_end_date = date_parse(period_end_date_str) @@ -110,3 +108,10 @@ class MapzenGeocoderConfigBuilder(object): mapzen_api_key, self._username, self._orgname) + + def _get_quota(self, mapzen_server_conf): + geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') + if not geocoding_quota: + geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota'] + + return geocoding_quota From 0672f2752b48a86b290f94f7e52193900bd779ff Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 19 Dec 2016 17:52:38 +0100 Subject: [PATCH 04/13] Improve tests and add more unit tests for the quota functionality --- .../test/metrics/test_config.py | 413 +++++++++++++++++- .../cartodb_services/test/test_config.py | 60 --- .../cartodb_services/test/test_helper.py | 77 ++-- .../test/test_quota_service.py | 64 ++- .../test/test_user_service.py | 21 +- 5 files changed, 493 insertions(+), 142 deletions(-) delete mode 100644 server/lib/python/cartodb_services/test/test_config.py diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 8d0ceb5..b2a184d 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -1,7 +1,282 @@ from unittest import TestCase from mockredis import MockRedis +from datetime import datetime, timedelta from ..test_helper import * -from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig +from cartodb_services.metrics.config import * + + +class TestGeocoderUserConfig(TestCase): + + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google'] + + def setUp(self): + self.redis_conn = MockRedis() + plpy_mock_config() + + def test_should_return_geocoder_config_for_user(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + provider=geocoder_provider, quota=100) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', None) + if geocoder_provider == 'heremaps': + assert geocoder_config.heremaps_geocoder is True + assert geocoder_config.geocoding_quota == 100 + elif geocoder_provider == 'mapzen': + assert geocoder_config.mapzen_geocoder is True + assert geocoder_config.geocoding_quota == 100 + elif geocoder_provider == 'google': + assert geocoder_config.google_geocoder is True + assert geocoder_config.geocoding_quota is None + assert geocoder_config.soft_geocoding_limit is False + + def test_should_return_quota_0_when_is_0_in_redis(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + quota=0, provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', None) + if geocoder_provider is not 'google': + assert geocoder_config.geocoding_quota == 0 + + def test_should_return_quota_0_if_quota_is_empty(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + quota='', provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', None) + if geocoder_provider is not 'google': + assert geocoder_config.geocoding_quota == 0 + + def test_should_return_quota_None_when_is_provider_is_google(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + quota=0, provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', None) + if geocoder_provider is 'google': + assert geocoder_config.geocoding_quota == None + + def test_should_return_true_if_soft_limit_is_true(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + quota=0, soft_limit=True, + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', None) + assert geocoder_config.soft_geocoding_limit == True + + def test_should_return_false_if_soft_limit_is_empty_string(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + quota=0, soft_limit='', + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', None) + assert geocoder_config.soft_geocoding_limit == False + +class TestGeocoderOrgConfig(TestCase): + + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google'] + + def setUp(self): + self.redis_conn = MockRedis() + plpy_mock_config() + + def test_should_return_org_config(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + yesterday = datetime.today() - timedelta(days=1) + + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + provider=geocoder_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'geocoding', + quota=200, end_date=yesterday, + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + if geocoder_provider == 'heremaps': + assert geocoder_config.heremaps_geocoder is True + assert geocoder_config.geocoding_quota == 200 + elif geocoder_provider == 'mapzen': + assert geocoder_config.mapzen_geocoder is True + assert geocoder_config.geocoding_quota == 200 + elif geocoder_provider == 'google': + assert geocoder_config.google_geocoder is True + assert geocoder_config.geocoding_quota is None + assert geocoder_config.soft_geocoding_limit is False + assert geocoder_config.period_end_date.date() == yesterday.date() + + def test_should_return_0_quota_if_has_0_in_redis_config(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + provider=geocoder_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'geocoding', + quota=0, end_date=yesterday, + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + if geocoder_provider is not 'google': + assert geocoder_config.geocoding_quota == 0 + + def test_should_return_0_if_quota_is_empty_for_org_in_redis(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + provider=geocoder_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'geocoding', + quota='', end_date=yesterday, + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + if geocoder_provider is not 'google': + assert geocoder_config.geocoding_quota == 0 + + def test_should_return_None_if_provider_is_google(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + provider=geocoder_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'geocoding', + quota='', end_date=yesterday, + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + if geocoder_provider is 'google': + assert geocoder_config.geocoding_quota == None + + def test_should_return_user_quota_if_is_not_defined_for_org(self): + for geocoder_provider in self.GEOCODER_PROVIDERS: + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'geocoding', + quota=100, provider=geocoder_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'geocoding', + quota=None, end_date=yesterday, + provider=geocoder_provider) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + if geocoder_provider is not 'google': + assert geocoder_config.geocoding_quota == 100 + + +class TestIsolinesUserConfig(TestCase): + + ISOLINES_PROVIDERS = ['heremaps', 'mapzen'] + + def setUp(self): + self.redis_conn = MockRedis() + plpy_mock_config() + + def test_should_return_user_config_for_isolines(self): + for isolines_provider in self.ISOLINES_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + quota=100, provider=isolines_provider) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user') + if isolines_provider is 'mapzen': + assert isolines_config.service_type is 'mapzen_isolines' + else: + assert isolines_config.service_type is 'here_isolines' + assert isolines_config.isolines_quota == 100 + assert isolines_config.soft_isolines_limit is False + + def test_should_return_0_quota_for_0_value(self): + for isolines_provider in self.ISOLINES_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider, quota=0, + soft_limit=True) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user') + assert isolines_config.isolines_quota == 0 + + def test_should_return_0_quota_for_empty_quota_value(self): + for isolines_provider in self.ISOLINES_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider, quota='') + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user') + assert isolines_config.isolines_quota == 0 + + def test_should_return_true_soft_limit(self): + for isolines_provider in self.ISOLINES_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider, quota=0, + soft_limit=True) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user') + assert isolines_config.soft_isolines_limit is True + + def test_should_return_false_soft_limit_with_empty_string(self): + for isolines_provider in self.ISOLINES_PROVIDERS: + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider, quota=0, + soft_limit='') + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user') + assert isolines_config.soft_isolines_limit is False + +class TestIsolinesOrgConfig(TestCase): + + ISOLINES_PROVIDERS = ['heremaps', 'mapzen'] + + def setUp(self): + self.redis_conn = MockRedis() + plpy_mock_config() + + def test_should_return_org_config_for_isolines(self): + yesterday = datetime.today() - timedelta(days=1) + for isolines_provider in self.ISOLINES_PROVIDERS: + + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'isolines', + quota=200, end_date=yesterday, + provider=isolines_provider) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert isolines_config.isolines_quota == 200 + assert isolines_config.soft_isolines_limit is False + assert isolines_config.period_end_date.date() == yesterday.date() + + def test_should_return_quota_0_for_0_redis_quota(self): + yesterday = datetime.today() - timedelta(days=1) + for isolines_provider in self.ISOLINES_PROVIDERS: + + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider, + soft_limit=True) + build_redis_org_config(self.redis_conn, 'test_org', 'isolines', + quota=0, end_date=yesterday, + provider=isolines_provider) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert isolines_config.isolines_quota == 0 + + def test_should_return_quota_0_for_empty_string_quota_in_org_config(self): + yesterday = datetime.today() - timedelta(days=1) + for isolines_provider in self.ISOLINES_PROVIDERS: + + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider) + build_redis_org_config(self.redis_conn, 'test_org', 'isolines', + quota='', end_date=yesterday, + provider=isolines_provider) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert isolines_config.isolines_quota == 0 + + def test_should_return_user_quota_for_non_existent_org_quota(self): + yesterday = datetime.today() - timedelta(days=1) + for isolines_provider in self.ISOLINES_PROVIDERS: + + build_redis_user_config(self.redis_conn, 'test_user', 'isolines', + provider=isolines_provider, quota=100) + build_redis_org_config(self.redis_conn, 'test_org', 'isolines', + quota=None, end_date=yesterday, + provider=isolines_provider) + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert isolines_config.isolines_quota == 100 class TestRoutingConfig(TestCase): @@ -13,11 +288,6 @@ class TestRoutingConfig(TestCase): self._user_key = "rails:users:{0}".format(self._username) self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') - def test_should_pick_quota_from_server_by_default(self): - orgname = None - config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) - assert config.monthly_quota == 1500000 - def test_should_pick_quota_from_redis_if_present(self): self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) orgname = None @@ -38,7 +308,6 @@ class TestRoutingConfig(TestCase): config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) assert config.monthly_quota == 5000 - def test_should_have_soft_limit_false_by_default(self): orgname = None config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) @@ -51,6 +320,136 @@ class TestRoutingConfig(TestCase): assert config.soft_limit == True +class TestDataObservatoryUserConfig(TestCase): + + def setUp(self): + self.redis_conn = MockRedis() + plpy_mock_config() + + def test_should_return_config_for_obs_snapshot(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=100, end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.monthly_quota == 100 + assert do_config.soft_limit is False + assert do_config.period_end_date.date() == yesterday.date() + + def test_should_return_true_if_soft_limit_is_true_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=0, soft_limit=True, end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.soft_limit is True + + def test_should_return_0_if_quota_is_0_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=0, end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.monthly_quota == 0 + + def test_should_return_0_if_quota_is_empty_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota='', end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.monthly_quota == 0 + + def test_should_return_config_for_obs_snapshot(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=100, end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.monthly_quota == 100 + assert do_config.soft_limit is False + assert do_config.period_end_date.date() == yesterday.date() + + def test_should_return_0_if_quota_is_0_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=0, end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.monthly_quota == 0 + + def test_should_return_0_if_quota_is_empty_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota='', end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.monthly_quota == 0 + + def test_should_return_true_if_soft_limit_is_true_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=0, soft_limit=True, end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.soft_limit is True + + def test_should_return_true_if_soft_limit_is_empty_string_in_redis(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=0, soft_limit='', end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user') + assert do_config.soft_limit is False + +class TestDataObservatoryOrgConfig(TestCase): + + def setUp(self): + self.redis_conn = MockRedis() + plpy_mock_config() + + def test_should_return_organization_config(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=100, end_date=yesterday) + build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory', + quota=200, end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert do_config.monthly_quota == 200 + assert do_config.period_end_date.date() == yesterday.date() + + def test_should_return_quota_0_for_0_in_org_quota_config(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=100) + build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory', + quota=0, end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert do_config.monthly_quota == 0 + + def test_should_return_quota_0_for_empty_in_org_quota_config(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=100) + build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory', + quota='', end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert do_config.monthly_quota == 0 + + def test_should_return_user_config_when_org_quota_is_not_defined(self): + yesterday = datetime.today() - timedelta(days=1) + build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', + quota=100) + build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory', + quota=None, end_date=yesterday) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + 'test_user', 'test_org') + assert do_config.monthly_quota == 100 + + class TestServicesRedisConfig(TestCase): def test_it_picks_mapzen_routing_quota_from_redis(self): redis_conn = MockRedis() diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py deleted file mode 100644 index b55b991..0000000 --- a/server/lib/python/cartodb_services/test/test_config.py +++ /dev/null @@ -1,60 +0,0 @@ -from test_helper import * -from unittest import TestCase -from nose.tools import assert_raises -from mockredis import MockRedis -from datetime import datetime, timedelta -from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException - - -class TestConfig(TestCase): - - def setUp(self): - self.redis_conn = MockRedis() - plpy_mock_config() - - def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): - build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, - 'test_user', None) - assert geocoder_config.heremaps_geocoder is True - assert geocoder_config.geocoding_quota == 100 - assert geocoder_config.soft_geocoding_limit is False - - def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user') - build_redis_org_config(self.redis_conn, 'test_org', - quota=200, end_date=yesterday) - geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, - 'test_user', 'test_org') - assert geocoder_config.heremaps_geocoder is True - assert geocoder_config.geocoding_quota == 200 - assert geocoder_config.soft_geocoding_limit is False - assert geocoder_config.period_end_date.date() == yesterday.date() - - def test_should_return_config_for_obs_snapshot(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user', - do_quota=100, soft_do_limit=True, - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - 'test_user') - assert do_config.monthly_quota == 100 - assert do_config.soft_limit is True - assert do_config.period_end_date.date() == yesterday.date() - - def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user', - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - 'test_user') - assert do_config.monthly_quota == 0 - assert do_config.soft_limit is False - assert do_config.period_end_date.date() == yesterday.date() - - def test_should_raise_exception_when_missing_parameters(self): - plpy_mock._reset() - build_redis_user_config(self.redis_conn, 'test_user') - assert_raises(ConfigException, GeocoderConfig, self.redis_conn, - plpy_mock, 'test_user', None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 2272278..87beef2 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,4 +1,5 @@ from datetime import datetime, date +from dateutil.tz import tzlocal from mock import Mock, MagicMock import random import sys @@ -8,46 +9,60 @@ plpy_mock = MockPlPy() sys.modules['plpy'] = plpy_mock -def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, - service="heremaps", isolines_quota=0, - do_quota=None, soft_do_limit=None, - do_general_quota=None, soft_do_general_limit=None, +def build_redis_user_config(redis_conn, username, service, quota=100, + soft_limit=False, provider="heremaps", end_date=datetime.today()): + end_date_tz = end_date.replace(tzinfo=tzlocal()) user_redis_name = "rails:users:{0}".format(username) - redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit) - redis_conn.hset(user_redis_name, 'geocoding_quota', quota) - redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota) - redis_conn.hset(user_redis_name, 'geocoder_provider', service) - redis_conn.hset(user_redis_name, 'isolines_provider', service) - redis_conn.hset(user_redis_name, 'routing_provider', service) - redis_conn.hset(user_redis_name, 'period_end_date', end_date) - if do_quota: - redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota) - if soft_do_limit: - redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', - soft_do_limit) - if do_general_quota: - redis_conn.hset(user_redis_name, 'obs_general_quota', do_general_quota) - if soft_do_general_limit: - redis_conn.hset(user_redis_name, 'soft_obs_general_limit', - soft_do_general_limit) + + if service is 'geocoding': + redis_conn.hset(user_redis_name, 'geocoder_provider', provider) + redis_conn.hset(user_redis_name, 'geocoding_quota', str(quota)) + redis_conn.hset(user_redis_name, 'soft_geocoding_limit', str(soft_limit).lower()) + elif service is 'isolines': + redis_conn.hset(user_redis_name, 'isolines_provider', provider) + redis_conn.hset(user_redis_name, 'here_isolines_quota', str(quota)) + redis_conn.hset(user_redis_name, 'soft_here_isolines_limit', str(soft_limit).lower()) + elif service is 'routing': + redis_conn.hset(user_redis_name, 'routing_provider', provider) + redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota)) + redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower()) + elif service is 'data_observatory': + redis_conn.hset(user_redis_name, 'obs_snapshot_quota', str(quota)) + redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota)) + redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', str(soft_limit).lower()) + redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower()) + redis_conn.hset(user_redis_name, 'google_maps_client_id', '') redis_conn.hset(user_redis_name, 'google_maps_api_key', '') + redis_conn.hset(user_redis_name, 'period_end_date', end_date_tz.strftime("%Y-%m-%d %H:%M:%S %z")) -def build_redis_org_config(redis_conn, orgname, quota=100, service="heremaps", - isolines_quota=0, do_quota=None, - do_general_quota=None, end_date=datetime.today()): +def build_redis_org_config(redis_conn, orgname, service, quota=100, + provider="heremaps", end_date=datetime.now(tzlocal())): org_redis_name = "rails:orgs:{0}".format(orgname) - redis_conn.hset(org_redis_name, 'geocoding_quota', quota) - redis_conn.hset(org_redis_name, 'here_isolines_quota', isolines_quota) - if do_quota: - redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota) - if do_general_quota: - redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota) - redis_conn.hset(org_redis_name, 'period_end_date', end_date) + end_date_tz = end_date.replace(tzinfo=tzlocal()) + + if service is 'geocoding': + redis_conn.hset(org_redis_name, 'geocoder_provider', provider) + if quota is not None: + redis_conn.hset(org_redis_name, 'geocoding_quota', str(quota)) + elif service is 'isolines': + redis_conn.hset(org_redis_name, 'isolines_provider', provider) + if quota is not None: + redis_conn.hset(org_redis_name, 'here_isolines_quota', str(quota)) + elif service is 'routing': + redis_conn.hset(org_redis_name, 'routing_provider', provider) + if quota is not None: + redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota)) + elif service is 'data_observatory': + if quota is not None: + redis_conn.hset(org_redis_name, 'obs_snapshot_quota', str(quota)) + redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota)) + redis_conn.hset(org_redis_name, 'google_maps_client_id', '') redis_conn.hset(org_redis_name, 'google_maps_api_key', '') + redis_conn.hset(org_redis_name, 'period_end_date', end_date_tz.strftime("%Y-%m-%d %H:%M:%S %z")) def increment_service_uses(redis_conn, username, orgname=None, diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index c104235..98ddddb 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -35,26 +35,26 @@ class TestQuotaService(TestCase): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') increment_service_uses(self.redis_conn, 'test_user', - orgname='test_org') + orgname='test_org') assert qs.check_user_quota() is True def test_should_return_false_if_user_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user') increment_service_uses(self.redis_conn, 'test_user', - amount=300) + amount=300) assert qs.check_user_quota() is False def test_should_return_false_if_org_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') increment_service_uses(self.redis_conn, 'test_user', - orgname='test_org', amount=400) + orgname='test_org', amount=400) assert qs.check_user_quota() is False def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) increment_service_uses(self.redis_conn, 'test_user', - amount=300) + amount=300) assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): @@ -62,7 +62,7 @@ class TestQuotaService(TestCase): orgname='test_org', soft_limit=True) increment_service_uses(self.redis_conn, 'test_user', - orgname='test_org', amount=400) + orgname='test_org', amount=400) assert qs.check_user_quota() is True def test_should_check_user_increment_and_quota_check_correctly(self): @@ -81,7 +81,7 @@ class TestQuotaService(TestCase): assert qs.check_user_quota() is False def test_should_check_user_mapzen_geocoder_quota_correctly(self): - qs = self.__build_geocoder_quota_service('test_user', service='mapzen') + qs = self.__build_geocoder_quota_service('test_user', provider='mapzen') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=1500000) @@ -89,7 +89,7 @@ class TestQuotaService(TestCase): def test_should_check_org_mapzen_geocoder_quota_correctly(self): qs = self.__build_geocoder_quota_service('test_user', orgname='testorg', - service='mapzen') + provider='mapzen') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=1500000) @@ -138,55 +138,51 @@ class TestQuotaService(TestCase): qs.increment_success_service_use(amount=100000) assert qs.check_user_quota() is False - def __prepare_quota_service(self, username, quota, service, orgname, - soft_limit, do_quota, soft_do_limit, end_date): - build_redis_user_config(self.redis_conn, username, - quota=quota, service=service, - soft_limit=soft_limit, - soft_do_limit=soft_do_limit, - do_quota=do_quota, - end_date=end_date) + def __prepare_quota_service(self, username, service, quota, provider, + orgname, soft_limit, end_date): + build_redis_user_config(self.redis_conn, username, service, + quota=quota, provider=provider, + soft_limit=soft_limit, end_date=end_date) if orgname: - build_redis_org_config(self.redis_conn, orgname, - quota=quota, service=service, - do_quota=do_quota, - end_date=end_date) + build_redis_org_config(self.redis_conn, orgname, service, + quota=quota, provider=provider, + end_date=end_date) def __build_geocoder_quota_service(self, username, quota=100, - service='heremaps', orgname=None, + provider='heremaps', orgname=None, soft_limit=False, end_date=datetime.today()): - self.__prepare_quota_service(username, quota, service, orgname, - soft_limit, 0, False, end_date) + self.__prepare_quota_service(username, 'geocoding', quota, + provider, orgname, soft_limit, end_date) geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) - def __build_routing_quota_service(self, username, service='mapzen', + def __build_routing_quota_service(self, username, provider='mapzen', orgname=None, soft_limit=False, quota=100, end_date=datetime.today()): - self.__prepare_quota_service(username, quota, service, orgname, - soft_limit, 0, False, end_date) + self.__prepare_quota_service(username, 'routing', quota, provider, + orgname, soft_limit, end_date) routing_config = RoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) - def __build_isolines_quota_service(self, username, service='mapzen', + def __build_isolines_quota_service(self, username, provider='mapzen', orgname=None, soft_limit=False, quota=100, end_date=datetime.today()): - self.__prepare_quota_service(username, quota, service, orgname, - soft_limit, 0, False, end_date) + self.__prepare_quota_service(username, 'isolines', quota, provider, + orgname, soft_limit, end_date) isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(isolines_config, redis_connection=self.redis_conn) def __build_obs_snapshot_quota_service(self, username, quota=100, - service='obs_snapshot', - orgname=None, - soft_limit=False, - end_date=datetime.today()): - self.__prepare_quota_service(username, 0, service, orgname, False, - quota, soft_limit, end_date) + provider='obs_snapshot', + orgname=None, + soft_limit=False, + end_date=datetime.today()): + self.__prepare_quota_service(username, 'data_observatory', quota, + None, orgname, soft_limit, end_date) do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(do_config, redis_connection=self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 2905f8b..c2bcec7 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -108,7 +108,7 @@ class TestUserService(TestCase): def zscore_counter(self): return self._zscore_counter self.redis_conn = MockRedisWithCounter() - us = self.__build_user_service('test_user', end_date=date.today()) + us = self.__build_user_service('test_user', end_date=datetime.today()) us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15)) #('user:test_user:geocoder_here:success_responses:201506', 15) @@ -132,16 +132,17 @@ class TestUserService(TestCase): assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506', '1') == None - def __build_user_service(self, username, quota=100, service='heremaps', - orgname=None, soft_limit=False, - end_date=date.today()): - build_redis_user_config(self.redis_conn, username, - quota=quota, service=service, - soft_limit=soft_limit, - end_date=end_date) + def __build_user_service(self, username, service='geocoding', quota=100, + provider='heremaps', orgname=None, + soft_limit=False, end_date=datetime.today()): + build_redis_user_config(self.redis_conn, username, service, + quota=quota, provider=provider, + soft_limit=soft_limit, + end_date=end_date) if orgname: - build_redis_org_config(self.redis_conn, orgname, - quota=quota, end_date=end_date) + build_redis_org_config(self.redis_conn, orgname, service, + provider=provider, quota=quota, + end_date=end_date) geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return UserMetricsService(geocoder_config, self.redis_conn) From 4d0abf9026e89580ef2fb4a0fff91aa0255df3ef Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 19 Dec 2016 17:53:07 +0100 Subject: [PATCH 05/13] Bump python library version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 38f6f6a..5e5be34 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.12.3', + version='0.13.0', description='CartoDB Services API Python Library', From 356135672bd0988584e205a9ad65b5345d3c6897 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 20 Dec 2016 16:12:48 +0100 Subject: [PATCH 06/13] Sanitize quota get in the refactor part --- .../refactor/service/mapzen_geocoder_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index ffe3906..4c5884a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -91,7 +91,7 @@ class MapzenGeocoderConfigBuilder(object): mapzen_server_conf = self._server_conf.get('mapzen_conf') mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] - geocoding_quota = int(self._get_quota(mapzen_server_conf)) + geocoding_quota = self._get_quota(mapzen_server_conf) soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' cost_per_hit = 0 period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') @@ -111,7 +111,7 @@ class MapzenGeocoderConfigBuilder(object): def _get_quota(self, mapzen_server_conf): geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') - if not geocoding_quota: - geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota'] + if geocoding_quota is '': + return 0 - return geocoding_quota + return int(geocoding_quota) From 38754fec26c094ac8efea7dcbf13b10331aba07b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 20 Dec 2016 16:13:10 +0100 Subject: [PATCH 07/13] Cover refactor mapzen geocoder config with tests --- .../service/test_mapzen_geocoder_config.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py diff --git a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py new file mode 100644 index 0000000..d67a2bf --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py @@ -0,0 +1,154 @@ +from unittest import TestCase +from mockredis import MockRedis +from datetime import datetime +from cartodb_services.refactor.service.mapzen_geocoder_config import * +from cartodb_services.refactor.storage.redis_config import * +from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage + + +class TestMapzenGeocoderUserConfig(TestCase): + + def setUp(self): + self._redis_connection = MockRedis() + self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"}, + "mapzen_conf": + {"geocoder": + {"api_key": "search-xxxxxxx", "monthly_quota": 1500000} + }, "logger_conf": {}}) + self._username = 'test_user' + self._user_key = "rails:users:{0}".format(self._username) + self._user_config = RedisUserConfigStorageBuilder(self._redis_connection, + self._username).get() + self._org_config = RedisOrgConfigStorageBuilder(self._redis_connection, + None).get() + self._set_default_config_values() + + def test_config_values_are_ok(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.geocoding_quota == 100 + assert config.soft_geocoding_limit == False + assert config.period_end_date == datetime.strptime('2016-12-31 00:00:00', "%Y-%m-%d %H:%M:%S") + assert config.service_type == 'geocoder_mapzen' + assert config.provider == 'mapzen' + assert config.is_high_resolution == True + assert config.cost_per_hit == 0 + assert config.mapzen_api_key == 'search-xxxxxxx' + assert config.username == 'test_user' + assert config.organization is None + + def test_quota_should_be_0_if_redis_value_is_0(self): + self._redis_connection.hset(self._user_key, 'geocoding_quota', '0') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.geocoding_quota == 0 + + def test_quota_should_be_0_if_redis_value_is_empty_string(self): + self._redis_connection.hset(self._user_key, 'geocoding_quota', '') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.geocoding_quota == 0 + + def test_soft_limit_should_be_true(self): + self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'true') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.soft_geocoding_limit == True + + def test_soft_limit_should_be_false_if_is_empty_string(self): + self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', '') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.soft_geocoding_limit == False + + def _set_default_config_values(self): + self._redis_connection.hset(self._user_key, 'geocoding_quota', '100') + self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') + self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00') + +class TestMapzenGeocoderOrgConfig(TestCase): + + def setUp(self): + self._redis_connection = MockRedis() + self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"}, + "mapzen_conf": + {"geocoder": + {"api_key": "search-xxxxxxx", "monthly_quota": 1500000} + }, "logger_conf": {}}) + self._username = 'test_user' + self._organization = 'test_org' + self._user_key = "rails:users:{0}".format(self._username) + self._org_key = "rails:orgs:{0}".format(self._organization) + self._user_config = RedisUserConfigStorageBuilder(self._redis_connection, + self._username).get() + self._org_config = RedisOrgConfigStorageBuilder(self._redis_connection, + self._organization).get() + self._set_default_config_values() + + + def test_config_org_values_are_ok(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.geocoding_quota == 200 + assert config.soft_geocoding_limit == False + assert config.period_end_date == datetime.strptime('2016-12-31 00:00:00', "%Y-%m-%d %H:%M:%S") + assert config.service_type == 'geocoder_mapzen' + assert config.provider == 'mapzen' + assert config.is_high_resolution == True + assert config.cost_per_hit == 0 + assert config.mapzen_api_key == 'search-xxxxxxx' + assert config.username == 'test_user' + assert config.organization is 'test_org' + + def test_quota_should_be_0_if_redis_value_is_0(self): + self._redis_connection.hset(self._org_key, 'geocoding_quota', '0') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.geocoding_quota == 0 + + def test_quota_should_use_user_quota_value_if_redis_value_is_empty_string(self): + self._redis_connection.hset(self._org_key, 'geocoding_quota', '') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.geocoding_quota == 100 + + def test_quota_should_be_0_if_both_user_and_org_have_empty_string(self): + self._redis_connection.hset(self._user_key, 'geocoding_quota', '') + self._redis_connection.hset(self._org_key, 'geocoding_quota', '') + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.geocoding_quota == 0 + + def _set_default_config_values(self): + self._redis_connection.hset(self._user_key, 'geocoding_quota', '100') + self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') + self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00') + self._redis_connection.hset(self._org_key, 'geocoding_quota', '200') + self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') \ No newline at end of file From 9ee1d045c810b09a20ea14d86fcfe26f22638dec Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 21 Dec 2016 09:54:08 +0100 Subject: [PATCH 08/13] Fix tests --- server/lib/python/cartodb_services/test/test_user_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 897a5df..201a1de 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -24,7 +24,7 @@ class TestUserService(TestCase): assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_user_quota_for_a_month_shorter_than_end_day(self): - us = self.__build_user_service('test_user', end_date=date(2016,1,31)) + us = self.__build_user_service('test_user', end_date=datetime(2016,1,31)) assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0 def test_org_used_quota_for_a_day(self): @@ -35,7 +35,7 @@ class TestUserService(TestCase): assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_org_quota_quota_for_a_month_shorter_than_end_day(self): - us = self.__build_user_service('test_user', orgname='test_org', end_date=date(2016,1,31)) + us = self.__build_user_service('test_user', orgname='test_org', end_date=datetime(2016,1,31)) assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0 def test_user_not_amount_in_used_quota_for_month_should_be_0(self): From 23a2de032165ec3d212e5dfc75a513e59f8461ce Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 22 Dec 2016 17:01:18 +0100 Subject: [PATCH 09/13] Add mapzen isolines to the quota checker --- .../cartodb_services/metrics/user.py | 6 +++ .../test/test_quota_service.py | 47 +++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 0411ee4..82aaf95 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -2,21 +2,25 @@ from datetime import date, timedelta from dateutil.relativedelta import relativedelta from calendar import monthrange + def last_day_of_month(year, month): """last valid day of a month""" return monthrange(year, month)[1] + def latest_valid_date(year, month, day): """latest date not later than the day specified""" valid_day = min(day, last_day_of_month(year, month)) return date(year, month, valid_day) + class UserMetricsService: """ Class to manage all the user info """ SERVICE_GEOCODER_NOKIA = 'geocoder_here' SERVICE_GEOCODER_CACHE = 'geocoder_cache' SERVICE_HERE_ISOLINES = 'here_isolines' + SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' SERVICE_OBSERVATORY = 'obs_general' DAY_OF_MONTH_ZERO_PADDED = '%d' @@ -30,6 +34,8 @@ class UserMetricsService: def used_quota(self, service_type, date): if service_type == self.SERVICE_HERE_ISOLINES: return self.__used_isolines_quota(service_type, date) + if service_type == self.SERVICE_MAPZEN_ISOLINES: + return self.__used_isolines_quota(service_type, date) elif service_type == self.SERVICE_MAPZEN_ROUTING: return self.__used_routing_quota(service_type, date) elif service_type == self.SERVICE_OBSERVATORY: diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 98ddddb..7c29559 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,7 +1,7 @@ from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import QuotaService -from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig +from cartodb_services.metrics import * from unittest import TestCase from nose.tools import assert_raises from datetime import datetime, date @@ -81,15 +81,17 @@ class TestQuotaService(TestCase): assert qs.check_user_quota() is False def test_should_check_user_mapzen_geocoder_quota_correctly(self): - qs = self.__build_geocoder_quota_service('test_user', provider='mapzen') + qs = self.__build_geocoder_quota_service('test_user', + provider='mapzen') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=1500000) assert qs.check_user_quota() is False def test_should_check_org_mapzen_geocoder_quota_correctly(self): - qs = self.__build_geocoder_quota_service('test_user', orgname='testorg', - provider='mapzen') + qs = self.__build_geocoder_quota_service('test_user', + orgname='testorg', + provider='mapzen') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=1500000) @@ -111,16 +113,17 @@ class TestQuotaService(TestCase): def test_should_check_user_isolines_quota_correctly(self): qs = self.__build_isolines_quota_service('test_user') - qs.increment_success_service_use() + qs.increment_isolines_service_use() assert qs.check_user_quota() is True - qs.increment_success_service_use(amount=1500000) + qs.increment_isolines_service_use(amount=1500000) assert qs.check_user_quota() is False def test_should_check_org_isolines_quota_correctly(self): - qs = self.__build_isolines_quota_service('test_user', orgname='testorg') - qs.increment_success_service_use() + qs = self.__build_isolines_quota_service('test_user', + orgname='testorg') + qs.increment_isolines_service_use() assert qs.check_user_quota() is True - qs.increment_success_service_use(amount=1500000) + qs.increment_isolines_service_use(amount=1500000) assert qs.check_user_quota() is False def test_should_check_user_obs_snapshot_quota_correctly(self): @@ -138,6 +141,21 @@ class TestQuotaService(TestCase): qs.increment_success_service_use(amount=100000) assert qs.check_user_quota() is False + def test_should_check_user_obs_quota_correctly(self): + qs = self.__build_obs_snapshot_quota_service('test_user') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=100000) + assert qs.check_user_quota() is False + + def test_should_check_org_obs_quota_correctly(self): + qs = self.__build_obs_quota_service('test_user', + orgname='testorg') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=100000) + assert qs.check_user_quota() is False + def __prepare_quota_service(self, username, service, quota, provider, orgname, soft_limit, end_date): build_redis_user_config(self.redis_conn, username, service, @@ -186,3 +204,14 @@ class TestQuotaService(TestCase): do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(do_config, redis_connection=self.redis_conn) + + def __build_obs_quota_service(self, username, quota=100, + provider='obs_general', + orgname=None, + soft_limit=False, + end_date=datetime.today()): + self.__prepare_quota_service(username, 'data_observatory', quota, + None, orgname, soft_limit, end_date) + do_config = ObservatoryConfig(self.redis_conn, plpy_mock, + username, orgname) + return QuotaService(do_config, redis_connection=self.redis_conn) From 80b23c62c3a4ef4dc8de17816904e913672f6e52 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 23 Dec 2016 11:40:30 +0100 Subject: [PATCH 10/13] QPS timeout was badly calculated timedelta microseconds is just the microseconds part of the timedelta object not the elapsed time in microseconds. I've change to use the total_seconds method to get all the elapsed time in seconds and transform to miliseconds. --- .../lib/python/cartodb_services/cartodb_services/mapzen/qps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py index 65569fb..ca96c57 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py @@ -55,7 +55,8 @@ class QPSService: def retry(self, first_request_time, retry_count): elapsed = datetime.now() - first_request_time - if elapsed.microseconds > (self._retry_timeout * 1000.0 * 1000.0): + elapsed_miliseconds = (elapsed.total_seconds() * 1000.0) + if elapsed_miliseconds > (self._retry_timeout * 1000.0): raise TimeoutException() # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per From bdf962758667bface7956dd84c55d30e7e458710 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 27 Dec 2016 17:11:39 +0100 Subject: [PATCH 11/13] Fix typo I needed to fix this; I felt some kitten was being killed somewhere because of this. And it made baby Jesus cry. --- .../python/cartodb_services/cartodb_services/mapzen/qps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py index ca96c57..9b3b163 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py @@ -55,8 +55,8 @@ class QPSService: def retry(self, first_request_time, retry_count): elapsed = datetime.now() - first_request_time - elapsed_miliseconds = (elapsed.total_seconds() * 1000.0) - if elapsed_miliseconds > (self._retry_timeout * 1000.0): + elapsed_milliseconds = (elapsed.total_seconds() * 1000.0) + if elapsed_milliseconds > (self._retry_timeout * 1000.0): raise TimeoutException() # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per From 9700ae966cbad8b76abf5c48dc21855880356659 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 28 Dec 2016 14:44:48 +0100 Subject: [PATCH 12/13] Remove uneeded math for simplicity's sake --- .../lib/python/cartodb_services/cartodb_services/mapzen/qps.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py index 9b3b163..59cec1e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py @@ -55,8 +55,7 @@ class QPSService: def retry(self, first_request_time, retry_count): elapsed = datetime.now() - first_request_time - elapsed_milliseconds = (elapsed.total_seconds() * 1000.0) - if elapsed_milliseconds > (self._retry_timeout * 1000.0): + if elapsed.total_seconds() > self._retry_timeout: raise TimeoutException() # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per From 9791a5badabbc3dbf292115bb2367f76eb2d3348 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Dec 2016 12:10:56 +0100 Subject: [PATCH 13/13] Add cartodb to the search path See https://github.com/CartoDB/dataservices-api/issues/324#issuecomment-269614566 --- client/cdb_dataservices_client--0.14.0--0.14.1.sql | 2 +- client/cdb_dataservices_client--0.14.1--0.14.0.sql | 2 +- client/cdb_dataservices_client--0.14.1.sql | 2 +- client/sql/00_header.sql | 2 +- client/upgrade_downgrade_template.erb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/cdb_dataservices_client--0.14.0--0.14.1.sql b/client/cdb_dataservices_client--0.14.0--0.14.1.sql index be48cbb..797e34c 100644 --- a/client/cdb_dataservices_client--0.14.0--0.14.1.sql +++ b/client/cdb_dataservices_client--0.14.0--0.14.1.sql @@ -3,7 +3,7 @@ \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.1'" to load this file. \quit -- Make sure we have a sane search path to create/update the extension -SET search_path = "$user",public,cdb_dataservices_client; +SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- This release introduces no changes other than the use of -- search path in the install and migration scripts diff --git a/client/cdb_dataservices_client--0.14.1--0.14.0.sql b/client/cdb_dataservices_client--0.14.1--0.14.0.sql index 2c12569..e89a860 100644 --- a/client/cdb_dataservices_client--0.14.1--0.14.0.sql +++ b/client/cdb_dataservices_client--0.14.1--0.14.0.sql @@ -3,7 +3,7 @@ \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.0'" to load this file. \quit -- Make sure we have a sane search path to create/update the extension -SET search_path = "$user",public,cdb_dataservices_client; +SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- This release introduces no changes other than the use of -- search path in the install and migration scripts diff --git a/client/cdb_dataservices_client--0.14.1.sql b/client/cdb_dataservices_client--0.14.1.sql index 9ee9b08..df980e3 100644 --- a/client/cdb_dataservices_client--0.14.1.sql +++ b/client/cdb_dataservices_client--0.14.1.sql @@ -3,7 +3,7 @@ \echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit -- Make sure we have a sane search path to create/update the extension -SET search_path = "$user",public,cdb_dataservices_client; +SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- -- Geocoder server connection config -- diff --git a/client/sql/00_header.sql b/client/sql/00_header.sql index b41a51c..ee241f9 100644 --- a/client/sql/00_header.sql +++ b/client/sql/00_header.sql @@ -3,4 +3,4 @@ \echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit -- Make sure we have a sane search path to create/update the extension -SET search_path = "$user",public,cdb_dataservices_client; +SET search_path = "$user",cartodb,public,cdb_dataservices_client; diff --git a/client/upgrade_downgrade_template.erb b/client/upgrade_downgrade_template.erb index 31be973..745f00c 100644 --- a/client/upgrade_downgrade_template.erb +++ b/client/upgrade_downgrade_template.erb @@ -3,6 +3,6 @@ \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit -- Make sure we have a sane search path to create/update the extension -SET search_path = "$user",public,cdb_dataservices_client; +SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade