cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added plpy_mock and check multiple instances for metricsdatagatherer

Browse Source
This commit is contained in:
Mario de Frutos 2016-11-04 12:14:48 +01:00
parent 6f0db20163
commit 88ed777785
10 changed files with 134 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/lib/python/cartodb_services/test/__init__.py
View File

@ -0,0 +1,5 @@
from test_helper import plpy_mock_config
def setup():
plpy_mock_config()

5
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -1,13 +1,14 @@
from unittest import TestCase
from mockredis import MockRedis
from ..test_helper import *
from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
from ..test_helper import build_plpy_mock
class TestRoutingConfig(TestCase):
def setUp(self):
self._redis_conn = MockRedis()
self._db_conn = build_plpy_mock()
self._db_conn = plpy_mock
self._username = 'my_test_user'
self._user_key = "rails:users:{0}".format(self._username)
self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')

2
server/lib/python/cartodb_services/test/metrics/test_quota.py
View File

@ -1,6 +1,6 @@
from unittest import TestCase
from mockredis import MockRedis
from ..test_helper import build_plpy_mock
from ..test_helper import *
from cartodb_services.metrics.quota import QuotaChecker
from cartodb_services.metrics import RoutingConfig
from datetime import datetime

53
server/lib/python/cartodb_services/test/mock_plpy.py Normal file
View File

@ -0,0 +1,53 @@
import re
class MockCursor:
def __init__(self, data):
self.cursor_pos = 0
self.data = data
def fetch(self, batch_size):
batch = self.data[self.cursor_pos: self.cursor_pos + batch_size]
self.cursor_pos += batch_size
return batch
class MockPlPy:
def __init__(self):
self._reset()
def _reset(self):
self.infos = []
self.notices = []
self.debugs = []
self.logs = []
self.warnings = []
self.errors = []
self.fatals = []
self.executes = []
self.results = []
self.prepares = []
self.results = {}
def _define_result(self, query, result):
pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
self.results[pattern] = result
def notice(self, msg):
self.notices.append(msg)
def debug(self, msg):
self.notices.append(msg)
def info(self, msg):
self.infos.append(msg)
def cursor(self, query):
data = self.execute(query)
return MockCursor(data)
def execute(self, query, rows=1):
for pattern, result in self.results.iteritems():
if pattern.search(query):
return result
return []

36
server/lib/python/cartodb_services/test/test_config.py
View File

@ -1,20 +1,20 @@
import test_helper
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
from test_helper import *
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
self.plpy_mock = test_helper.build_plpy_mock()
plpy_mock_config()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
'test_user', None)
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
@ -22,10 +22,10 @@ class TestConfig(TestCase):
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user')
build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
@ -34,10 +34,10 @@ class TestConfig(TestCase):
def test_should_return_config_for_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user',
do_quota=100, soft_do_limit=True,
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user',
do_quota=100, soft_do_limit=True,
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 100
assert do_config.soft_limit is True
@ -45,16 +45,16 @@ class TestConfig(TestCase):
def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user',
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user',
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 0
assert do_config.soft_limit is False
assert do_config.period_end_date.date() == yesterday.date()
def test_should_raise_exception_when_missing_parameters(self):
plpy_mock = test_helper.build_plpy_mock(empty=True)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
plpy_mock._reset()
build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
plpy_mock, 'test_user', None)

34
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -1,7 +1,11 @@
from datetime import datetime, date
from mock import Mock
from mock import Mock, MagicMock
import random
import sys
sys.modules['plpy'] = Mock()
from mock_plpy import MockPlPy
plpy_mock = MockPlPy()
sys.modules['plpy'] = plpy_mock
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None,
redis_conn.zincrby(redis_name, date.day, amount)
def build_plpy_mock(empty=False):
plpy_mock = Mock()
if not empty:
plpy_mock.execute.side_effect = _plpy_execute_side_effect
return plpy_mock
def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf":
return [{'conf': '{"environment": "testing"}'}]
def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}])

23
server/lib/python/cartodb_services/test/test_metrics_log.py Normal file
View File

@ -0,0 +1,23 @@
from test_helper import *
from cartodb_services.metrics import MetricsDataGatherer
from unittest import TestCase
from mock import Mock, MagicMock
from nose.tools import assert_raises
from datetime import datetime, date
class TestMetricsDataGatherer(TestCase):
def setUp(self):
plpy_mock_config()
def test_should_use_multiple_instances_for_multiples_requests(self):
plpy_mock._define_result("select txid_current", [{'txid': 100}])
MetricsDataGatherer.add('test', 1)
plpy_mock._define_result("select txid_current", [{'txid': 101}])
MetricsDataGatherer.add('test', 2)
plpy_mock._define_result("select txid_current", [{'txid': 100}])
assert MetricsDataGatherer.get_element('test') is 1
plpy_mock._define_result("select txid_current", [{'txid': 101}])
assert MetricsDataGatherer.get_element('test') is 2

1
server/lib/python/cartodb_services/test/test_qps.py
View File

@ -1,4 +1,3 @@
import test_helper
import requests
from unittest import TestCase
from nose.tools import assert_raises

27
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@ -1,4 +1,4 @@
import test_helper
from test_helper import *
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
@ -28,32 +28,32 @@ class TestQuotaService(TestCase):
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user')
increment_service_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
@ -61,7 +61,7 @@ class TestQuotaService(TestCase):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org',
soft_limit=True)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
@ -140,18 +140,17 @@ class TestQuotaService(TestCase):
def __prepare_quota_service(self, username, quota, service, orgname,
soft_limit, do_quota, soft_do_limit, end_date):
test_helper.build_redis_user_config(self.redis_conn, username,
build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
soft_do_limit=soft_do_limit,
do_quota=do_quota,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
build_redis_org_config(self.redis_conn, orgname,
quota=quota, service=service,
do_quota=do_quota,
end_date=end_date)
self._plpy_mock = test_helper.build_plpy_mock()
def __build_geocoder_quota_service(self, username, quota=100,
service='heremaps', orgname=None,
@ -159,7 +158,7 @@ class TestQuotaService(TestCase):
end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock,
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
@ -168,7 +167,7 @@ class TestQuotaService(TestCase):
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
routing_config = RoutingConfig(self.redis_conn, self._plpy_mock,
routing_config = RoutingConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn)
@ -177,7 +176,7 @@ class TestQuotaService(TestCase):
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock,
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(isolines_config, redis_connection=self.redis_conn)
@ -188,6 +187,6 @@ class TestQuotaService(TestCase):
end_date=datetime.today()):
self.__prepare_quota_service(username, 0, service, orgname, False,
quota, soft_limit, end_date)
do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock,
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(do_config, redis_connection=self.redis_conn)

11
server/lib/python/cartodb_services/test/test_user_service.py
View File

@ -1,4 +1,4 @@
import test_helper
from test_helper import *
from mockredis import MockRedis
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
@ -19,13 +19,13 @@ class TestUserService(TestCase):
def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
@ -135,14 +135,13 @@ class TestUserService(TestCase):
def __build_user_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date=date.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
plpy_mock = test_helper.build_plpy_mock()
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname)
return UserMetricsService(geocoder_config, self.redis_conn)