Merge pull request #202 from CartoDB/201-read-quota-redis-both-formats

201 read quota redis both formats

server/lib/python/cartodb_services/README.md

@@ -0,0 +1,46 @@
+# CartoDB dataservices API python module
+
+This directory contains the python library used by the server side of CartoDB LDS (Location Data Services).
+
+It is used from pl/python functions contained in the `cdb_dataservices_server` extension. It goes hand in hand with the extension so please consider running the integration tests.
+
+On the other hand, it is pretty independent from the client, as long as the signatures of the public pl/python functions match.
+
+## Dependencies
+See the [[`requirements.txt`]] or better the Basically:
+- pip
+- redis and hiredis
+- dateutil
+- googlemaps
+- request
+
+## Installation
+Install the requirements:
+```shell
+sudo pip install -r requirements.txt
+```
+
+Install the library:
+```shell
+sudo pip install .
+```
+
+NOTE: a system installation is required at present because the library is meant to be used from postgres pl/python, which runs an embedded python interpreter.
+
+
+## Running the unit tests
+Just run `nosetests`
+```shell
+$ nosetests
+.................................................
+----------------------------------------------------------------------
+Ran 49 tests in 0.131s
+
+OK
+```
+
+## Running the integration tests
+TBD
+
+## TODO
+- Move dependencies expressed in `requirements.txt` to `setup.py`

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -85,6 +85,11 @@ class UserMetricsService:
                                                      service, metric, date)
             score = self._redis_connection.zscore(redis_prefix, date.day)
             aggregated_metric += score if score else 0
+            zero_padded_day = date.strftime('%d')
+            if str(date.day) != zero_padded_day:
+                score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
+                aggregated_metric += score if score else 0
+
         return aggregated_metric
 
     # Private functions

server/lib/python/cartodb_services/requirements.txt

@@ -1,6 +1,5 @@
 redis==2.10.5
 hiredis==0.1.5
-# Dependency with incsv in the import
 python-dateutil==2.2
 googlemaps==2.4.2
 # Dependency for googlemaps package
@@ -11,3 +10,4 @@ mock==1.3.0
 mockredispy==2.9.0.11
 nose==1.3.7
 requests-mock==0.7.0
+freezegun==0.3.7

server/lib/python/cartodb_services/setup.py

@@ -10,11 +10,11 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.6.2',
+    version='0.6.3',
 
     description='CartoDB Services API Python Library',
 
-    url='https://github.com/CartoDB/geocoder-api',
+    url='https://github.com/CartoDB/dataservices-api',
 
     author='Data Services Team - CartoDB',
     author_email='dataservices@cartodb.com',

server/lib/python/cartodb_services/test/test_user_service.py

@@ -7,6 +7,7 @@ from unittest import TestCase
 from mock import Mock
 from nose.tools import assert_raises
 from datetime import timedelta
+from freezegun import freeze_time
 
 
 class TestUserService(TestCase):
@@ -75,9 +76,50 @@ class TestUserService(TestCase):
         us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
 
+    @freeze_time("2015-06-01")
+    def test_should_account_for_zero_paddded_keys(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
+
+    @freeze_time("2015-06-01")
+    def test_should_account_for_wrongly_stored_non_padded_keys(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
+
+    @freeze_time("2015-06-01")
+    def test_should_sum_amounts_from_both_key_formats(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506',  '1', 400)
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700
+
+    @freeze_time("2015-06-15")
+    def test_should_not_request_redis_twice_when_unneeded(self):
+        class MockRedisWithCounter(MockRedis):
+            def __init__(self):
+                super(MockRedisWithCounter, self).__init__()
+                self._zscore_counter = 0
+            def zscore(self, *args):
+                print args
+                self._zscore_counter += 1
+                return super(MockRedisWithCounter, self).zscore(*args)
+            def zscore_counter(self):
+                return self._zscore_counter
+        self.redis_conn = MockRedisWithCounter()
+        us = self.__build_user_service('test_user', end_date=date.today())
+        us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15))
+
+        #('user:test_user:geocoder_here:success_responses:201506', 15)
+        #('user:test_user:geocoder_here:empty_responses:201506', 15)
+        #('user:test_user:geocoder_cache:success_responses:201506', 15)
+        assert self.redis_conn.zscore_counter() == 3
+
+
     def __build_user_service(self, username, quota=100, service='heremaps',
                              orgname=None, soft_limit=False,
-                             end_date=datetime.today()):
+                             end_date=date.today()):
         test_helper.build_redis_user_config(self.redis_conn, username,
                                             quota=quota, service=service,
                                             soft_limit=soft_limit,