New method to get the geocoder used quota from Redis

Gemfile

@@ -97,6 +97,7 @@ group :test do
   gem 'activerecord-nulldb-adapter', '0.3.1'
   # Need to use specific branch from this fork as original gem is broken and outdated
   gem 'fake_net_ldap', git: 'https://github.com/kuldeepaggarwal/fake_net_ldap.git', :branch => 'fix-responder'
+  gem 'mock_redis'
 end
 
 group :development, :test do

Gemfile.lock

@@ -171,6 +171,7 @@ GEM
     mini_portile (0.6.2)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
+    mock_redis (0.15.4)
     mono_logger (1.1.0)
     multi_json (1.11.2)
     multi_xml (0.5.5)
@@ -363,6 +364,7 @@ DEPENDENCIES
   hiredis (= 0.6.0)
   instagram (= 1.1.6)
   mocha (= 1.1.0)
+  mock_redis
   net-ldap (= 0.11)
   nokogiri (~> 1.6.6.2)
   oauth (= 0.4.5)
@@ -404,4 +406,4 @@ DEPENDENCIES
   webrick (= 1.3.1)
 
 BUNDLED WITH
-   1.10.6
+   1.11.2

app/helpers/geocoder_metrics_helper.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+require_relative '../../services/table-geocoder/lib/table_geocoder_factory'
+
+module GeocoderMetricsHelper
+  def get_user_geocoding_data(user, from, to)
+    get_geocoding_data(user, from, to)
+  end
+
+  def get_organization_geocoding_data(organization, from, to)
+    return if organization.owner.nil?
+    get_geocoding_data(organization.owner, from, to)
+  end
+
+  private
+
+  def get_geocoding_data(user, from, to)
+    usage_metrics = Carto::TableGeocoderFactory.get_geocoder_metrics_instance(user)
+    geocoder_key = user.google_maps_geocoder_enabled? ? :geocoder_google : :geocoder_here
+    cache_hits = 0
+    countable_requests = 0
+    from.upto(to).each do |date|
+      success = usage_metrics.get(geocoder_key, :success_responses, date)
+      countable_requests += success unless success.nil?
+      empty = usage_metrics.get(geocoder_key, :empty_responses, date)
+      countable_requests += empty unless empty.nil?
+      hit = usage_metrics.get(:geocoder_cache, :success_responses, date)
+      cache_hits += hit unless hit.nil?
+    end
+    countable_requests + cache_hits
+  end
+end

app/models/carto/organization.rb

@@ -1,7 +1,9 @@
 require 'active_record'
+require_relative '../../helpers/geocoder_metrics_helper'
 
 module Carto
   class Organization < ActiveRecord::Base
+    include GeocoderMetricsHelper
 
     has_many :users, inverse_of: :organization, order: :username
     belongs_to :owner, class_name: Carto::User, inverse_of: :owned_organization
@@ -26,6 +28,12 @@ module Carto
         sum("processed_rows + cache_hits".lit).to_i
     end
 
+    def get_new_system_geocoding_calls(options = {})
+      date_to = (options[:to] ? options[:to].to_date : Date.current)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      get_organization_geocoding_data(self, date_from, date_to)
+    end
+
     def twitter_imports_count(options = {})
       date_to = (options[:to] ? options[:to].to_date : Date.today)
       date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)

app/models/carto/user.rb

@@ -4,10 +4,12 @@ require 'active_record'
 require_relative 'user_service'
 require_relative 'user_db_service'
 require_relative 'synchronization_oauth'
+require_relative '../../helpers/geocoder_metrics_helper'
 
 # TODO: This probably has to be moved as the service of the proper User Model
 class Carto::User < ActiveRecord::Base
   extend Forwardable
+  include GeocoderMetricsHelper
 
   MIN_PASSWORD_LENGTH = 6
   MAX_PASSWORD_LENGTH = 64
@@ -278,6 +280,12 @@ class Carto::User < ActiveRecord::Base
       .sum("processed_rows + cache_hits".lit).to_i
   end
 
+  def get_new_system_geocoding_calls(options = {})
+    date_to = (options[:to] ? options[:to].to_date : Date.today)
+    date_from = (options[:from] ? options[:from].to_date : last_billing_cycle)
+    get_user_geocoding_data(self, date_from, date_to)
+  end
+
   #TODO: Remove unused param `use_total`
   def remaining_quota(use_total = false, db_size = service.db_size_in_bytes)
     self.quota_in_bytes - db_size

app/models/organization.rb

@@ -2,6 +2,7 @@
 
 require_relative '../controllers/carto/api/group_presenter'
 require_relative './organization/organization_decorator'
+require_relative '../helpers/geocoder_metrics_helper'
 require_relative './permission'
 
 class Organization < Sequel::Model
@@ -9,6 +10,7 @@ class Organization < Sequel::Model
 
   include CartoDB::OrganizationDecorator
   include Concerns::CartodbCentralSynchronizable
+  include GeocoderMetricsHelper
 
   Organization.raise_on_save_failure = true
   self.strict_param_setting = false
@@ -147,6 +149,12 @@ class Organization < Sequel::Model
     Geocoding.get_geocoding_calls(users_dataset.join(:geocodings, :user_id => :id), date_from, date_to)
   end
 
+  def get_new_system_geocoding_calls(options = {})
+    date_to = (options[:to] ? options[:to].to_date : Date.current)
+    date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+    get_organization_geocoding_data(self, date_from, date_to)
+  end
+
   def get_twitter_imports_count(options = {})
     date_from, date_to = quota_dates(options)
 

app/models/user.rb

@@ -5,6 +5,7 @@ require_relative './user/oauths'
 require_relative './synchronization/synchronization_oauth'
 require_relative './visualization/member'
 require_relative '../helpers/redis_vizjson_cache'
+require_relative '../helpers/geocoder_metrics_helper'
 require_relative './visualization/collection'
 require_relative './user/user_organization'
 require_relative './synchronization/collection.rb'
@@ -22,6 +23,7 @@ class User < Sequel::Model
   include CartoDB::UserDecorator
   include Concerns::CartodbCentralSynchronizable
   include CartoDB::ConfigUtils
+  include GeocoderMetricsHelper
 
   self.strict_param_setting = false
 
@@ -787,6 +789,11 @@ class User < Sequel::Model
     Geocoding.get_geocoding_calls(geocodings_dataset, date_from, date_to)
   end
 
+  def get_new_system_geocoding_calls(options = {})
+    date_from, date_to = quota_dates(options)
+    get_user_geocoding_data(self, date_from, date_to)
+  end
+
   def get_not_aggregated_geocoding_calls(options = {})
     date_from, date_to = quota_dates(options)
     Geocoding.get_not_aggregated_user_geocoding_calls(geocodings_dataset.db, self.id, date_from, date_to)

services/table-geocoder/lib/geocoder_usage_metrics.rb

@@ -39,9 +39,19 @@ module CartoDB
       @redis.zincrby("#{user_key_prefix(service, metric, date)}", amount, "#{date_day(date)}")
     end
 
+    def get(service, metric, date = DateTime.current)
+      check_valid_data(service, metric)
+
+      if !@orgname.nil?
+        @redis.zscore("#{org_key_prefix(service, metric, date)}", "#{date_day(date)}")
+      else
+        @redis.zscore("#{user_key_prefix(service, metric, date)}", "#{date_day(date)}")
+      end
+    end
+
     private
 
-    def check_valid_data(service, metric, amount)
+    def check_valid_data(service, metric, amount = 0)
       raise 'Invalid service' unless VALID_SERVICES.include?(service)
       raise 'invalid metric' unless VALID_METRICS.include?(metric)
       raise 'invalid amount' if amount < 0

spec/models/user_shared_examples.rb

@@ -1,14 +1,15 @@
 # encoding: UTF-8
 
+require 'mock_redis'
+require 'active_support/time'
 require_relative '../spec_helper'
+require_relative '../../services/table-geocoder/lib/geocoder_usage_metrics'
 
 # Tests should define the following method:
 # - `get_twitter_imports_count_by_user_id`
 # - `get_user_by_id`
 shared_examples_for "user models" do
-
   describe '#get_twitter_imports_count' do
-
     include_context 'users helper'
 
     it "should count tweet imports" do
@@ -18,13 +19,13 @@ shared_examples_for "user models" do
 
       get_twitter_imports_count_by_user_id(@user1.id).should == 5
     end
-
   end
 
   describe 'twitter_datasource_enabled for org users' do
     include_context 'organization with users helper'
 
-    it 'is enabled if organization has it enabled, no matter whether user has it or not, and enabled if he has it enabled, no matter whether org has it or not' do
+    it 'is enabled if organization has it enabled, no matter whether user has it or not,
+        and enabled if he has it enabled, no matter whether org has it or not' do
       @organization.twitter_datasource_enabled = false
       @organization.save.reload
 
@@ -49,50 +50,120 @@ shared_examples_for "user models" do
       @org_user_1.save.reload
       get_user_by_id(@org_user_1.id).twitter_datasource_enabled.should == true
     end
-
   end
 
   describe 'User#remaining_geocoding_quota' do
-
     include_context 'users helper'
     include_context 'organization with users helper'
 
     it 'calculates the remaining quota for a non-org user correctly' do
       @user1.geocoding_quota = 500
       @user1.save
-      Geocoding.new({
-          kind: 'high-resolution',
-          user: @user1,
-          formatter: '{dummy}',
-          processed_rows: 100
-        }).save
+      Geocoding.new(kind: 'high-resolution',
+                    user: @user1,
+                    formatter: '{dummy}',
+                    processed_rows: 100).save
 
       get_user_by_id(@user1.id).remaining_geocoding_quota.should == 400
     end
 
-
     it 'takes into account geocodings performed by the org users #4033' do
       @organization.geocoding_quota = 500
       @organization.save.reload
 
-      Geocoding.new({
-          kind: 'high-resolution',
-          user: @org_user_1,
-          formatter: '{dummy}',
-          processed_rows: 100
-        }).save
+      Geocoding.new(kind: 'high-resolution',
+                    user: @org_user_1,
+                    formatter: '{dummy}',
+                    processed_rows: 100).save
 
-      Geocoding.new({
-          kind: 'high-resolution',
-          user: @org_user_2,
-          formatter: '{dummy}',
-          processed_rows: 100
-        }).save
+      Geocoding.new(kind: 'high-resolution',
+                    user: @org_user_2,
+                    formatter: '{dummy}',
+                    processed_rows: 100).save
 
       get_user_by_id(@org_user_1.id).remaining_geocoding_quota.should == 300
       get_user_by_id(@org_user_2.id).remaining_geocoding_quota.should == 300
     end
-
   end
 
+  describe 'User#used_geocoding_quota' do
+    include_context 'users helper'
+    include_context 'organization with users helper'
+
+    before(:each) do
+      @mock_redis = MockRedis.new
+      @user1.geocoding_quota = 500
+      @user1.period_end_date = (DateTime.current + 1) << 1
+      @user1.save.reload
+      @organization.geocoding_quota = 500
+      @organization.save.reload
+      @organization.owner.period_end_date = (DateTime.current + 1) << 1
+      @organization.owner.save.reload
+    end
+
+    it 'calculates the used geocoder quota in the current billing cycle' do
+      usage_metrics = CartoDB::GeocoderUsageMetrics.new(@mock_redis, @user1.username, nil)
+      Carto::TableGeocoderFactory.stubs(:get_geocoder_metrics_instance).returns(usage_metrics)
+      Geocoding.new(kind: 'high-resolution',
+                    user: @user1,
+                    formatter: '{dummy}',
+                    processed_rows: 0,
+                    cache_hits: 100,
+                    created_at: (DateTime.current - 1)).save
+      Geocoding.new(kind: 'high-resolution',
+                    user: @user1,
+                    formatter: '{dummy}',
+                    processed_rows: 100,
+                    cache_hits: 0,
+                    created_at: (DateTime.current - 2)).save
+      Geocoding.new(kind: 'high-resolution',
+                    user: @user1,
+                    formatter: '{dummy}',
+                    processed_rows: 10,
+                    cache_hits: 0,
+                    created_at: DateTime.current).save
+      usage_metrics.incr(:geocoder_here, :success_responses, 10, DateTime.current)
+      usage_metrics.incr(:geocoder_here, :success_responses, 100, (DateTime.current - 2))
+      usage_metrics.incr(:geocoder_cache, :success_responses, 100, (DateTime.current - 1))
+
+      get_user_by_id(@user1.id).get_new_system_geocoding_calls.should == 210
+      get_user_by_id(@user1.id).get_geocoding_calls.should == 210
+    end
+
+    it 'calculates the used geocoding quota for an organization' do
+      usage_metrics_1 = CartoDB::GeocoderUsageMetrics.new(@mock_redis, @org_user_1.username, @organization.name)
+      usage_metrics_2 = CartoDB::GeocoderUsageMetrics.new(@mock_redis, @org_user_2.username, @organization.name)
+      # We are going to get the organization data show we could use both usage_metrics objects
+      Carto::TableGeocoderFactory.stubs(:get_geocoder_metrics_instance).returns(usage_metrics_1)
+      Geocoding.new(kind: 'high-resolution',
+                    user: @org_user_1,
+                    formatter: '{dummy}',
+                    processed_rows: 100,
+                    created_at: DateTime.current).save
+
+      Geocoding.new(kind: 'high-resolution',
+                    user: @org_user_2,
+                    formatter: '{dummy}',
+                    processed_rows: 120,
+                    cache_hits: 10,
+                    created_at: DateTime.current - 1).save
+
+      usage_metrics_1.incr(:geocoder_here, :success_responses, 100, DateTime.current)
+      usage_metrics_2.incr(:geocoder_here, :success_responses, 120, DateTime.current - 1)
+      usage_metrics_2.incr(:geocoder_cache, :success_responses, 10, DateTime.current - 1)
+
+      @organization.get_new_system_geocoding_calls.should == 230
+    end
+
+    it 'calculates the used geocoder quota in the current billing cycle including empty requests' do
+      usage_metrics = CartoDB::GeocoderUsageMetrics.new(@mock_redis, @user1.username, nil)
+      Carto::TableGeocoderFactory.stubs(:get_geocoder_metrics_instance).returns(usage_metrics)
+      usage_metrics.incr(:geocoder_here, :success_responses, 10, DateTime.current)
+      usage_metrics.incr(:geocoder_here, :success_responses, 100, (DateTime.current - 2))
+      usage_metrics.incr(:geocoder_here, :empty_responses, 10, (DateTime.current - 2))
+      usage_metrics.incr(:geocoder_cache, :success_responses, 100, (DateTime.current - 1))
+
+      get_user_by_id(@user1.id).get_new_system_geocoding_calls.should == 220
+    end
+  end
 end