Rake task to migrate the current cycle geocoding metrics from DB to Redis

app/models/geocoding.rb

@@ -38,6 +38,14 @@ class Geocoding < Sequel::Model
     dataset.where(kind: 'high-resolution').where('geocodings.created_at >= ? and geocodings.created_at <= ?', date_from, date_to + 1.days).sum("processed_rows + cache_hits".lit).to_i
   end
 
+  def self.get_not_aggregated_geocoding_calls(dataset, date_from, date_to)
+    geocoding_calls_sql = "SELECT date(created_at), sum(processed_rows) as processed_rows, " \
+                          "sum(cache_hits) as cache_hits FROM geocodings WHERE kind = 'high-resolution' " \
+                          "AND geocodings.created_at >= ? and geocodings.created_at <= ?" \
+                          "GROUP BY date(created_at) ORDER BY date(created_at) DESC"
+    dataset.db.fetch(geocoding_calls_sql, date_from, date_to + 1.days).all
+  end
+
   def public_values
     Hash[PUBLIC_ATTRIBUTES.map{ |k| [k, (self.send(k) rescue self[k].to_s)] }]
   end

app/models/user.rb

@@ -784,8 +784,13 @@ class User < Sequel::Model
 
   def get_geocoding_calls(options = {})
     date_from, date_to = quota_dates(options)
-    Geocoding.get_geocoding_calls(self.geocodings_dataset, date_from, date_to)
-  end # get_geocoding_calls
+    Geocoding.get_geocoding_calls(geocodings_dataset, date_from, date_to)
+  end
+
+  def get_not_aggregated_geocoding_calls(options = {})
+    date_from, date_to = quota_dates(options)
+    Geocoding.get_not_aggregated_geocoding_calls(geocodings_dataset, date_from, date_to)
+  end
 
   def effective_twitter_block_price
     organization.present? ? organization.twitter_datasource_block_price : self.twitter_datasource_block_price

lib/tasks/db_maintenance.rake

@@ -1,4 +1,5 @@
 require_relative 'thread_pool'
+require_relative '../../services/table-geocoder/lib/table_geocoder_factory'
 require 'timeout'
 
 namespace :cartodb do
@@ -1256,5 +1257,25 @@ namespace :cartodb do
         }, 1, 0.3)
       end
     end
+
+    desc 'Migrate the current billing geocoding data to Redis'
+    task :migrate_current_geocoder_billing_to_redis => [:environment] do |task, args|
+      execute_on_users_with_index(:migrate_current_geocoder_billing_to_redis.to_s, Proc.new { |user, i|
+        begin
+          usage_metrics = Carto::TableGeocoderFactory.get_geocoder_metrics_instance(user)
+          geocoder_key = user.google_maps_geocoder_enabled? ? :geocoder_google : :geocoder_here
+          geocoding_calls = user.get_not_aggregated_geocoding_calls({from: date_from, to: date_to})
+          geocoding_calls.each do |metric|
+            usage_metrics.incr(geocoder_key, :success_responses, metric[:processed_rows], metric[:date])
+            usage_metrics.incr(geocoder_key, :total_requests, metric[:processed_rows], metric[:date])
+            usage_metrics.incr(:geocoder_cache, :success_responses, metric[:cache_hits], metric[:date])
+            usage_metrics.incr(:geocoder_cache, :total_requests, metric[:cache_hits], metric[:date])
+            puts "Imported metrics for day #{metric[:date]} and user #{user.username}: #{metric}"
+          end
+        rescue => e
+          puts "Error trying to migrate user current billing cycle to redis #{user.name}: #{e.message}"
+        end
+      }, 1, 0.3)
+    end
   end
 end

services/table-geocoder/lib/geocoder_usage_metrics.rb

@@ -27,16 +27,16 @@ module CartoDB
       @redis = redis
     end
 
-    def incr(service, metric, amount = 1)
+    def incr(service, metric, amount = 1, date = DateTime.current)
       check_valid_data(service, metric, amount)
       return if amount == 0
 
       # TODO We could add EXPIRE command to add TTL to the keys
       if !@orgname.nil?
-        @redis.zincrby("#{org_key_prefix(service, metric)}", amount, "#{current_day}")
+        @redis.zincrby("#{org_key_prefix(service, metric, date)}", amount, "#{date_day(date)}")
       end
 
-      @redis.zincrby("#{user_key_prefix(service, metric)}", amount, "#{current_day}")
+      @redis.zincrby("#{user_key_prefix(service, metric, date)}", amount, "#{date_day(date)}")
     end
 
     private
@@ -47,20 +47,20 @@ module CartoDB
       raise 'invalid amount' if amount < 0
     end
 
-    def user_key_prefix(service, metric)
-      "user:#{@username}:#{service}:#{metric}:#{current_year_month}"
+    def user_key_prefix(service, metric, date)
+      "user:#{@username}:#{service}:#{metric}:#{date_year_month(date)}"
     end
 
-    def org_key_prefix(service, metric)
-      "org:#{@orgname}:#{service}:#{metric}:#{current_year_month}"
+    def org_key_prefix(service, metric, date)
+      "org:#{@orgname}:#{service}:#{metric}:#{date_year_month(date)}"
     end
 
-    def current_day
-      DateTime.current.strftime('%d')
+    def date_day(date)
+      date.strftime('%d')
     end
 
-    def current_year_month
-      DateTime.current.strftime('%Y%m')
+    def date_year_month(date)
+      date.strftime('%Y%m')
     end
 
   end