diff --git a/app/models/geocoding.rb b/app/models/geocoding.rb index 5f77bde960..54f038d894 100644 --- a/app/models/geocoding.rb +++ b/app/models/geocoding.rb @@ -38,6 +38,14 @@ class Geocoding < Sequel::Model dataset.where(kind: 'high-resolution').where('geocodings.created_at >= ? and geocodings.created_at <= ?', date_from, date_to + 1.days).sum("processed_rows + cache_hits".lit).to_i end + def self.get_not_aggregated_geocoding_calls(dataset, date_from, date_to) + geocoding_calls_sql = "SELECT date(created_at), sum(processed_rows) as processed_rows, " \ + "sum(cache_hits) as cache_hits FROM geocodings WHERE kind = 'high-resolution' " \ + "AND geocodings.created_at >= ? and geocodings.created_at <= ?" \ + "GROUP BY date(created_at) ORDER BY date(created_at) DESC" + dataset.db.fetch(geocoding_calls_sql, date_from, date_to + 1.days).all + end + def public_values Hash[PUBLIC_ATTRIBUTES.map{ |k| [k, (self.send(k) rescue self[k].to_s)] }] end diff --git a/app/models/user.rb b/app/models/user.rb index d63784b840..dde0fc8742 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -784,8 +784,13 @@ class User < Sequel::Model def get_geocoding_calls(options = {}) date_from, date_to = quota_dates(options) - Geocoding.get_geocoding_calls(self.geocodings_dataset, date_from, date_to) - end # get_geocoding_calls + Geocoding.get_geocoding_calls(geocodings_dataset, date_from, date_to) + end + + def get_not_aggregated_geocoding_calls(options = {}) + date_from, date_to = quota_dates(options) + Geocoding.get_not_aggregated_geocoding_calls(geocodings_dataset, date_from, date_to) + end def effective_twitter_block_price organization.present? ? organization.twitter_datasource_block_price : self.twitter_datasource_block_price diff --git a/lib/tasks/db_maintenance.rake b/lib/tasks/db_maintenance.rake index 1e9fce9474..944ac7101a 100644 --- a/lib/tasks/db_maintenance.rake +++ b/lib/tasks/db_maintenance.rake @@ -1,4 +1,5 @@ require_relative 'thread_pool' +require_relative '../../services/table-geocoder/lib/table_geocoder_factory' require 'timeout' namespace :cartodb do @@ -1256,5 +1257,25 @@ namespace :cartodb do }, 1, 0.3) end end + + desc 'Migrate the current billing geocoding data to Redis' + task :migrate_current_geocoder_billing_to_redis => [:environment] do |task, args| + execute_on_users_with_index(:migrate_current_geocoder_billing_to_redis.to_s, Proc.new { |user, i| + begin + usage_metrics = Carto::TableGeocoderFactory.get_geocoder_metrics_instance(user) + geocoder_key = user.google_maps_geocoder_enabled? ? :geocoder_google : :geocoder_here + geocoding_calls = user.get_not_aggregated_geocoding_calls({from: date_from, to: date_to}) + geocoding_calls.each do |metric| + usage_metrics.incr(geocoder_key, :success_responses, metric[:processed_rows], metric[:date]) + usage_metrics.incr(geocoder_key, :total_requests, metric[:processed_rows], metric[:date]) + usage_metrics.incr(:geocoder_cache, :success_responses, metric[:cache_hits], metric[:date]) + usage_metrics.incr(:geocoder_cache, :total_requests, metric[:cache_hits], metric[:date]) + puts "Imported metrics for day #{metric[:date]} and user #{user.username}: #{metric}" + end + rescue => e + puts "Error trying to migrate user current billing cycle to redis #{user.name}: #{e.message}" + end + }, 1, 0.3) + end end end diff --git a/services/table-geocoder/lib/geocoder_usage_metrics.rb b/services/table-geocoder/lib/geocoder_usage_metrics.rb index 5b881178f8..065ffd286d 100644 --- a/services/table-geocoder/lib/geocoder_usage_metrics.rb +++ b/services/table-geocoder/lib/geocoder_usage_metrics.rb @@ -27,16 +27,16 @@ module CartoDB @redis = redis end - def incr(service, metric, amount = 1) + def incr(service, metric, amount = 1, date = DateTime.current) check_valid_data(service, metric, amount) return if amount == 0 # TODO We could add EXPIRE command to add TTL to the keys if !@orgname.nil? - @redis.zincrby("#{org_key_prefix(service, metric)}", amount, "#{current_day}") + @redis.zincrby("#{org_key_prefix(service, metric, date)}", amount, "#{date_day(date)}") end - @redis.zincrby("#{user_key_prefix(service, metric)}", amount, "#{current_day}") + @redis.zincrby("#{user_key_prefix(service, metric, date)}", amount, "#{date_day(date)}") end private @@ -47,20 +47,20 @@ module CartoDB raise 'invalid amount' if amount < 0 end - def user_key_prefix(service, metric) - "user:#{@username}:#{service}:#{metric}:#{current_year_month}" + def user_key_prefix(service, metric, date) + "user:#{@username}:#{service}:#{metric}:#{date_year_month(date)}" end - def org_key_prefix(service, metric) - "org:#{@orgname}:#{service}:#{metric}:#{current_year_month}" + def org_key_prefix(service, metric, date) + "org:#{@orgname}:#{service}:#{metric}:#{date_year_month(date)}" end - def current_day - DateTime.current.strftime('%d') + def date_day(date) + date.strftime('%d') end - def current_year_month - DateTime.current.strftime('%Y%m') + def date_year_month(date) + date.strftime('%Y%m') end end