cartodb/services/table-geocoder/lib/gme/table_geocoder.rb

require_relative '../abstract_table_geocoder'
require_relative 'client'
require_relative 'geocoder_client'

module Carto
  module Gme
    class TableGeocoder < CartoDB::AbstractTableGeocoder

      DEFAULT_MAX_BLOCK_SIZE = 1000

      # See https://developers.google.com/maps/documentation/geocoding/#Types
      ACCEPTED_ADDRESS_TYPES = ['street_address', 'route', 'intersection', 'neighborhood']

      attr_reader :original_formatter, :processed_rows, :successful_processed_rows, :failed_processed_rows,
                  :empty_processed_rows, :state, :max_block_size

      def initialize(arguments)
        super(arguments)
        @original_formatter = arguments.fetch(:original_formatter)
        client_id = arguments.fetch(:client_id)
        private_key = arguments.fetch(:private_key)
        @max_block_size = arguments[:max_block_size] || DEFAULT_MAX_BLOCK_SIZE
        gme_client = Client.new(client_id, private_key)
        @geocoder_client = GeocoderClient.new(gme_client)
        @usage_metrics = arguments.fetch(:usage_metrics)
        @log = arguments.fetch(:log)
        @geocoding_model = arguments.fetch(:geocoding_model)
      end

      def cancel; end

      def run
        change_status('running')
        init_rows_count
        ensure_georef_status_colummn_valid

        # Here's the actual stuff
        data_input_blocks.each do |data_block|
          geocode(data_block)
          update_table(data_block)
          @processed_rows += data_block.size
        end

        change_status('completed')
      rescue => e
        change_status('failed')
        raise e
      ensure
        total_requests = @successful_processed_rows + @empty_processed_rows + @failed_processed_rows
        @usage_metrics.incr(:geocoder_google, :success_responses, @successful_processed_rows)
        @usage_metrics.incr(:geocoder_google, :empty_responses, @empty_processed_rows)
        @usage_metrics.incr(:geocoder_google, :failed_responses, @failed_processed_rows)
        @usage_metrics.incr(:geocoder_google, :total_requests, total_requests)
        update_log_stats
      end

      # Empty methods, needed because they're triggered from geocoding.rb
      def remote_id; end
      def process_results; end # TODO: can be removed from here and abstract class

      def update_geocoding_status
        { processed_rows: processed_rows, state: @geocoding_model.state }
      end

      def name
        'google'
      end

      private

      # Returns a "generator"
      def data_input_blocks
        Enumerator.new do |enum|
          begin
            data_input = connection.select(:cartodb_id, searchtext_expression)
              .from(@sequel_qualified_table_name)
              .where(cartodb_georef_status: nil)
              .limit(max_block_size)
              .all
            enum.yield data_input
            # last iteration when data_input.length < max_block_size, no need for another query
          end while data_input.length == max_block_size
        end
      end

      def searchtext_expression
        # The original_formatter has the following format:
        #   `{street_column_name}[[, additional_free_text][, {province_column_name}][, country_free_text]]`
        # See https://github.com/jeremyevans/sequel/blob/master/doc/security.rdoc
        # See http://sequel.jeremyevans.net/rdoc/classes/Sequel/SQL/Builders.html
        atoms = original_formatter.split(',').map {|s| s.strip }
        Sequel.join(atoms.map { |atom|
                      if match = /\A{(?<column_name>.*)}\z/.match(atom)
                        Sequel.identifier(match[:column_name])
                      else
                        atom
                      end
                    }, ',').as(:searchtext)
      end

      def geocode(data_block)
        data_block.each do |row|
          response = fetch_from_gme(row[:searchtext])
          # If we get an error we get nil so we pass to the next row
          next if response.nil?
          if response['status'] != Client::OK_STATUS
            process_error_or_empty_status(response['status'])
            row.merge!(cartodb_georef_status: false)
          else
            result = response['results'].select { |r| r['types'] & ACCEPTED_ADDRESS_TYPES }.first
            if result.nil?
              @empty_processed_rows += 1
              row.merge!(cartodb_georef_status: false)
            else
              @successful_processed_rows += 1
              location = result['geometry']['location']
              row.merge!(location.deep_symbolize_keys.merge(cartodb_georef_status: true))
            end
          end
        end
      end

      def update_table(data_block)
        # At this point, data_block is an Array that looks like this:
        #   [{:cartodb_id=>1, :searchtext=>"Some real street name", :lat=>19.29544, :lng=>-99.1472101, :cartodb_georef_status=>true},
        #    {:cartodb_id=>2, :searchtext=>"foo", :cartodb_georef_status=>false}]
        geocoded = data_block.select {|row| row[:cartodb_georef_status] == true}
        if geocoded.count > 0
          geocoded_to_sql = geocoded.map {|row| "(#{row[:cartodb_id]}, #{row[:lat]}, #{row[:lng]})"}.join(',')
          query_geocoded = %Q{
            UPDATE #{@qualified_table_name} as target SET
              the_geom = CDB_LatLng(geocoded.lat,geocoded.lng),
              cartodb_georef_status = TRUE
            FROM (VALUES
              #{geocoded_to_sql}
            ) as geocoded(cartodb_id,lat,lng)
            WHERE target.cartodb_id = geocoded.cartodb_id;
          }
          connection.run(query_geocoded)
        end

        non_geocoded = data_block.select {|row| row[:cartodb_georef_status] == false}
        if non_geocoded.count > 0
          non_geocoded_to_sql = non_geocoded.map {|row| "(#{row[:cartodb_id]})"}.join(',')
          query_non_geocoded = %Q{
            UPDATE #{@qualified_table_name} as target SET
              cartodb_georef_status = FALSE
            FROM (VALUES
              #{non_geocoded_to_sql}
            ) as nongeocoded(cartodb_id)
            WHERE target.cartodb_id = nongeocoded.cartodb_id;
          }
          connection.run(query_non_geocoded)
        end
      end

      private

      def init_rows_count
        @processed_rows = 0
        @successful_processed_rows = 0
        @failed_processed_rows = 0
        @empty_processed_rows = 0
      end

      def fetch_from_gme(search_text)
        @geocoder_client.geocode(search_text)
      rescue => e
        # Remove temporarily because it's flooding the logs
        # @log.append_and_store "Error geocoding using GME for text #{search_text}: #{e.message}"
        # CartoDB.notify_error('Error geocoding using GME', error: e.backtrace.join('\n'), search_text: search_text)
        @failed_processed_rows += 1
        nil
      end

      def process_error_or_empty_status(status)
        case status
          when Client::ZERO_RESULTS_STATUS then @empty_processed_rows += 1
          else @failed_processed_rows += 1
        end
      end

      def update_log_stats
        @log.append_and_store "Geocoding using Google maps, job status update. "\
          "Status: #{@status} --- Processed rows: #{@processed_rows} "\
          "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
          "--- Failed: #{@failed_processed_rows}"
      end

      def change_status(status)
        @status = status
        @geocoding_model.state = status
        @geocoding_model.save
      end
    end
  end
end