247 lines
10 KiB
Ruby
247 lines
10 KiB
Ruby
require 'open3'
|
|
require_relative '../../../lib/gme/table_geocoder'
|
|
require_relative '../../../../../lib/url_signer'
|
|
require_relative '../../../lib/gme/exceptions'
|
|
require_relative '../../factories/pg_connection'
|
|
require_relative '../../../../../spec/spec_helper.rb'
|
|
require_relative '../../../../../spec/rspec_configuration.rb'
|
|
|
|
describe Carto::Gme::TableGeocoder do
|
|
before(:all) do
|
|
connection_stub = mock
|
|
connection_stub.stubs(:run)
|
|
@usage_metrics_stub = stub
|
|
@log = mock
|
|
@log.stubs(:append)
|
|
@log.stubs(:append_and_store)
|
|
@geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}')
|
|
|
|
@mandatory_args = {
|
|
connection: connection_stub,
|
|
original_formatter: '{mock}',
|
|
client_id: 'my_client_id',
|
|
private_key: 'my_private_key',
|
|
usage_metrics: @usage_metrics_stub,
|
|
log: @log,
|
|
geocoding_model: @geocoding_model
|
|
}
|
|
end
|
|
|
|
describe '#initialize' do
|
|
it 'returns an object that responds to AbstractTableGeocoder interface' do
|
|
table_geocoder = Carto::Gme::TableGeocoder.new(@mandatory_args)
|
|
interface_methods = [:ensure_georef_status_colummn_valid,
|
|
:cancel,
|
|
:run,
|
|
:remote_id,
|
|
:update_geocoding_status,
|
|
:process_results]
|
|
interface_methods.each do |method|
|
|
table_geocoder.respond_to?(method, true).should == true
|
|
end
|
|
end
|
|
|
|
it 'raises an exception if not fed with mandatory arguments' do
|
|
expect { Carto::Gme::TableGeocoder.new }.to raise_error(ArgumentError)
|
|
|
|
@mandatory_args.each do |arg|
|
|
args_missing_one = @mandatory_args.dup
|
|
args_missing_one.delete(arg[0])
|
|
lambda { Carto::Gme::TableGeocoder.new(args_missing_one) }.should raise_error(KeyError)
|
|
end
|
|
end
|
|
|
|
it 'creates a client with the provided credentials' do
|
|
gme_client_mock = mock
|
|
Carto::Gme::Client.expects(:new).with('my_client_id', 'my_private_key').once.returns(gme_client_mock)
|
|
Carto::Gme::GeocoderClient.expects(:new).with(gme_client_mock).once
|
|
Carto::Gme::TableGeocoder.new(@mandatory_args)
|
|
end
|
|
end
|
|
|
|
describe '#run' do
|
|
before(:each) do
|
|
Carto::UrlSigner.any_instance.stubs(:sign_url).returns('https://maps.googleapis.com/maps/api/geocode/json')
|
|
@table_geocoder = Carto::Gme::TableGeocoder.new(@mandatory_args)
|
|
end
|
|
|
|
it "set's the state to 'processing' when it starts" do
|
|
pending 'actually as a requirement this does not make much sense'
|
|
end
|
|
|
|
it "set's the state to 'completed' when it ends" do
|
|
# TODO: there's something weird that needs review here
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
|
|
|
|
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
|
|
@table_geocoder.stubs(:data_input_blocks).returns([])
|
|
|
|
@table_geocoder.run
|
|
@geocoding_model.state.should == 'completed'
|
|
end
|
|
|
|
it "if there's an uncontrolled exception, sets the state to 'failed' and raises it" do
|
|
# TODO: there's something weird that needs review here
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
|
|
|
|
@table_geocoder.stubs(:ensure_georef_status_aolummn_valid)
|
|
@table_geocoder.stubs(:data_input_blocks).returns([{ cartodb_id: 1, searchtext: 'dummy text' }])
|
|
@table_geocoder.stubs(:geocode).raises(StandardError, 'unexpected exception')
|
|
|
|
expect { @table_geocoder.run }.to raise_error('unexpected exception')
|
|
@geocoding_model.state.should == 'failed'
|
|
end
|
|
|
|
it "processes 1 block at a time, keeping track of processed rows in each block" do
|
|
# TODO: there's something weird that needs review here
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 4)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 4)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
|
|
|
|
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
|
|
mocked_input = Enumerator.new do |enum|
|
|
# 2 blocks of 2 rows each as input
|
|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }, { cartodb_id: 2, searchtext: 'dummy text' }]
|
|
enum.yield [{ cartodb_id: 3, searchtext: 'dummy text' }, { cartodb_id: 4, searchtext: 'dummy text' }]
|
|
end
|
|
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
|
|
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_ok.json'))
|
|
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
|
|
@table_geocoder.expects(:update_table).twice
|
|
|
|
@table_geocoder.run
|
|
@table_geocoder.processed_rows.should == 4
|
|
end
|
|
|
|
it "processes empty response" do
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 1)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
|
|
|
|
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
|
|
mocked_input = Enumerator.new do |enum|
|
|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
|
|
end
|
|
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
|
|
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_empty.json'))
|
|
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
|
|
@table_geocoder.expects(:update_table).once
|
|
|
|
@table_geocoder.run
|
|
@table_geocoder.processed_rows.should == 1
|
|
end
|
|
|
|
it "processes error rows response" do
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 1)
|
|
|
|
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
|
|
mocked_input = Enumerator.new do |enum|
|
|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
|
|
end
|
|
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
|
|
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_error.json'))
|
|
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
|
|
@table_geocoder.expects(:update_table).once
|
|
|
|
@table_geocoder.run
|
|
@table_geocoder.processed_rows.should == 1
|
|
end
|
|
|
|
it "processes error with message response" do
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
|
|
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 1)
|
|
|
|
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
|
|
mocked_input = Enumerator.new do |enum|
|
|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
|
|
end
|
|
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
|
|
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_error_with_message.json'))
|
|
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
|
|
@table_geocoder.expects(:update_table).once
|
|
# CartoDB.expects(:notify_error).once
|
|
|
|
@table_geocoder.run
|
|
@table_geocoder.processed_rows.should == 1
|
|
end
|
|
end
|
|
|
|
describe '#data_input_blocks' do
|
|
before do
|
|
conn = CartoDB::Importer2::Factories::PGConnection.new
|
|
@db = conn.connection
|
|
@pg_options = conn.pg_options
|
|
@table_name = "ne_10m_populated_places_simple_#{rand.to_s[2..11]}"
|
|
|
|
# Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
|
|
filename = 'populated_places_short.csv'
|
|
_stdout, stderr, _status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
|
|
raise if stderr != ''
|
|
load_csv "/tmp/#{filename}"
|
|
|
|
params = {
|
|
connection: @db,
|
|
table_name: @table_name,
|
|
qualified_table_name: @table_name,
|
|
sequel_qualified_table_name: @table_name,
|
|
original_formatter: "{name}, {iso3}",
|
|
client_id: 'my_client_id',
|
|
private_key: 'my_private_key',
|
|
max_block_size: 4,
|
|
usage_metrics: @usage_metrics_stub,
|
|
log: @log,
|
|
geocoding_model: @geocoding_model
|
|
}
|
|
|
|
@table_geocoder = Carto::Gme::TableGeocoder.new(params)
|
|
end
|
|
|
|
after do
|
|
@db.drop_table @table_name
|
|
end
|
|
|
|
it 'performs (floor(rows / max_block_size) + 1) queries' do
|
|
rows = @db[@table_name.to_sym].count
|
|
@table_geocoder.send(:ensure_georef_status_colummn_valid)
|
|
|
|
count = 0
|
|
@table_geocoder.send(:data_input_blocks).each do |data_block|
|
|
data_block.each do |row|
|
|
row.merge!(cartodb_georef_status: false)
|
|
end
|
|
@table_geocoder.send(:update_table, data_block)
|
|
count += 1
|
|
end
|
|
count.should == (rows / @table_geocoder.max_block_size).floor + 1
|
|
end
|
|
end
|
|
|
|
def path_to(filepath = '')
|
|
File.expand_path(
|
|
File.join(File.dirname(__FILE__), "../../fixtures/#{filepath}")
|
|
)
|
|
end
|
|
|
|
def read_fixture_file(filename)
|
|
File.read(path_to(filename))
|
|
end
|
|
|
|
def load_csv(path)
|
|
@db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
|
|
@db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
|
|
end
|
|
end
|