138 lines
5.4 KiB
Ruby
138 lines
5.4 KiB
Ruby
require 'ostruct'
|
|
require_relative '../../../../spec/spec_helper_min'
|
|
require_relative '../../lib/importer/loader'
|
|
require_relative '../../lib/importer/source_file'
|
|
require_relative '../../lib/importer/exceptions'
|
|
require_relative '../doubles/job'
|
|
require_relative '../doubles/ogr2ogr'
|
|
require_relative '../doubles/georeferencer'
|
|
require_relative '../../spec/doubles/importer_stats'
|
|
|
|
describe CartoDB::Importer2::Loader do
|
|
before do
|
|
CartoDB::Stats::Aggregator.stubs(:read_config).returns({})
|
|
resultset = OpenStruct.new(:first => {:num_rows => 10})
|
|
db = Object.new
|
|
db.stubs(:fetch).returns(resultset)
|
|
@job = CartoDB::Importer2::Doubles::Job.new(db)
|
|
@source_file = CartoDB::Importer2::SourceFile.new('/var/tmp/foo')
|
|
@ogr2ogr = CartoDB::Importer2::Doubles::Ogr2ogr.new
|
|
@georeferencer = CartoDB::Importer2::Doubles::Georeferencer.new
|
|
@loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
|
|
end
|
|
|
|
before(:each) do
|
|
CartoDB::Stats::Aggregator.stubs(:read_config).returns({})
|
|
end
|
|
|
|
describe '#run' do
|
|
|
|
it 'logs the database connection options used' do
|
|
@loader.run
|
|
(@job.logger.to_s. =~ /#{@job.pg_options.keys.first}/).should_not be nil
|
|
end
|
|
|
|
it 'runs the ogr2ogr command to load the file' do
|
|
ogr2ogr_mock = mock
|
|
ogr2ogr_mock.stubs(:generic_error?).returns(false).twice
|
|
ogr2ogr_mock.stubs(:command).returns('')
|
|
ogr2ogr_mock.stubs(:command_output).returns('')
|
|
ogr2ogr_mock.stubs(:encoding_error?).returns(false)
|
|
ogr2ogr_mock.stubs(:invalid_dates?).returns(false)
|
|
ogr2ogr_mock.stubs(:duplicate_column?).returns(false)
|
|
ogr2ogr_mock.stubs(:invalid_geojson?).returns(false)
|
|
ogr2ogr_mock.stubs(:too_many_columns?).returns(false)
|
|
ogr2ogr_mock.stubs(:unsupported_format?).returns(false)
|
|
ogr2ogr_mock.stubs(:file_too_big?).returns(false)
|
|
ogr2ogr_mock.stubs(:statement_timeout?).returns(false)
|
|
ogr2ogr_mock.stubs(:duplicate_column?).returns(false)
|
|
ogr2ogr_mock.stubs(:segfault_error?).returns(false)
|
|
ogr2ogr_mock.stubs(:kml_style_missing?).returns(false)
|
|
ogr2ogr_mock.stubs(:missing_srs?).returns(false)
|
|
ogr2ogr_mock.stubs(:geometry_validity_error?).returns(false)
|
|
ogr2ogr_mock.stubs(:exit_code).returns(0)
|
|
ogr2ogr_mock.stubs(:run).returns(Object.new).at_least_once
|
|
|
|
loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, ogr2ogr_mock, @georeferencer)
|
|
|
|
loader.run
|
|
end
|
|
|
|
it 'logs the exit code from ogr2ogr' do
|
|
@loader.run
|
|
(@job.logger.to_s =~ /ogr2ogr exit code:\s+\d+/).should_not be nil
|
|
end
|
|
|
|
it 'logs any output from ogr2ogr' do
|
|
@loader.run
|
|
(@job.logger.to_s =~ /ogr2ogr output: \w*/).should_not be nil
|
|
end
|
|
|
|
it 'encoding problem importing but return 0, should try fallback and then raise an error' do
|
|
resultset = OpenStruct.new(:first => {:num_rows => 0})
|
|
db = Object.new
|
|
db.stubs(:fetch).returns(resultset)
|
|
@job = CartoDB::Importer2::Doubles::Job.new(db)
|
|
# Enter fallback
|
|
@ogr2ogr.stubs(:generic_error?).returns(true)
|
|
# Fails after fallback
|
|
@ogr2ogr.stubs(:encoding_error?).returns(true)
|
|
@ogr2ogr.stubs(:exit_code).returns(0)
|
|
loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
|
|
loader.expects(:try_fallback).once
|
|
expect { loader.run }.to raise_error(CartoDB::Importer2::RowsEncodingColumnError)
|
|
end
|
|
end
|
|
|
|
describe '#ogr2ogr' do
|
|
|
|
it 'returns the passed ogr2ogr instance' do
|
|
ogr2ogr = Object.new
|
|
loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, ogr2ogr, @georeferencer)
|
|
|
|
loader.ogr2ogr.should eq ogr2ogr
|
|
end
|
|
|
|
it 'initializes an ogr2ogr command wrapper if none passed' do
|
|
loader = CartoDB::Importer2::Loader.new(@job, @source_file)
|
|
loader.ogr2ogr.class.name.should eq 'CartoDB::Importer2::Ogr2ogr'
|
|
end
|
|
|
|
it 'processes ogr2ogr generic errors' do
|
|
loader = CartoDB::Importer2::Loader.new(@job, @source_file)
|
|
loader.ogr2ogr.stubs(generic_error?: true, command: ['ogr2ogr', '-some', '-option'])
|
|
loader.send(:job).logger.stubs(:append)
|
|
|
|
expect { loader.send(:check_for_import_errors) }.to raise_error CartoDB::Importer2::LoadError
|
|
end
|
|
end
|
|
|
|
describe 'stats logger' do
|
|
before do
|
|
resultset = OpenStruct.new(:first => {:num_rows => 10})
|
|
db = Object.new
|
|
db.stubs(:fetch).returns(resultset)
|
|
@job = CartoDB::Importer2::Doubles::Job.new(db)
|
|
@source_file = CartoDB::Importer2::SourceFile.new('/var/tmp/foo')
|
|
@ogr2ogr = CartoDB::Importer2::Doubles::Ogr2ogr.new
|
|
@georeferencer = CartoDB::Importer2::Doubles::Georeferencer.new
|
|
@loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
|
|
@importer_stats_spy = CartoDB::Doubles::Stats::Importer.instance
|
|
end
|
|
|
|
it 'logs stats' do
|
|
loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
|
|
loader.set_importer_stats(@importer_stats_spy)
|
|
loader.run
|
|
@importer_stats_spy.timed_block_suffix_count('loader').should eq 1
|
|
@importer_stats_spy.timed_block_suffix_count('loader.normalize').should eq 1
|
|
@importer_stats_spy.timed_block_suffix_count('loader.ogr2ogr').should eq 1
|
|
@importer_stats_spy.timed_block_suffix_count('loader.post_ogr2ogr_tasks').should eq 1
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|