cartodb-4.42/services/importer/spec/unit/loader_spec.rb

require 'ostruct'
require_relative '../../../../spec/spec_helper_min'
require_relative '../../lib/importer/loader'
require_relative '../../lib/importer/source_file'
require_relative '../../lib/importer/exceptions'
require_relative '../doubles/job'
require_relative '../doubles/ogr2ogr'
require_relative '../doubles/georeferencer'
require_relative '../../spec/doubles/importer_stats'

describe CartoDB::Importer2::Loader do
  before do
    CartoDB::Stats::Aggregator.stubs(:read_config).returns({})
    resultset = OpenStruct.new(:first => {:num_rows => 10})
    db = Object.new
    db.stubs(:fetch).returns(resultset)
    @job            = CartoDB::Importer2::Doubles::Job.new(db)
    @source_file    = CartoDB::Importer2::SourceFile.new('/var/tmp/foo')
    @ogr2ogr        = CartoDB::Importer2::Doubles::Ogr2ogr.new
    @georeferencer  = CartoDB::Importer2::Doubles::Georeferencer.new
    @loader         = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
  end

  before(:each) do
    CartoDB::Stats::Aggregator.stubs(:read_config).returns({})
  end

  describe '#run' do

    it 'logs the database connection options used' do
      @loader.run
      (@job.logger.to_s. =~ /#{@job.pg_options.keys.first}/).should_not be nil
    end

    it 'runs the ogr2ogr command to load the file' do
      ogr2ogr_mock = mock
      ogr2ogr_mock.stubs(:generic_error?).returns(false).twice
      ogr2ogr_mock.stubs(:command).returns('')
      ogr2ogr_mock.stubs(:command_output).returns('')
      ogr2ogr_mock.stubs(:encoding_error?).returns(false)
      ogr2ogr_mock.stubs(:invalid_dates?).returns(false)
      ogr2ogr_mock.stubs(:duplicate_column?).returns(false)
      ogr2ogr_mock.stubs(:invalid_geojson?).returns(false)
      ogr2ogr_mock.stubs(:too_many_columns?).returns(false)
      ogr2ogr_mock.stubs(:unsupported_format?).returns(false)
      ogr2ogr_mock.stubs(:file_too_big?).returns(false)
      ogr2ogr_mock.stubs(:statement_timeout?).returns(false)
      ogr2ogr_mock.stubs(:duplicate_column?).returns(false)
      ogr2ogr_mock.stubs(:segfault_error?).returns(false)
      ogr2ogr_mock.stubs(:kml_style_missing?).returns(false)
      ogr2ogr_mock.stubs(:missing_srs?).returns(false)
      ogr2ogr_mock.stubs(:geometry_validity_error?).returns(false)
      ogr2ogr_mock.stubs(:exit_code).returns(0)
      ogr2ogr_mock.stubs(:run).returns(Object.new).at_least_once

      loader   = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, ogr2ogr_mock, @georeferencer)

      loader.run
    end

    it 'logs the exit code from ogr2ogr' do
      @loader.run
      (@job.logger.to_s =~ /ogr2ogr exit code:\s+\d+/).should_not be nil
    end

    it 'logs any output from ogr2ogr' do
      @loader.run
      (@job.logger.to_s =~ /ogr2ogr output: \w*/).should_not be nil
    end

    it 'encoding problem importing but return 0, should try fallback and then raise an error' do
      resultset = OpenStruct.new(:first => {:num_rows => 0})
      db = Object.new
      db.stubs(:fetch).returns(resultset)
      @job  = CartoDB::Importer2::Doubles::Job.new(db)
      # Enter fallback
      @ogr2ogr.stubs(:generic_error?).returns(true)
      # Fails after fallback
      @ogr2ogr.stubs(:encoding_error?).returns(true)
      @ogr2ogr.stubs(:exit_code).returns(0)
      loader = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
      loader.expects(:try_fallback).once
      expect { loader.run }.to raise_error(CartoDB::Importer2::RowsEncodingColumnError)
    end
  end

  describe '#ogr2ogr' do

    it 'returns the passed ogr2ogr instance' do
      ogr2ogr = Object.new
      loader  = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, ogr2ogr, @georeferencer)

      loader.ogr2ogr.should eq ogr2ogr
    end

    it 'initializes an ogr2ogr command wrapper if none passed' do
      loader  = CartoDB::Importer2::Loader.new(@job, @source_file)
      loader.ogr2ogr.class.name.should eq 'CartoDB::Importer2::Ogr2ogr'
    end

    it 'processes ogr2ogr generic errors' do
      loader = CartoDB::Importer2::Loader.new(@job, @source_file)
      loader.ogr2ogr.stubs(generic_error?: true, command: ['ogr2ogr', '-some', '-option'])
      loader.send(:job).logger.stubs(:append)

      expect { loader.send(:check_for_import_errors) }.to raise_error CartoDB::Importer2::LoadError
    end
  end

  describe 'stats logger' do
    before do
      resultset = OpenStruct.new(:first => {:num_rows => 10})
      db = Object.new
      db.stubs(:fetch).returns(resultset)
      @job            = CartoDB::Importer2::Doubles::Job.new(db)
      @source_file    = CartoDB::Importer2::SourceFile.new('/var/tmp/foo')
      @ogr2ogr        = CartoDB::Importer2::Doubles::Ogr2ogr.new
      @georeferencer  = CartoDB::Importer2::Doubles::Georeferencer.new
      @loader         = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
      @importer_stats_spy = CartoDB::Doubles::Stats::Importer.instance
    end

    it 'logs stats' do
      loader  = CartoDB::Importer2::Loader.new(@job, @source_file, layer=nil, @ogr2ogr, @georeferencer)
      loader.set_importer_stats(@importer_stats_spy)
      loader.run
      @importer_stats_spy.timed_block_suffix_count('loader').should eq 1
      @importer_stats_spy.timed_block_suffix_count('loader.normalize').should eq 1
      @importer_stats_spy.timed_block_suffix_count('loader.ogr2ogr').should eq 1
      @importer_stats_spy.timed_block_suffix_count('loader.post_ogr2ogr_tasks').should eq 1

    end

  end

end