cartodb/services/importer/spec/unit/ogr2ogr_spec.rb

require 'pg'
require 'sequel'
require_relative '../../../../spec/spec_helper'
require_relative '../../lib/importer/ogr2ogr'
require_relative '../doubles/job'
require_relative '../factories/csv'
require_relative '../factories/pg_connection'

include CartoDB::Importer2

describe Ogr2ogr do
  before(:all) do
    @user             = create_user
    @user.save
    @pg_options       = @user.db_service.db_configuration_for.with_indifferent_access
    @db               = @user.in_database
    @db.execute('CREATE SCHEMA IF NOT EXISTS cdb_importer')
    @db.execute('SET search_path TO cdb_importer,public')
  end

  before(:each) do
    @csv              = Factories::CSV.new.write
    @filepath         = @csv.filepath
    @table_name       = "importer_#{rand(99999)}"
    @full_table_name  = "cdb_importer.#{@table_name}"
    @dataset          = @db[@table_name.to_sym]
    @wrapper          = CartoDB::Importer2::Ogr2ogr.new(@table_name, @filepath, @pg_options)
  end

  after(:each) do
    @csv.delete
    @db.drop_table? @full_table_name
  end

  after(:all) do
    @db.execute('DROP SCHEMA cdb_importer cascade')
    @db.disconnect
    @user.destroy
  end

  describe '#initialize' do
    it 'requires a filepath and postgres options' do
      expect{
        CartoDB::Importer2::Ogr2ogr.new
      }.to raise_error ArgumentError
      expect{
        CartoDB::Importer2::Ogr2ogr.new('bogus.txt')
      }.to raise_error ArgumentError
    end
  end

  describe '#command' do
    it 'includes an encoding' do
      @wrapper.environment.should include 'PGCLIENTENCODING'
    end

    it 'includes the postgres options passed at initialization time' do
      @wrapper.command.any? { |c| c.include?(@pg_options.fetch(:host)) }.should be_true
      @wrapper.command.any? { |c| c.include?(@pg_options.fetch(:port).to_s) }.should be_true
      @wrapper.command.any? { |c| c.include?(@pg_options.fetch(:username)) }.should be_true
      @wrapper.command.any? { |c| c.include?(@pg_options.fetch(:database)) }.should be_true
    end

    it 'includes the desired output table name' do
      @wrapper.command.should include @full_table_name
    end

    it 'includes the filepath to process' do
      @wrapper.command.should include @filepath
    end
  end

  describe '#executable_path' do
    it 'returns the path to the ogr2ogr binary' do
      (@wrapper.executable_path =~ /ogr2ogr/).should_not be nil
    end
  end

  describe '#run' do
    it 'imports a CSV to a Postgres table' do
      @wrapper.run
      records = @dataset.to_a

      records.length.should eq 10
      records.first.keys.include?(:header_1).should eq true
      records.first.keys.include?(:header_2).should eq true
    end

    it 'keeps an existing cartodb_id column in imported records' do
      header = ["cartodb_id", "header_2"]
      data   = ["5", "cell_#{rand(999)}"]
      csv    = Factories::CSV.new.write(header, data)

      @wrapper = CartoDB::Importer2::Ogr2ogr.new(@table_name, csv.filepath, @pg_options)
      @wrapper.run

      record    = @dataset.first
      record.fetch(:cartodb_id).should eq '5'
    end

    it 'Does not create header if one column is numerical' do
      header = ["id", "2"]
      data   = ["5", "cell_#{rand(999)}"]
      csv    = Factories::CSV.new.write(header, data)

      @wrapper = CartoDB::Importer2::Ogr2ogr.new(@table_name, csv.filepath, @pg_options)
      @wrapper.run

      record    = @dataset.first
      record.fetch(:field_1).should eq 'id'
    end

  end

  describe '#command_output' do
    it 'returns stdout and stderr from ogr2ogr binary' do
      wrapper = CartoDB::Importer2::Ogr2ogr.new(@table_name, 'non_existent', @pg_options)
      wrapper.run
      wrapper.command_output.should_not eq ''

      wrapper = CartoDB::Importer2::Ogr2ogr.new(@table_name, @filepath, @pg_options)
      wrapper.run
      wrapper.command_output.should eq ''
    end
  end

  describe '#exit_code' do
    it 'returns the exit code from the ogr2ogr binary' do
      wrapper = CartoDB::Importer2::Ogr2ogr.new(@table_name, 'non_existent', @pg_options)
      wrapper.run
      wrapper.exit_code.should_not eq 0

      wrapper = CartoDB::Importer2::Ogr2ogr.new(@table_name, @filepath, @pg_options)
      wrapper.run
      wrapper.exit_code.should eq 0
    end
  end

  describe '#append_mode' do
    it "tests that ogr2ogr's append mode works as expected" do
      header = ["cartodb_id", "header_2"]
      data_1   = ["1", "cell_#{rand(999)}"]
      data_2   = ["2", "cell_#{rand(999)}"]

      csv_1 = Factories::CSV.new(name=nil, how_many_duplicates=0)
        .write(header, data_1)

      csv_2 = Factories::CSV.new(name=nil, how_many_duplicates=0)
        .write(header, data_2)

      ogr2ogr = CartoDB::Importer2::Ogr2ogr.new(@table_name, csv_1.filepath, @pg_options)
      ogr2ogr.run

      ogr2ogr.filepath = csv_2.filepath
      ogr2ogr.run(append_mode=true)

      @dataset.all[0].fetch(:cartodb_id).should eq '1'
      @dataset.all[1].fetch(:cartodb_id).should eq '2'
      @dataset.all.count.should eq 2
    end
  end

end