Fix auto guessing when importing CSV

services/importer/lib/importer/loader.rb

@@ -295,7 +295,9 @@ module CartoDB
           job.log "Fallback: Disabling autoguessing because there are wrong dates in the source file"
           @job.fallback_executed = "date"
           ogr2ogr.overwrite = true
-          ogr2ogr.csv_guessing = false
+          ogr2ogr.csv_guessing = true
+          ogr2ogr.quoted_fields_guessing = false
+          try_fix_invalid_field(ogr2ogr.filepath, ogr2ogr.command_output)
           ogr2ogr.run(append_mode)
         elsif ogr2ogr.encoding_error?
           job.log "Fallback: There is an encoding problem, trying with ISO-8859-1"
@@ -385,6 +387,14 @@ module CartoDB
       def statement_timeout?(error)
         error =~ /canceling statement due to statement timeout/i
       end
+
+      def try_fix_invalid_field(filepath, command_output)
+        line = command_output.split("line")[1].split(':')[0].split(',')[0].delete(' ').to_i - 1
+        column = command_output.split("column")[1].split(':')[0].split(',')[0].delete(' ')
+        csv_content = CSV.read(filepath, headers: true)
+        csv_content[line][column] = '"' + csv_content[line][column] + '"'
+        File.open(filepath, "w") {|file| file.puts csv_content.to_s }
+      end
     end
   end
 end

services/importer/lib/importer/ogr2ogr.rb

@@ -151,13 +151,13 @@ module CartoDB
       end
 
       attr_accessor :append_mode, :filepath, :csv_guessing, :overwrite, :encoding, :shape_encoding,
-                    :shape_coordinate_system, :memory_limit
+                    :shape_coordinate_system, :memory_limit, :quoted_fields_guessing
       attr_reader   :exit_code, :command_output
 
       private
 
       attr_writer   :exit_code, :command_output
-      attr_accessor :pg_options, :options, :table_name, :layer, :ogr2ogr_binary, :quoted_fields_guessing
+      attr_accessor :pg_options, :options, :table_name, :layer, :ogr2ogr_binary
 
       def is_csv?
         !(filepath =~ /\.csv$/i).nil?

services/importer/spec/acceptance/csv_spec.rb

@@ -374,9 +374,21 @@ describe 'csv regression tests' do
     runner = runner_with_fixture('wrong_date.csv', nil, true)
     runner.run
 
+    byebug
     runner.results.first.success?.should eq true
   end
 
+  it 'auto guessing should work in csv columns if one field is causing troubles' do
+    runner = runner_with_fixture('wrong_date_auto_guessing.csv', nil, true)
+    runner.run
+
+    result = runner.results.first
+    @user.in_database[%Q{
+    SELECT *
+    from #{result.schema}.#{result.table_name}
+    }].first.fetch(:age).class.should eq (Integer)
+  end
+
   def sample_for(job)
     job.db[%Q{
       SELECT *

services/importer/spec/fixtures/wrong_date_auto_guessing.csv

@@ -0,0 +1,12 @@
+age,date,name
+20,2000-01-01,name_1
+21,2000-01-01,name_2
+22,2000-1-NA,name_3
+23,2000-01-01,name_4
+24,2000-1-NA,name_5
+25,2000-01-01,name_6
+26,2000-1-NA,name_7
+27,2000-01-01,name_8
+28,2000-01-01,name_9
+29,2000-1-NA,name_10
+30,2000-01-01,name_11