Improve readability

pull/16326/head
moicalcob 3 years ago
parent 31aecf9f4c
commit 9edd10c705

@ -292,12 +292,12 @@ module CartoDB
# Sometimes we could try to recover from a known failure
def try_fallback(append_mode)
if ogr2ogr.invalid_dates?
job.log "Fallback: Disabling autoguessing because there are wrong dates in the source file"
job.log "Fallback: Autoguessing problem, trying to disable the problematic column"
@job.fallback_executed = "date"
ogr2ogr.overwrite = true
ogr2ogr.csv_guessing = true
ogr2ogr.quoted_fields_guessing = false
try_fix_invalid_field(ogr2ogr.filepath, ogr2ogr.command_output)
disable_autoguessing_on_wrong_column(ogr2ogr.filepath, ogr2ogr.command_output)
ogr2ogr.run(append_mode)
elsif ogr2ogr.encoding_error?
job.log "Fallback: There is an encoding problem, trying with ISO-8859-1"
@ -388,9 +388,9 @@ module CartoDB
error =~ /canceling statement due to statement timeout/i
end
def try_fix_invalid_field(filepath, command_output)
line = command_output.split('line')[1].split(':')[0].split(',')[0].delete(' ').to_i - 1
column = command_output.split('column')[1].split(':')[0].split(',')[0].delete(' ')
def disable_autoguessing_on_wrong_column(filepath, command_output)
line = /(?<=, line )\d+(?=,)/.match(command_output).to_s.to_i - 1
column = /(?<=, column )[a-zA-Z]+(?=:)/.match(command_output).to_s.strip()
csv_content = CSV.read(filepath, headers: true)
csv_content[line][column] = "\"#{csv_content[line][column]}\""
File.open(filepath, 'w') { |file| file.puts csv_content.to_s }

@ -377,7 +377,7 @@ describe 'csv regression tests' do
runner.results.first.success?.should eq true
end
it 'auto guessing should work in csv columns if one field is causing troubles' do
it 'should autoguess columns type correctly when the CSV contains invalid field values' do
runner = runner_with_fixture('wrong_date_auto_guessing.csv', nil, true)
runner.run
@ -387,7 +387,7 @@ describe 'csv regression tests' do
from #{result.schema}.#{result.table_name}
}].first.fetch(:age)
age.class.should eq Integer
expect(age).to be_an(Integer)
end
def sample_for(job)

@ -1,12 +1,3 @@
age,date,name
20,2000-01-01,name_1
21,2000-01-01,name_2
22,2000-1-NA,name_3
23,2000-01-01,name_4
24,2000-1-NA,name_5
25,2000-01-01,name_6
26,2000-1-NA,name_7
27,2000-01-01,name_8
28,2000-01-01,name_9
29,2000-1-NA,name_10
30,2000-01-01,name_11

1 age date name
2 20 2000-01-01 name_1
21 2000-01-01 name_2
3 22 2000-1-NA name_3
23 2000-01-01 name_4
24 2000-1-NA name_5
25 2000-01-01 name_6
26 2000-1-NA name_7
27 2000-01-01 name_8
28 2000-01-01 name_9
29 2000-1-NA name_10
30 2000-01-01 name_11
Loading…
Cancel
Save