Merge pull request #16385 from CartoDB/feature/sc-201043/denver-mile-high-admin-not-possible-to-autoguess

NEWS.md

@@ -18,6 +18,7 @@ Development
 * Show user's database location in profile [16349](https://github.com/CartoDB/cartodb/pull/16349)
 * Setting to enable/disable import notifications [16354](https://github.com/CartoDB/cartodb/pull/16354)
 * Setting to enable/disable random username generation on SAML authentication process [16372](https://github.com/CartoDB/cartodb/pull/16372)
+* Add type guessing capabilities to the ArcGIS connector [#16385](https://github.com/CartoDB/cartodb/pull/16385)
 
 ### Bug fixes / enhancements
 - Fix rubocop integration [#16382](https://github.com/CartoDB/cartodb/pull/16382)

services/importer/lib/importer/arcgis_autoguessing.rb

@@ -0,0 +1,31 @@
+require 'open3'
+require_relative './shp_helper'
+
+module CartoDB
+  module Importer2
+    class ArcGISAutoguessing
+
+      def initialize(db, schema_name, table_name, fields_metadata)
+        @db              = db
+        @schema_name     = schema_name
+        @table_name      = table_name
+        @fields_metadata = fields_metadata
+      end
+
+      def run
+        autoguess_dates
+      end
+
+      def autoguess_dates
+        date_fields = @fields_metadata.select { |field| field['type'] == 'esriFieldTypeDate' }
+        date_fields.each do |field|
+          @db.run(%{
+            ALTER TABLE #{@schema_name}.#{@table_name} ALTER COLUMN #{field['name'].downcase} TYPE DATE
+            using to_timestamp(cast(#{field['name'].downcase}::bigint/1000 as bigint))::date
+          })
+        end
+      end
+
+    end
+  end
+end

services/importer/lib/importer/loader.rb

@@ -10,6 +10,7 @@ require_relative './georeferencer'
 require_relative '../importer/post_import_handler'
 require_relative './geometry_fixer'
 require_relative './typecaster'
+require_relative './arcgis_autoguessing'
 
 require_relative '../../../../lib/cartodb/stats/importer'
 
@@ -92,13 +93,13 @@ module CartoDB
         run_ogr2ogr(append_mode=true)
       end
 
-      def streamed_run_finish(post_import_handler_instance=nil)
+      def streamed_run_finish(post_import_handler_instance = nil, datasource_name = nil)
         @post_import_handler = post_import_handler_instance
 
-        post_ogr2ogr_tasks
+        post_ogr2ogr_tasks(datasource_name)
       end
 
-      def post_ogr2ogr_tasks
+      def post_ogr2ogr_tasks(datasource_name = nil)
         georeferencer.mark_as_from_geojson_with_transform if post_import_handler.has_transform_geojson_geom_column?
 
         job.log 'Georeferencing...'
@@ -107,21 +108,27 @@ module CartoDB
 
         if post_import_handler.has_fix_geometries_task?
           job.log 'Fixing geometry...'
-          # At this point the_geom column is renamed
+          fix_geometries(job)
-          begin
-            GeometryFixer.new(job.db, job.table_name, SCHEMA, 'the_geom', job).run
-          rescue StandardError => e
-            raise e unless statement_timeout?(e.to_s)
-            # Ignore timeouts in query batcher
-            log_warning(exception: e, message: 'Could not fix geometries during import')
-            job.log "Error fixing geometries during import, skipped (#{e.message})"
-          end
         end
+
+        # If autoguessing is enabled, we try it on arcgis data
+        autoguessing_on_arcgis_import if datasource_name == 'arcgis' && options[:ogr2ogr_csv_guessing]
       rescue StandardError => e
         raise CartoDB::Datasources::InvalidInputDataError.new(e.to_s, ERRORS_MAP[CartoDB::Datasources::InvalidInputDataError]) unless statement_timeout?(e.to_s)
         raise StatementTimeoutError.new(e.to_s, ERRORS_MAP[CartoDB::Importer2::StatementTimeoutError])
       end
 
+      def fix_geometries(job)
+        # At this point the_geom column is renamed
+        GeometryFixer.new(job.db, job.table_name, SCHEMA, 'the_geom', job).run
+      rescue StandardError => e
+        raise e unless statement_timeout?(e.to_s)
+
+        # Ignore timeouts in query batcher
+        log_warning(exception: e, message: 'Could not fix geometries during import')
+        job.log "Error fixing geometries during import, skipped (#{e.message})"
+      end
+
       def normalize
         converted_filepath = normalizers_for(source_file.extension)
           .inject(source_file.fullpath) { |filepath, normalizer_klass|
@@ -395,6 +402,13 @@ module CartoDB
         csv_content[line][column] = "\"#{csv_content[line][column]}\""
         File.open(filepath, 'w') { |file| file.puts csv_content.to_s }
       end
+
+      def autoguessing_on_arcgis_import
+        job.log 'Autoguessing ArcGIS data types...'
+        file = File.read(@source_file.fullpath)
+        file_content = JSON.parse(file)
+        ArcGISAutoguessing.new(job.db, SCHEMA, job.table_name, file_content['fields']).run
+      end
     end
   end
 end

services/importer/lib/importer/runner.rb

@@ -230,7 +230,7 @@ module CartoDB
           loader.streamed_run_continue(downloader.source_file) if got_data
         end while got_data
 
-        loader.streamed_run_finish(@post_import_handler)
+        loader.streamed_run_finish(@post_import_handler, @downloader.datasource.class::DATASOURCE_NAME)
       end
 
       def file_based_loader_run(job, loader)