diff --git a/app/controllers/api/json/tables_controller.rb b/app/controllers/api/json/tables_controller.rb index e763ef0286..2f1cc4d005 100644 --- a/app/controllers/api/json/tables_controller.rb +++ b/app/controllers/api/json/tables_controller.rb @@ -46,37 +46,62 @@ class Api::Json::TablesController < Api::ApplicationController end def create - @table = Table.new - @data_import = DataImport.new(:user_id => current_user.id) @data_import.updated_at = Time.now @data_import.save - @table.user_id = current_user.id - @table.data_import_id = @data_import.id - @table.name = params[:name] if params[:name]# && !params[:table_copy] - @table.import_from_file = params[:file] if params[:file] - @table.import_from_url = params[:url] if params[:url] - @table.import_from_table_copy = params[:table_copy] if params[:table_copy] - @table.import_from_query = params[:from_query] if params[:from_query] - @table.migrate_existing_table = params[:migrate_table] if params[:migrate_table] - @table.importing_SRID = params[:srid] || CartoDB::SRID - @table.force_schema = params[:schema] if params[:schema] - @table.the_geom_type = params[:the_geom_type] if params[:the_geom_type] - if @table.valid? && @table.save - render_jsonp({ :id => @table.id, - :name => @table.name, - :schema => @table.schema }, 200, :location => table_path(@table)) + multifiles = ['.bz2','.osm'] + if params[:url] + ext = File.extname(params[:url]) + elsif params[:file] + ext = File.extname(params[:file]) + end + + if ext.present? and multifiles.include?(ext) + owner = User.select(:id,:database_name,:crypted_password,:quota_in_bytes,:username, :private_tables_enabled, :table_quota).filter(:id => current_user.id).first + hash_in = ::Rails::Sequel.configuration.environment_for(Rails.env).merge( + "database" => owner.database_name, + :logger => ::Rails.logger, + "username" => owner.database_username, + "password" => owner.database_password, + :import_from_file => params[:file], + :debug => (Rails.env.development?), + :remaining_quota => owner.remaining_quota, + :data_import_id => @data_import.id + ).symbolize_keys + + importer = CartoDB::Importer.new hash_in + importer = importer.import! + render_jsonp({:tag => importer.tag }, 200, :location => '/dashboard?tag_name=#{importer.tag}') else - @data_import.reload - CartoDB::Logger.info "Errors on tables#create", @table.errors.full_messages - if @table.data_import_id - render_jsonp({ :description => @table.errors.full_messages , - :stack => @data_import.log_json, - :code=>@data_import.error_code }, - 400) + @table = Table.new + @table.user_id = current_user.id + @table.data_import_id = @data_import.id + @table.name = params[:name] if params[:name]# && !params[:table_copy] + @table.import_from_file = params[:file] if params[:file] + @table.import_from_url = params[:url] if params[:url] + @table.import_from_table_copy = params[:table_copy] if params[:table_copy] + @table.import_from_query = params[:from_query] if params[:from_query] + @table.migrate_existing_table = params[:migrate_table] if params[:migrate_table] + @table.importing_SRID = params[:srid] || CartoDB::SRID + @table.force_schema = params[:schema] if params[:schema] + @table.the_geom_type = params[:the_geom_type] if params[:the_geom_type] + + if @table.valid? && @table.save + render_jsonp({ :id => @table.id, + :name => @table.name, + :schema => @table.schema }, 200, :location => table_path(@table)) else - render_jsonp({ :description => '', :stack => @table.errors.full_messages, :code=>@data_import.error_code }, 400) + @data_import.reload + CartoDB::Logger.info "Errors on tables#create", @table.errors.full_messages + if @table.data_import_id + render_jsonp({ :description => @table.errors.full_messages , + :stack => @data_import.log_json, + :code=>@data_import.error_code }, + 400) + else + render_jsonp({ :description => '', :stack => @table.errors.full_messages, :code=>@data_import.error_code }, 400) + end end end rescue => e diff --git a/app/models/data_import.rb b/app/models/data_import.rb index d776e1933c..056f823b4e 100644 --- a/app/models/data_import.rb +++ b/app/models/data_import.rb @@ -100,6 +100,7 @@ class DataImport < Sequel::Model transition any => :failure end end + def after_rollback(*args, &block) self.save set_callback(:rollback, :after, *args, &block) diff --git a/app/models/table.rb b/app/models/table.rb index 3165e7ce34..23bd4c8a09 100644 --- a/app/models/table.rb +++ b/app/models/table.rb @@ -301,6 +301,7 @@ class Table < Sequel::Model(:user_tables) importer_result_name = import_to_cartodb @data_import.table_name = importer_result_name + self[:name] = importer_result_name schema = self.schema(:reload => true) @@ -1292,7 +1293,23 @@ TRIGGER type = "point" end - raise InvalidArgument unless CartoDB::VALID_GEOMETRY_TYPES.include?(type.to_s.downcase) + #if the geometry is LINESTRING or POLYGON we convert it to MULTILINESTRING and MULTIPOLYGON resp. + if ["linestring","polygon"].include?(type.to_s.downcase) + owner.in_database do |user_database| + if type.to_s.downcase == 'polygon' + user_database.run("SELECT AddGeometryColumn('#{self.name}','the_geom_simple',4326, 'MULTIPOLYGON', 2);") + else + user_database.run("SELECT AddGeometryColumn('#{self.name}','the_geom_simple',4326, 'MULTILINESTRING', 2);") + end + user_database.run("UPDATE #{self.name} SET the_geom_simple = ST_Multi(the_geom);") + user_database.run("SELECT DropGeometryColumn('#{self.name}','the_geom');"); + user_database.run("ALTER TABLE #{self.name} RENAME COLUMN the_geom_simple TO the_geom;") + type = owner.in_database["select GeometryType(#{THE_GEOM}) FROM #{self.name} where #{THE_GEOM} is not null limit 1"].first[:geometrytype] + end + end + + raise "Error: unsupported geometry type #{type.to_s.downcase} in CartoDB" unless CartoDB::VALID_GEOMETRY_TYPES.include?(type.to_s.downcase) + #raise InvalidArgument unless CartoDB::VALID_GEOMETRY_TYPES.include?(type.to_s.downcase) updates = false type = type.to_s.upcase owner.in_database do |user_database| diff --git a/db/seeds.rb b/db/seeds.rb index 5a0c56d01f..60aeb6e0a5 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -35,7 +35,7 @@ if Rails.env.development? table = Table.new :privacy => Table::PUBLIC, :name => 'Foursq check-ins', :tags => '4sq, personal' table.user_id = user.id - table.force_schema = "name varchar, surname varchar, address varchar, city varchar, country varchar, nif varchar, age integer, twitter_account varchar, postal_code integer" + table.force_schema = "name text, surname text, address text, city text, country text, nif text, age integer, twitter_account text, postal_code integer" table.save user.in_database do |user_database| @@ -47,7 +47,7 @@ if Rails.env.development? table = Table.new :privacy => Table::PRIVATE, :name => 'Madrid Bars', :tags => 'movies, personal' table.user_id = user.id - table.force_schema = "name varchar, address varchar, latitude float, longitude float" + table.force_schema = "name text, address text, latitude float, longitude float" table.save table.insert_row!({:name => "Hawai", :address => "Calle de Pérez Galdós 9, Madrid, Spain", :latitude => 40.423012, :longitude => -3.699732}) table.insert_row!({:name => "El Estocolmo", :address => "Calle de la Palma 72, Madrid, Spain", :latitude => 40.426949, :longitude => -3.708969}) diff --git a/lib/importer/lib/cartodb-importer/loaders/osm.rb b/lib/importer/lib/cartodb-importer/loaders/osm.rb index 171ed0a898..5e014f125d 100644 --- a/lib/importer/lib/cartodb-importer/loaders/osm.rb +++ b/lib/importer/lib/cartodb-importer/loaders/osm.rb @@ -42,58 +42,77 @@ module CartoDB valid_tables = Array.new ["line", "polygon", "roads", "point"].each do |feature| - @old_table_name = "#{random_table_prefix}_#{feature}" - rows_imported = @db_connection["SELECT count(*) as count from #{@old_table_name}"].first[:count] - if !rows_imported.nil? and 0 < rows_imported + old_table_name = "#{random_table_prefix}_#{feature}" + rows_imported = @db_connection["SELECT count(*) as count from #{old_table_name}"].first[:count] + unless rows_imported.nil? || rows_imported == 0 valid_tables << feature else @db_connection.drop_table @old_table_name end end + import_tag = "#{@suggested_name}_#{Time.now.to_i}" import_tables = Array.new valid_tables.each do |feature| @old_table_name = "#{random_table_prefix}_#{feature}" @table_name = get_valid_name("#{@suggested_name}_#{feature}") begin - begin - @db_connection.run("ALTER TABLE \"#{@old_table_name}\" RENAME TO \"#{@table_name}\"") - @table_created = true - rescue Exception => msg - @runlog.err << msg - @data_import.set_error_code(5000) - @data_import.log_error(msg) - @data_import.log_error("ERROR: unable to rename \"#{@old_table_name}\" to \"#{@table_name}\"") - @db_connection.drop_table @old_table_name - end - - @entries.each{ |e| FileUtils.rm_rf(e) } if @entries.any? - @rows_imported = @db_connection["SELECT count(*) as count from #{@table_name}"].first[:count] - - osm_geom_name = "way" - @db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN #{osm_geom_name} TO the_geom") + @db_connection.run("ALTER TABLE \"#{@old_table_name}\" RENAME TO \"#{@table_name}\"") + @table_created = true + #begin + @entries.each{ |e| FileUtils.rm_rf(e) } if @entries.any? - # Sanitize column names where needed - column_names = @db_connection.schema(@table_name).map{ |s| s[0].to_s } - need_sanitizing = column_names.each do |column_name| - if column_name != column_name.sanitize_column_name - @db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN \"#{column_name}\" TO #{column_name.sanitize_column_name}") + osm_geom_name = "way" + geoms = @db_connection["SELECT count(*) as count from #{@table_name}"].first[:count] + unless geoms.nil? || geoms == 0 + @db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN #{osm_geom_name} TO the_geom") end - end - import_tables << @table_name - rescue + + # Sanitize column names where needed + column_names = @db_connection.schema(@table_name).map{ |s| s[0].to_s } + need_sanitizing = column_names.each do |column_name| + if column_name != column_name.sanitize_column_name + @db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN \"#{column_name}\" TO #{column_name.sanitize_column_name}") + end + end + + @rows_imported = @db_connection["SELECT count(*) as count from #{@table_name}"].first[:count] + + # import_tables << @table_name + # @last_table = @table_name + + @new_table = Table.new :tags => "#{import_tag}" + @di = DataImport.new(:user_id => @data_import.user_id) + @di.updated_at = Time.now + @di.save + @new_table.user_id = @data_import.user_id + @new_table.data_import_id = @di.id + @new_table.name = @table_name + @new_table.migrate_existing_table = @table_name + if @new_table.valid? + @new_table.save + end + + # rescue + # @data_import.set_error_code(5000) + # @data_import.log_error("ERROR: unable to format table \"#{@table_name}\" for CartoDB") + # @db_connection.drop_table @old_table_name + # end + rescue Exception => msg + @runlog.err << msg + @data_import.set_error_code(5000) + @data_import.log_error(msg) + @data_import.log_error("ERROR: unable to rename \"#{@old_table_name}\" to \"#{@table_name}\"") + begin @db_connection.drop_table @old_table_name - end + rescue + @data_import.log_error("ERROR: \"#{@old_table_name}\" doesn't exist") + end + end end - @import_from_file.unlink - payload = OpenStruct.new({ - :name => @table_name, - :rows_imported => @rows_imported, - :import_type => @import_type, - :import_tables => import_tables, - :log => @runlog + :tag => "#{import_tag}" }) # construct return variables diff --git a/lib/importer/lib/cartodb-importer/loaders/shp.rb b/lib/importer/lib/cartodb-importer/loaders/shp.rb index 61a0d568dc..57fb5f10fe 100644 --- a/lib/importer/lib/cartodb-importer/loaders/shp.rb +++ b/lib/importer/lib/cartodb-importer/loaders/shp.rb @@ -49,18 +49,6 @@ module CartoDB unless (reg = stdout.read).empty? @runlog.stdout << reg end - - # out = `#{full_shp_command}` - # - # if $?.exitstatus != 0 - # @data_import.log_error("failed to convert shp to sql") - # raise "failed to convert shp to sql" - # end - # - # if 0 < out.strip.length - # @data_import.log_update(out) - # @runlog.stdout << out - # end # TODO: THIS SHOULD BE UPDATE IF NOT NULL TO PREVENT CRASHING if shp_args_command[0] != '4326' diff --git a/spec/models/table_spec.rb b/spec/models/table_spec.rb index dcaa007f45..ffbefa4e78 100644 --- a/spec/models/table_spec.rb +++ b/spec/models/table_spec.rb @@ -1089,6 +1089,16 @@ describe Table do table.name.should == "esp_adm1" end end + context "osm standard tests" do + it "should import guinea.osm.bz2" do + table = new_table :name => nil + #table.import_from_file = "#{Rails.root}/db/fake_data/EjemploVizzuality.zip" + table.import_from_file = "#{Rails.root}/db/fake_data/guinea.osm.bz2" + table.save + table.rows_counted.should == 308 + table.name.should == "vizzuality" + end + end context "import exceptions tests" do it "should import reserved_names.csv" do user = create_user