zhongjin
/
cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

working, needs url redirect

Browse Source
1.0
Andrew W Hill 13 years ago
parent 20fd594dc8
commit 64284644a2
7 changed files with 135 additions and 75 deletions
Show Stats Download Patch File Download Diff File

75
app/controllers/api/json/tables_controller.rb
Unescape View File

@ -46,37 +46,62 @@ class Api::Json::TablesController < Api::ApplicationController
end
def create
@table = Table.new
@data_import = DataImport.new(:user_id => current_user.id)
@data_import.updated_at = Time.now
@data_import.save
@table.user_id = current_user.id
@table.data_import_id = @data_import.id
@table.name = params[:name] if params[:name]# && !params[:table_copy]
@table.import_from_file = params[:file] if params[:file]
@table.import_from_url = params[:url] if params[:url]
@table.import_from_table_copy = params[:table_copy] if params[:table_copy]
@table.import_from_query = params[:from_query] if params[:from_query]
@table.migrate_existing_table = params[:migrate_table] if params[:migrate_table]
@table.importing_SRID = params[:srid] || CartoDB::SRID
@table.force_schema = params[:schema] if params[:schema]
@table.the_geom_type = params[:the_geom_type] if params[:the_geom_type]
if @table.valid? && @table.save
render_jsonp({ :id => @table.id,
:name => @table.name,
:schema => @table.schema }, 200, :location => table_path(@table))
multifiles = ['.bz2','.osm']
if params[:url]
ext = File.extname(params[:url])
elsif params[:file]
ext = File.extname(params[:file])
end
if ext.present? and multifiles.include?(ext)
owner = User.select(:id,:database_name,:crypted_password,:quota_in_bytes,:username, :private_tables_enabled, :table_quota).filter(:id => current_user.id).first
hash_in = ::Rails::Sequel.configuration.environment_for(Rails.env).merge(
"database" => owner.database_name,
:logger => ::Rails.logger,
"username" => owner.database_username,
"password" => owner.database_password,
:import_from_file => params[:file],
:debug => (Rails.env.development?),
:remaining_quota => owner.remaining_quota,
:data_import_id => @data_import.id
).symbolize_keys
importer = CartoDB::Importer.new hash_in
importer = importer.import!
render_jsonp({:tag => importer.tag }, 200, :location => '/dashboard?tag_name=#{importer.tag}')
else
@data_import.reload
CartoDB::Logger.info "Errors on tables#create", @table.errors.full_messages
if @table.data_import_id
render_jsonp({ :description => @table.errors.full_messages ,
:stack => @data_import.log_json,
:code=>@data_import.error_code },
400)
@table = Table.new
@table.user_id = current_user.id
@table.data_import_id = @data_import.id
@table.name = params[:name] if params[:name]# && !params[:table_copy]
@table.import_from_file = params[:file] if params[:file]
@table.import_from_url = params[:url] if params[:url]
@table.import_from_table_copy = params[:table_copy] if params[:table_copy]
@table.import_from_query = params[:from_query] if params[:from_query]
@table.migrate_existing_table = params[:migrate_table] if params[:migrate_table]
@table.importing_SRID = params[:srid] || CartoDB::SRID
@table.force_schema = params[:schema] if params[:schema]
@table.the_geom_type = params[:the_geom_type] if params[:the_geom_type]
if @table.valid? && @table.save
render_jsonp({ :id => @table.id,
:name => @table.name,
:schema => @table.schema }, 200, :location => table_path(@table))
else
render_jsonp({ :description => '', :stack => @table.errors.full_messages, :code=>@data_import.error_code }, 400)
@data_import.reload
CartoDB::Logger.info "Errors on tables#create", @table.errors.full_messages
if @table.data_import_id
render_jsonp({ :description => @table.errors.full_messages ,
:stack => @data_import.log_json,
:code=>@data_import.error_code },
400)
else
render_jsonp({ :description => '', :stack => @table.errors.full_messages, :code=>@data_import.error_code }, 400)
end
end
end
rescue => e

1
app/models/data_import.rb
Unescape View File

@ -100,6 +100,7 @@ class DataImport < Sequel::Model
transition any => :failure
end
end
def after_rollback(*args, &block)
self.save
set_callback(:rollback, :after, *args, &block)

19
app/models/table.rb
Unescape View File

@ -301,6 +301,7 @@ class Table < Sequel::Model(:user_tables)
importer_result_name = import_to_cartodb
@data_import.table_name = importer_result_name
self[:name] = importer_result_name
schema = self.schema(:reload => true)
@ -1292,7 +1293,23 @@ TRIGGER
type = "point"
end
raise InvalidArgument unless CartoDB::VALID_GEOMETRY_TYPES.include?(type.to_s.downcase)
#if the geometry is LINESTRING or POLYGON we convert it to MULTILINESTRING and MULTIPOLYGON resp.
if ["linestring","polygon"].include?(type.to_s.downcase)
owner.in_database do |user_database|
if type.to_s.downcase == 'polygon'
user_database.run("SELECT AddGeometryColumn('#{self.name}','the_geom_simple',4326, 'MULTIPOLYGON', 2);")
else
user_database.run("SELECT AddGeometryColumn('#{self.name}','the_geom_simple',4326, 'MULTILINESTRING', 2);")
end
user_database.run("UPDATE #{self.name} SET the_geom_simple = ST_Multi(the_geom);")
user_database.run("SELECT DropGeometryColumn('#{self.name}','the_geom');");
user_database.run("ALTER TABLE #{self.name} RENAME COLUMN the_geom_simple TO the_geom;")
type = owner.in_database["select GeometryType(#{THE_GEOM}) FROM #{self.name} where #{THE_GEOM} is not null limit 1"].first[:geometrytype]
end
end
raise "Error: unsupported geometry type #{type.to_s.downcase} in CartoDB" unless CartoDB::VALID_GEOMETRY_TYPES.include?(type.to_s.downcase)
#raise InvalidArgument unless CartoDB::VALID_GEOMETRY_TYPES.include?(type.to_s.downcase)
updates = false
type = type.to_s.upcase
owner.in_database do |user_database|

4
db/seeds.rb
Unescape View File

@ -35,7 +35,7 @@ if Rails.env.development?
table = Table.new :privacy => Table::PUBLIC, :name => 'Foursq check-ins',
:tags => '4sq, personal'
table.user_id = user.id
table.force_schema = "name varchar, surname varchar, address varchar, city varchar, country varchar, nif varchar, age integer, twitter_account varchar, postal_code integer"
table.force_schema = "name text, surname text, address text, city text, country text, nif text, age integer, twitter_account text, postal_code integer"
table.save
user.in_database do |user_database|
@ -47,7 +47,7 @@ if Rails.env.development?
table = Table.new :privacy => Table::PRIVATE, :name => 'Madrid Bars',
:tags => 'movies, personal'
table.user_id = user.id
table.force_schema = "name varchar, address varchar, latitude float, longitude float"
table.force_schema = "name text, address text, latitude float, longitude float"
table.save
table.insert_row!({:name => "Hawai", :address => "Calle de Pérez Galdós 9, Madrid, Spain", :latitude => 40.423012, :longitude => -3.699732})
table.insert_row!({:name => "El Estocolmo", :address => "Calle de la Palma 72, Madrid, Spain", :latitude => 40.426949, :longitude => -3.708969})

89
lib/importer/lib/cartodb-importer/loaders/osm.rb
Unescape View File

@ -42,58 +42,77 @@ module CartoDB
valid_tables = Array.new
["line", "polygon", "roads", "point"].each do |feature|
@old_table_name = "#{random_table_prefix}_#{feature}"
rows_imported = @db_connection["SELECT count(*) as count from #{@old_table_name}"].first[:count]
if !rows_imported.nil? and 0 < rows_imported
old_table_name = "#{random_table_prefix}_#{feature}"
rows_imported = @db_connection["SELECT count(*) as count from #{old_table_name}"].first[:count]
unless rows_imported.nil? || rows_imported == 0
valid_tables << feature
else
@db_connection.drop_table @old_table_name
end
end
import_tag = "#{@suggested_name}_#{Time.now.to_i}"
import_tables = Array.new
valid_tables.each do |feature|
@old_table_name = "#{random_table_prefix}_#{feature}"
@table_name = get_valid_name("#{@suggested_name}_#{feature}")
begin
begin
@db_connection.run("ALTER TABLE \"#{@old_table_name}\" RENAME TO \"#{@table_name}\"")
@table_created = true
rescue Exception => msg
@runlog.err << msg
@data_import.set_error_code(5000)
@data_import.log_error(msg)
@data_import.log_error("ERROR: unable to rename \"#{@old_table_name}\" to \"#{@table_name}\"")
@db_connection.drop_table @old_table_name
end
@entries.each{ |e| FileUtils.rm_rf(e) } if @entries.any?
@rows_imported = @db_connection["SELECT count(*) as count from #{@table_name}"].first[:count]
osm_geom_name = "way"
@db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN #{osm_geom_name} TO the_geom")
@db_connection.run("ALTER TABLE \"#{@old_table_name}\" RENAME TO \"#{@table_name}\"")
@table_created = true
#begin
@entries.each{ |e| FileUtils.rm_rf(e) } if @entries.any?
# Sanitize column names where needed
column_names = @db_connection.schema(@table_name).map{ |s| s[0].to_s }
need_sanitizing = column_names.each do |column_name|
if column_name != column_name.sanitize_column_name
@db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN \"#{column_name}\" TO #{column_name.sanitize_column_name}")
osm_geom_name = "way"
geoms = @db_connection["SELECT count(*) as count from #{@table_name}"].first[:count]
unless geoms.nil? || geoms == 0
@db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN #{osm_geom_name} TO the_geom")
end
end
import_tables << @table_name
rescue
# Sanitize column names where needed
column_names = @db_connection.schema(@table_name).map{ |s| s[0].to_s }
need_sanitizing = column_names.each do |column_name|
if column_name != column_name.sanitize_column_name
@db_connection.run("ALTER TABLE #{@table_name} RENAME COLUMN \"#{column_name}\" TO #{column_name.sanitize_column_name}")
end
end
@rows_imported = @db_connection["SELECT count(*) as count from #{@table_name}"].first[:count]
# import_tables << @table_name
# @last_table = @table_name
@new_table = Table.new :tags => "#{import_tag}"
@di = DataImport.new(:user_id => @data_import.user_id)
@di.updated_at = Time.now
@di.save
@new_table.user_id = @data_import.user_id
@new_table.data_import_id = @di.id
@new_table.name = @table_name
@new_table.migrate_existing_table = @table_name
if @new_table.valid?
@new_table.save
end
# rescue
# @data_import.set_error_code(5000)
# @data_import.log_error("ERROR: unable to format table \"#{@table_name}\" for CartoDB")
# @db_connection.drop_table @old_table_name
# end
rescue Exception => msg
@runlog.err << msg
@data_import.set_error_code(5000)
@data_import.log_error(msg)
@data_import.log_error("ERROR: unable to rename \"#{@old_table_name}\" to \"#{@table_name}\"")
begin
@db_connection.drop_table @old_table_name
end
rescue
@data_import.log_error("ERROR: \"#{@old_table_name}\" doesn't exist")
end
end
end
@import_from_file.unlink
payload = OpenStruct.new({
:name => @table_name,
:rows_imported => @rows_imported,
:import_type => @import_type,
:import_tables => import_tables,
:log => @runlog
:tag => "#{import_tag}"
})
# construct return variables

12
lib/importer/lib/cartodb-importer/loaders/shp.rb
Unescape View File

@ -49,18 +49,6 @@ module CartoDB
unless (reg = stdout.read).empty?
@runlog.stdout << reg
end
# out = `#{full_shp_command}`
#
# if $?.exitstatus != 0
# @data_import.log_error("failed to convert shp to sql")
# raise "failed to convert shp to sql"
# end
#
# if 0 < out.strip.length
# @data_import.log_update(out)
# @runlog.stdout << out
# end
# TODO: THIS SHOULD BE UPDATE IF NOT NULL TO PREVENT CRASHING
if shp_args_command[0] != '4326'

10
spec/models/table_spec.rb
Unescape View File

@ -1089,6 +1089,16 @@ describe Table do
table.name.should == "esp_adm1"
end
end
context "osm standard tests" do
it "should import guinea.osm.bz2" do
table = new_table :name => nil
#table.import_from_file = "#{Rails.root}/db/fake_data/EjemploVizzuality.zip"
table.import_from_file = "#{Rails.root}/db/fake_data/guinea.osm.bz2"
table.save
table.rows_counted.should == 308
table.name.should == "vizzuality"
end
end
context "import exceptions tests" do
it "should import reserved_names.csv" do
user = create_user

Write Preview
Loading…
Cancel
Save