Merge branch 'create_table_importing_data'

app/controllers/api/json/tables_controller.rb

@@ -252,15 +252,19 @@ class Api::Json::TablesController < ApplicationController
     @table = Table.new
     @table.user_id = current_user.id
     if params[:file]
+      @table.import_from_file = params[:file]
       if $progress[params["X-Progress-ID".to_sym]].nil?
         $progress[params["X-Progress-ID".to_sym]] = 0
       end
     end
+    @table.force_schema = params[:schema] if params[:schema]
     if @table.valid? && @table.save
       render :json => { :id => @table.id }.to_json, :status => 200, :location => table_path(@table)
     else
       render :json => { :errors => @table.errors.full_messages }.to_json, :status => 400
     end
+  rescue => e
+    render :json => { :errors => [translate_error(e.message.split("\n").first)] }.to_json, :status => 400 and return
   end
 
   protected

app/models/table.rb

@@ -12,6 +12,8 @@ class Table < Sequel::Model(:user_tables)
   # Allowed columns
   set_allowed_columns(:name, :privacy, :tags)
 
+  attr_accessor :force_schema, :import_from_file
+
   ## Callbacks
   def validate
     super
@@ -26,25 +28,14 @@ class Table < Sequel::Model(:user_tables)
     super
   end
 
-  def name=(new_name)
-    new_name = set_table_name if new_name.blank?
-    new_name = new_name.sanitize
-    if !new? && !new_name.blank? && !name.blank? && new_name != name
-      owner.in_database do |user_database|
-        user_database.rename_table name, new_name
-      end
-    end
-    self[:name] = new_name unless new_name.blank?
-  end
-
   # Before creating a user table a table should be created in the database.
   # This table has an empty schema
   def before_create
     update_updated_at
     unless self.user_id.blank? || self.name.blank?
-      unless self.name.blank?
-        owner.in_database do |user_database|
-          unless user_database.table_exists?(self.name.to_sym)
+      owner.in_database do |user_database|
+        if !user_database.table_exists?(self.name.to_sym)
+          if force_schema.blank?
             user_database.create_table self.name.to_sym do
               primary_key :id
               String :name
@@ -52,9 +43,19 @@ class Table < Sequel::Model(:user_tables)
               String :description, :text => true
               constraint(:enforce_geotype_location){"(geometrytype(location) = 'POINT'::text OR location IS NULL)"}
             end
+          else
+            sanitized_force_schema = force_schema.split(',').map do |column|
+              if column =~ /^\s*\"([^\"]+)\"(.*)$/
+                "#{$1.sanitize} #{$2}"
+              else
+                column
+              end
+            end
+            user_database.run("CREATE TABLE #{self.name} (#{sanitized_force_schema.join(', ')})")
           end
         end
       end
+      import_data! unless import_from_file.nil?
     end
     super
   end
@@ -98,6 +99,17 @@ class Table < Sequel::Model(:user_tables)
   end
   ## End of Callbacks
 
+  def name=(new_name)
+    new_name = set_table_name if new_name.blank?
+    new_name = new_name.sanitize
+    if !new? && !new_name.blank? && !name.blank? && new_name != name
+      owner.in_database do |user_database|
+        user_database.rename_table name, new_name
+      end
+    end
+    self[:name] = new_name unless new_name.blank?
+  end
+
   def tags=(value)
     self[:tags] = value.split(',').map{ |t| t.strip }.compact.delete_if{ |t| t.blank? }.uniq.join(',')
   end
@@ -235,4 +247,20 @@ class Table < Sequel::Model(:user_tables)
     base_name
   end
 
+  def import_data!
+    return if self.import_from_file.nil?
+    path = if import_from_file.respond_to?(:tempfile)
+      import_from_file.tempfile.path
+    else
+      import_from_file.path
+    end
+    filename = "#{Rails.root}/tmp/importing_csv_#{self.user_id}.csv"
+    system("tail -n `wc -l #{path}` > #{filename}")
+    owner.in_database(:as => :superuser) do |user_database|
+      user_database.run("copy #{self.name} from '#{filename}' WITH CSV")
+    end
+  ensure
+    FileUtils.rm filename
+  end
+
 end

app/models/user.rb

@@ -62,13 +62,18 @@ class User < Sequel::Model
     crypted_password + database_username
   end
 
-  def in_database(&block)
-    connection = ::Sequel.connect(
+  def in_database(options = {}, &block)
+    configuration = if options[:as] && options[:as] == :superuser
+      ::Rails::Sequel.configuration.environment_for(Rails.env).merge(
+        'database' => self.database_name, :logger => ::Rails.logger
+      )
+    else
       ::Rails::Sequel.configuration.environment_for(Rails.env).merge(
         'database' => self.database_name, :logger => ::Rails.logger,
         'username' => database_username, 'password' => database_password
       )
-    )
+    end
+    connection = ::Sequel.connect(configuration)
     result = nil
     begin
       result = yield(connection)

db/fake_data/twitters.csv

@@ -0,0 +1,8 @@
+"URL","login","Country","Followers count",
+"http://twitter.com/vzlaturistica/statuses/23424668752936961","vzlaturistica "," Venezuela ",211,
+"http://twitter.com/CarlsTrendy/statuses/23461167175307264","CarlsTrendy "," Mexico Veracruz ",140,
+"http://twitter.com/imperialtravel1/statuses/23741658398990336","imperialtravel1 "," Medellin Colombia ",51,
+"http://twitter.com/wfsanhueza/statuses/24442490811588608","wfsanhueza "," Concepción ",466,
+"http://twitter.com/AndyPerez7/statuses/24262374739410944","AndyPerez7 "," Ccs, Venezuela ",226,
+"http://twitter.com/AndyPerez7/statuses/24266621946171392","AndyPerez7 "," Ccs, Venezuela ",226,
+"http://twitter.com/joseppique/statuses/24515380554760193","joseppique "," En navegación... ",245,

spec/acceptance/api/tables_spec.rb

@@ -270,7 +270,7 @@ feature "Tables JSON API" do
     Table[table_other.id].should_not be_nil
   end
 
-  scenario "Create a new table" do
+  scenario "Create a new table without schema" do
     user = create_user
 
     authenticate_api user
@@ -288,4 +288,66 @@ feature "Tables JSON API" do
     json_response['id'].should == response.location.match(/\/(\d+)$/)[1].to_i
   end
 
+  scenario "Create a new table specifing a name and a schema" do
+    user = create_user
+
+    authenticate_api user
+
+    post_json "/api/json/tables", {:name => "My new imported table", :schema => "bla bla blat"}
+    response.status.should == 400
+
+    post_json "/api/json/tables", {
+      :name => "My new imported table",
+      :schema => "code char(5) CONSTRAINT firstkey PRIMARY KEY, title  varchar(40) NOT NULL, did  integer NOT NULL, date_prod date, kind varchar(10)"
+    }
+    response.status.should == 200
+    response.location.should =~ /tables\/(\d+)$/
+    json_response = JSON(response.body)
+    json_response['id'].should == response.location.match(/\/(\d+)$/)[1].to_i
+
+    get_json "/api/json/tables/#{response.location.match(/\/(\d+)$/)[1].to_i}/schema"
+    response.status.should == 200
+    json_response = JSON(response.body)
+    json_response.should == [["code", "character(5)"], ["title", "character varying(40)"], ["did", "integer"], ["date_prod", "date"], ["kind", "character varying(10)"]]
+  end
+
+  scenario "Import a file when the schema is wrong" do
+    user = create_user
+
+    authenticate_api user
+
+    post_json "/api/json/tables", {
+                    :name => "Twitts",
+                    :schema => "url varchar(255) not null, login varchar(255), country varchar(255), \"followers count\" integer",
+                    :file => Rack::Test::UploadedFile.new("#{Rails.root}/db/fake_data/twitters.csv", "text/csv")
+               }
+    response.status.should == 400
+  end
+
+  scenario "Create a new table specifing an schema and a file from which import data" do
+    user = create_user
+
+    authenticate_api user
+
+    post_json "/api/json/tables", {
+                    :name => "Twitts",
+                    :schema => "url varchar(255) not null, login varchar(255), country varchar(255), \"followers count\" integer, foo varchar(255)",
+                    :file => Rack::Test::UploadedFile.new("#{Rails.root}/db/fake_data/twitters.csv", "text/csv")
+               }
+    response.status.should == 200
+    response.location.should =~ /tables\/(\d+)$/
+    json_response = JSON(response.body)
+    json_response['id'].should == response.location.match(/\/(\d+)$/)[1].to_i
+
+    get_json "/api/json/tables/#{json_response['id']}?rows_per_page=10"
+    response.status.should == 200
+    json_response = JSON(response.body)
+    json_response['total_rows'].should == 7
+    row0 = json_response['rows'][0].symbolize_keys
+    row0[:url].should == "http://twitter.com/vzlaturistica/statuses/23424668752936961"
+    row0[:login].should == "vzlaturistica "
+    row0[:country].should == " Venezuela "
+    row0[:followers_count].should == 211
+  end
+
 end

spec/models/table_spec.rb

@@ -249,4 +249,37 @@ describe Table do
     Table.count == 0
   end
 
+  it "has a default schema if is not imported from a file" do
+    table = create_table
+    table.schema.should == [[:id, "integer"], [:name, "text"], [:location, "geometry"], [:description, "text"]]
+  end
+
+  it "can be created with a given schema if it is valid" do
+    table = new_table
+    table.force_schema = "code char(5) CONSTRAINT firstkey PRIMARY KEY, title  varchar(40) NOT NULL, did  integer NOT NULL, date_prod date, kind varchar(10)"
+    table.save
+    table.schema.should == [[:code, "character(5)"], [:title, "character varying(40)"], [:did, "integer"], [:date_prod, "date"], [:kind, "character varying(10)"]]
+  end
+
+  it "should sanitize columns from a given schema" do
+    table = new_table
+    table.force_schema = "\"code wadus\" char(5) CONSTRAINT firstkey PRIMARY KEY, title  varchar(40) NOT NULL, did  integer NOT NULL, date_prod date, kind varchar(10)"
+    table.save
+    table.schema.should == [[:code_wadus, "character(5)"], [:title, "character varying(40)"], [:did, "integer"], [:date_prod, "date"], [:kind, "character varying(10)"]]
+  end
+
+  it "should import a CSV if the schema is valid" do
+    table = new_table
+    table.force_schema = "url varchar(255) not null, login varchar(255), country varchar(255), \"followers count\" integer, foo varchar(255)"
+    table.import_from_file = Rack::Test::UploadedFile.new("#{Rails.root}/db/fake_data/twitters.csv", "text/csv")
+    table.save
+
+    table.rows_counted.should == 7
+    row0 = table.to_json[:rows][0]
+    row0[:url].should == "http://twitter.com/vzlaturistica/statuses/23424668752936961"
+    row0[:login].should == "vzlaturistica "
+    row0[:country].should == " Venezuela "
+    row0[:followers_count].should == 211
+  end
+
 end