Create a table providing a external url with JSON format

app/models/table.rb

@@ -12,7 +12,7 @@ class Table < Sequel::Model(:user_tables)
   # Allowed columns
   set_allowed_columns(:name, :privacy, :tags)
 
-  attr_accessor :force_schema, :import_from_file
+  attr_accessor :force_schema, :import_from_file, :import_from_external_url
 
   CARTODB_COLUMNS = %W{ cartodb_id created_at updated_at }
 
@@ -34,6 +34,9 @@ class Table < Sequel::Model(:user_tables)
   # This table has an empty schema
   def before_create
     update_updated_at
+    unless import_from_external_url.blank?
+      import_data_from_external_url!
+    end
     guess_schema if force_schema.blank? && !import_from_file.blank?
     unless self.user_id.blank? || self.name.blank?
       owner.in_database do |user_database|
@@ -72,7 +75,9 @@ class Table < Sequel::Model(:user_tables)
           end
         end
       end
-      import_data! unless import_from_file.nil?
+      unless import_from_file.blank?
+        import_data_from_file!
+      end
       set_triggers
     end
     super
@@ -270,8 +275,35 @@ class Table < Sequel::Model(:user_tables)
     base_name
   end
 
-  def import_data!
-    return if self.import_from_file.nil?
+  def import_data_from_external_url!
+    return if self.import_from_external_url.blank?
+    url = URI.parse(self.import_from_external_url)
+    req = Net::HTTP::Get.new(url.path)
+    res = Net::HTTP.start(url.host, url.port){ |http| http.request(req) }
+    json = JSON.parse(res.body)
+    columns = []
+    filepath = "#{Rails.root}/tmp/importing_#{user_id}.csv"
+    if json.is_a?(Array)
+      raise "Invalid JSON format" unless json.first.is_a?(Hash)
+      CSV.open(filepath, "wb") do |csv|
+        csv << json.first.keys
+        json.each do |row|
+          csv << row.values
+        end
+      end
+    elsif json.is_a?(Hash)
+      CSV.open(filepath, "wb") do |csv|
+        csv << json.keys
+        csv << json.values
+      end
+    else
+      raise "Invalid JSON format"
+    end
+    self.import_from_file = File.open(filepath,'r')
+  end
+
+  def import_data_from_file!
+    return if self.import_from_file.blank?
     path = if import_from_file.respond_to?(:tempfile)
       import_from_file.tempfile.path
     else
@@ -294,6 +326,7 @@ class Table < Sequel::Model(:user_tables)
     end
   ensure
     FileUtils.rm filename
+    FileUtils.rm path
   end
 
   def guess_schema

spec/models/table_spec.rb

@@ -404,4 +404,26 @@ describe Table do
     row[:views].should == 540
   end
 
+  it "should import data from an external url returning JSON data" do
+    json = JSON.parse(File.read("#{Rails.root}/spec/support/bus_gijon.json"))
+    JSON.stubs(:parse).returns(json)
+    table = new_table
+    table.import_from_external_url = "http://externaldata.com/bus_gijon.json"
+    table.save
+
+    table.rows_counted.should == 7
+    table.schema.should == [[:cartodb_id, "integer"], [:idautobus, "integer"], [:utmx, "integer"],
+        [:utmy, "integer"], [:horaactualizacion, "character varying"], [:fechaactualizacion, "character varying"],
+        [:idtrayecto, "integer"], [:idparada, "integer"], [:minutos, "integer"], [:distancia, "double precision"],
+        [:idlinea, "integer"], [:matricula, "character varying"], [:modelo, "character varying"],
+        [:ordenparada, "integer"], [:idsiguienteparada, "integer"], [:created_at, "timestamp"],
+        [:updated_at, "timestamp"]
+    ]
+    row = table.to_json[:rows][0]
+    row[:cartodb_id].should == 1
+    row[:idautobus].should == 330
+    row[:horaactualizacion].should == "14:23:10"
+    row[:idparada] == 34
+  end
+
 end

spec/support/bus_gijon.json

@@ -0,0 +1,106 @@
+[ {
+  "idautobus" : "330",
+  "utmx" : 284858,
+  "utmy" : 4824178,
+  "horaactualizacion" : "14:23:10",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 1,
+  "idparada" : 34,
+  "minutos" : 0,
+  "distancia" : 299.65238,
+  "idlinea" : 1,
+  "matricula" : "3647 FYR",
+  "modelo" : "SCANIA N310UA6X2/2",
+  "ordenparada" : 30,
+  "idsiguienteparada" : 18
+}, {
+  "idautobus" : "331",
+  "utmx" : 283144,
+  "utmy" : 4824370,
+  "horaactualizacion" : "14:23:11",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 1,
+  "idparada" : 34,
+  "minutos" : 0,
+  "distancia" : 64.760647,
+  "idlinea" : 1,
+  "matricula" : "3755 FYR",
+  "modelo" : "SCANIA N310UA6X2/2",
+  "ordenparada" : 30,
+  "idsiguienteparada" : 138
+}, {
+  "idautobus" : "333",
+  "utmx" : 286668,
+  "utmy" : 4823706,
+  "horaactualizacion" : "14:23:11",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 1,
+  "idparada" : 34,
+  "minutos" : 0,
+  "distancia" : 253.077583,
+  "idlinea" : 1,
+  "matricula" : "3966 FYR",
+  "modelo" : "SCANIA N310UA6X2/2",
+  "ordenparada" : 30,
+  "idsiguienteparada" : 85
+}, {
+  "idautobus" : "266",
+  "utmx" : 286624,
+  "utmy" : 4823821,
+  "horaactualizacion" : "14:23:21",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 2,
+  "idparada" : 94,
+  "minutos" : 0,
+  "distancia" : 7.134992,
+  "idlinea" : 1,
+  "matricula" : "O-6.309-CB",
+  "modelo" : "NO94UB4X2",
+  "ordenparada" : 8,
+  "idsiguienteparada" : 94
+}, {
+  "idautobus" : "335",
+  "utmx" : 285066,
+  "utmy" : 4824089,
+  "horaactualizacion" : "14:23:20",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 2,
+  "idparada" : 180,
+  "minutos" : 0,
+  "distancia" : 367.787473,
+  "idlinea" : 1,
+  "matricula" : "1892-GKH",
+  "modelo" : "SCANIA N310UA6X2/2",
+  "ordenparada" : 30,
+  "idsiguienteparada" : 131
+}, {
+  "idautobus" : "336",
+  "utmx" : 281894,
+  "utmy" : 4824528,
+  "horaactualizacion" : "14:23:12",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 2,
+  "idparada" : 180,
+  "minutos" : 0,
+  "distancia" : 312.467394,
+  "idlinea" : 1,
+  "matricula" : "2076-GKH",
+  "modelo" : "SCANIA N310UA6X2/2",
+  "ordenparada" : 30,
+  "idsiguienteparada" : 604
+}, {
+  "idautobus" : "334",
+  "utmx" : 289310,
+  "utmy" : 4822603,
+  "horaactualizacion" : "14:23:13",
+  "fechaactualizacion" : "2011-02-15T00:00:00",
+  "idtrayecto" : 2,
+  "idparada" : 34,
+  "minutos" : 0,
+  "distancia" : 128.32723,
+  "idlinea" : 1,
+  "matricula" : "4095 FYR",
+  "modelo" : "SCANIA N310UA6X2/2",
+  "ordenparada" : 0,
+  "idsiguienteparada" : 35
+} ]