Merge branch 'master' into 903-remove-cumulative-for-torque-cat

Conflicts:
	README.md

README.md

@@ -612,5 +612,5 @@ See TESTING.md
   - Diego Muñoz ([kartones](https://twitter.com/kartones))
   - Raul Ochoa ([rochoa](https://twitter.com/rochoa))
   - Nicolás M. Jaremek ([NickJaremek](https://twitter.com/NickJaremek))
+  - Jaime Chapinal ([Xatpy](https://twitter.com/chapi13))
   - Nicklas Gummesson ([ViddoBamBam](https://twitter.com/ViddoBamBam))
-

app/connectors/importer.rb

@@ -50,7 +50,6 @@ module CartoDB
         runner.log.append("Before persisting metadata '#{name}' data_import_id: #{data_import_id}")
         persist_metadata(name, data_import_id)
         runner.log.append("Table '#{name}' registered")
-      rescue
       end
 
       def success?
@@ -91,16 +90,22 @@ module CartoDB
 
         rename_the_geom_index_if_exists(current_name, new_name)
         new_name
-      rescue
-        retry unless rename_attempts > MAX_RENAME_RETRIES
+      rescue => exception
+        runner.log.append("Silently retrying renaming #{current_name} to #{new_name}. ")
+        if rename_attempts <= MAX_RENAME_RETRIES
+          retry
+        else
+          raise CartoDB::Importer2::InvalidNameError.new
+        end
       end
 
       def rename_the_geom_index_if_exists(current_name, new_name)
         database.execute(%Q{
-          ALTER INDEX "#{ORIGIN_SCHEMA}"."#{current_name}_geom_idx"
+          ALTER INDEX IF EXISTS "#{ORIGIN_SCHEMA}"."#{current_name}_geom_idx"
           RENAME TO "the_geom_#{UUIDTools::UUID.timestamp_create.to_s.gsub('-', '_')}"
         })
-      rescue
+      rescue => exception
+        runner.log.append("Silently failed rename_the_geom_index_if_exists from #{current_name} to #{new_name} with exception #{exception}. Backtrace: #{exception.backtrace.to_s}. ")
       end
 
       def persist_metadata(name, data_import_id)

app/controllers/application_controller.rb

@@ -13,6 +13,7 @@ class ApplicationController < ActionController::Base
   before_filter :allow_cross_domain_access
   before_filter :set_asset_debugging
   after_filter  :remove_flash_cookie
+  after_filter  :add_revision_header
 
   rescue_from NoHTML5Compliant, :with => :no_html5_compliant
   rescue_from RecordNotFound,   :with => :render_404
@@ -168,6 +169,10 @@ class ApplicationController < ActionController::Base
     end
   end
 
+  def add_revision_header
+    response.headers['X-CartoDB-Rev'] = CartoDB::CARTODB_REV unless CartoDB::CARTODB_REV.nil?
+  end
+
   def current_user
     super(CartoDB.extract_subdomain(request))
   end

app/models/data_import.rb

@@ -297,7 +297,7 @@ class DataImport < Sequel::Model
 
     candidates =  current_user.tables.select_map(:name)
     table_name = ::Table.get_valid_table_name(name, {
-        name_candidates: candidates,
+        connection: current_user.in_database,
         database_schema: current_user.database_schema
     })
     current_user.in_database.run(%Q{CREATE TABLE #{table_name} AS #{query}})

app/models/map.rb

@@ -82,7 +82,7 @@ class Map < Sequel::Model
       view_bounds_sw: "[#{result[:miny]}, #{result[:minx]}]"
     )
   rescue Sequel::DatabaseError => exception
-    notify_airbrake(exception)
+    CartoDB::notify_exception(exception, { user: user } )
   end
 
   def viz_updated_at

app/models/table.rb

@@ -489,7 +489,7 @@ class Table < Sequel::Model(:user_tables)
         @data_import  = DataImport.find(:id=>self.data_import_id)
       end
 
-      importer_result_name = import_to_cartodb
+      importer_result_name = import_to_cartodb(name)
 
       @data_import.reload
       @data_import.table_name = importer_result_name
@@ -1498,6 +1498,7 @@ class Table < Sequel::Model(:user_tables)
     name_candidates = self.owner.tables.select_map(:name) if owner
 
     options.merge!(name_candidates: name_candidates)
+    options.merge!(connection: self.owner.connection) unless self.owner.nil?
     unless options[:database_schema].present? || self.owner.nil?
       options.merge!(database_schema: self.owner.database_schema)
     end
@@ -1526,8 +1527,9 @@ class Table < Sequel::Model(:user_tables)
     database_schema = options[:database_schema].present? ? options[:database_schema] : 'public'
 
     # We don't want to use an existing table name
-    existing_names = options[:name_candidates] || \
-      options[:connection]["select relname from pg_stat_user_tables WHERE schemaname='#{database_schema}'"].map(:relname)
+    # 
+    existing_names = []
+    existing_names = options[:name_candidates] || options[:connection]["select relname from pg_stat_user_tables WHERE schemaname='#{database_schema}'"].map(:relname) if options[:connection]
     existing_names = existing_names + User::SYSTEM_TABLE_NAMES
     rx = /_(\d+)$/
     count = name[rx][1].to_i rescue 0

app/models/table_registrar.rb

@@ -10,7 +10,8 @@ module CartoDB
     def register(table_name, data_import_id)
       self.table                    = table_klass.new
       table.user_id                 = user.id
-      table.name                    = table_name
+      # INFO: we're not creating but registering an existent table, so we want fixed, known name
+      table.instance_eval { self[:name] = table_name }
       table.migrate_existing_table  = table_name
       table.data_import_id          = data_import_id
       table.save
@@ -24,7 +25,7 @@ module CartoDB
 
     def get_valid_table_name(table_name)
       table_klass.get_valid_table_name(table_name, {
-          name_candidates: user.reload.tables.map(&:name),
+          connection: user.in_database,
           database_schema: user.database_schema
         })
     end

app/models/user.rb

@@ -400,9 +400,7 @@ class User < Sequel::Model
     configuration = get_db_configuration_for(options[:as])
 
     connection = $pool.fetch(configuration) do
-      db = ::Sequel.connect(configuration.merge(:after_connect=>(proc do |conn|
-        conn.execute(%Q{ SET search_path TO "#{self.database_schema}", cartodb, public }) unless options[:as] == :cluster_admin
-      end)))
+      db = get_database(options, configuration)
       db.extension(:connection_validator)
       db.pool.connection_validation_timeout = configuration.fetch('conn_validator_timeout', -1)
       db
@@ -415,6 +413,20 @@ class User < Sequel::Model
     end
   end
 
+  def connection(options = {})
+    configuration = get_db_configuration_for(options[:as])
+
+    $pool.fetch(configuration) do
+      get_database(options, configuration)
+    end
+  end
+
+  def get_database(options, configuration)
+      ::Sequel.connect(configuration.merge(:after_connect=>(proc do |conn|
+        conn.execute(%Q{ SET search_path TO "#{self.database_schema}", cartodb, public }) unless options[:as] == :cluster_admin
+      end)))
+  end
+
   def get_db_configuration_for(user = nil)
     logger = (Rails.env.development? || Rails.env.test? ? ::Rails.logger : nil)
     if user == :superuser

app/views/admin/users/_form.html.erb

@@ -76,12 +76,14 @@
                 </div>
               </div>
 
-            <div class="row margin-bottom-10">
-              <div class="field">
-                <label>JOBS PROFILE</label>
-                <label class="simple-label" for="user_available_for_hire"><%= f.check_box :available_for_hire %> Available for hire</label>
+              <!-- 
+              <div class="row margin-bottom-10">
+                <div class="field">
+                  <label>JOBS PROFILE</label>
+                  <label class="simple-label" for="user_available_for_hire"><%= f.check_box :available_for_hire %> Available for hire</label>
+                </div>
               </div>
-            </div>
+              -->
 
 
             </div>
@@ -95,9 +97,11 @@
               <div class="row">
                 <p class="margin35">Moderate the comments in your maps using your Disqus shortname. <a href="#/how-to" class="disqus_help">Here's how</a>.</p>
               </div>
+              <!--
             <div class="row low-padding">
               <p>Show a banner in your public profile linked to your email.</p>
             </div>
+             -->
 
             </div>
 

config/initializers/carto_db.rb

@@ -95,6 +95,12 @@ module CartoDB
     VERSION_1 = "v1"
   end
 
+  begin
+    CARTODB_REV = File.read("#{Rails.root}/REVISION").strip
+  rescue
+    CARTODB_REV = nil
+  end
+
   PUBLIC_DB_USER  = 'publicuser'
   PUBLIC_DB_USER_PASSWORD  = 'publicuser'
   TILE_DB_USER    = 'tileuser'
@@ -196,6 +202,10 @@ module CartoDB
       title: 'Invalid ArcGIS version',
       what_about: "The specified ArcGIS server runs an unsupported version. Supported versions are 10.1 onwards."
     },
+    1014 => {
+      title: 'Invalid name',
+      what_about: "File name is not valid. Maybe too many tables with similar names. Please change file name and try again."
+    },
     2001 => {
       title: 'Unable to load data',
       what_about: "We couldn't load data from your file into the database.  Please <a href='mailto:support@cartodb.com?subject=Import load error'>contact us</a> and we will help you to load your data."
@@ -206,7 +216,7 @@ module CartoDB
     },
     2003 => {
       title: 'Malformed CSV',
-      what_about: "The CSV or converted XLS/XLSX to CSV file contains malformed or invalid characters. Some reasons for this error can be for example multiline header fields."
+      what_about: "The CSV or converted XLS/XLSX to CSV file contains malformed or invalid characters. Some reasons for this error can be for example multiline header fields or multiline cells at Excel files or unquoted CSV."
     },
     3007 => {
       title: 'JSON may not be valid GeoJSON',

lib/assets/javascripts/cartodb/table/views/cartocss_doc.jst.ejs

@@ -1,4 +1,4 @@
 <p>If you know how to use CSS to style websites, you already know how to use Carto to style your maps. The Carto language is an easy, flexible, and powerful way to making a better looking map.</p>
 <nav>
-  <a class="alone" href="http://mapbox.com/carto/api/2.1.0/" target="_blank">Take a look at the carto reference</a>
+  <a class="alone" href="https://github.com/mapbox/carto/blob/master/docs/latest.md" target="_blank">Take a look at the carto reference</a>
 </nav>

lib/importer/lib/cartodb-migrator/migrator.rb

@@ -170,9 +170,12 @@ module CartoDB
         }
       ]
 
+      sanitization_count = 0
+
       sanitization_map = sanitization_map.inject({}) { |memo, pair|
         if memo.values.include?(pair.last) || correct_columns.include?(pair.last)
-          memo.merge(pair.first => "#{pair.last}_1")
+          sanitization_count += 1 
+          memo.merge(pair.first => "#{pair.last}_#{sanitization_count}")
         else
           memo.merge(pair.first => pair.last)
         end

services/importer/lib/importer/exceptions.rb

@@ -16,6 +16,7 @@ module CartoDB
     class InvalidGeoJSONError                   < StandardError; end
     class InvalidShpError                       < StandardError; end
     class KmlNetworkLinkError                   < StandardError; end
+    class InvalidNameError                      < StandardError; end
     class LoadError                             < StandardError; end
     class MissingProjectionError                < StandardError; end
     class ShpNormalizationError                 < StandardError; end
@@ -46,6 +47,7 @@ module CartoDB
       InvalidShpError                       => 1006,
       TooManyNodesError                     => 1007,
       GDriveNotPublicError                  => 1010,
+      InvalidNameError                      => 1014,
       LoadError                             => 2001,
       EncodingDetectionError                => 2002,
       MalformedCSVException                 => 2003,

services/importer/lib/importer/query_batcher.rb

@@ -25,8 +25,6 @@ module CartoDB
 
         batched_query = query
 
-        total_rows_processed = 0
-        affected_rows_count = 0
         temp_column = "cartodb_processed_#{table_name.hash.abs}"
 
         where_fragment = %Q{
@@ -50,8 +48,23 @@ module CartoDB
           batched_query << ' WHERE ' + limit_subquery_fragment
         end
 
-        add_processed_column(db_object, table_name, temp_column)
+        begin
+          add_processed_column(db_object, table_name, temp_column)
+          process_batched_query(db_object, batched_query, log, capture_exceptions)
+          remove_processed_column(db_object, table_name, temp_column)
+        rescue => exception
+          raise exception unless capture_exceptions
+          log.log "QUERY:\n#{batched_query}\n---------------------------"
+          log.log "#{exception.to_s}\n---------------------------"
+          log.log "#{exception.backtrace}\n---------------------------"
+        end
+
+        log.log "FINISHED: #{log_message}"
+      end #self.execute
 
+      def self.process_batched_query(db_object, batched_query, log, capture_exceptions)
+        total_rows_processed = 0
+        affected_rows_count = 0
         begin
           begin
             affected_rows_count = db_object.execute(batched_query)
@@ -65,11 +78,7 @@ module CartoDB
             affected_rows_count = -1
           end
         end while affected_rows_count > 0
-
-        remove_processed_column(db_object, table_name, temp_column)
-
-        log.log "FINISHED: #{log_message}"
-      end #self.execute
+      end
 
       def self.add_processed_column(db_object, table_name, column_name)
         db_object.run(%Q{
@@ -95,4 +104,4 @@ module CartoDB
     end #ConsoleLog
 
   end #Importer2
-end #CartoDB
+end #CartoDB

services/importer/lib/importer/runner.rb

@@ -137,7 +137,6 @@ module CartoDB
             end
 
             @importer_stats.timing('import') do
-              begin
               unpacker.source_files.each { |source_file|
                 # TODO: Move this stats inside import, for streaming scenarios, or differentiate
                 log.append "Filename: #{source_file.fullpath} Size (bytes): #{source_file.size}"
@@ -147,9 +146,6 @@ module CartoDB
                 }
                 import(source_file, @downloader)
               }
-              rescue => exception
-                raise exception
-              end
             end
 
             @importer_stats.timing('cleanup') do

services/importer/spec/unit/georeferencer_spec.rb

@@ -181,9 +181,7 @@ describe Importer2::Georeferencer do
   # Attempts to create a new database schema
   # Does not raise exception if the schema already exists
   def create_schema(db, schema)
-    db.run(%Q{CREATE SCHEMA #{schema}})
-  rescue Sequel::DatabaseError => e
-    raise unless e.message =~ /schema .* already exists/
+    db.run(%Q{CREATE SCHEMA IF NOT EXISTS #{schema}})
   end #create_schema
 
   def create_table(db, options={})

spec/connectors/importer_spec.rb

@@ -18,6 +18,9 @@ describe CartoDB::Connector::Importer do
     # if you remove that rescue this test will fail
 
     runner = mock
+    log = mock
+    runner.stubs(:log).returns(log)
+    log.expects(:append).at_least(0)
     quota_checker = mock
     id = UUIDTools::UUID.timestamp_create.to_s
     destination_schema = 'public'

spec/support/factories/tables.rb

@@ -5,16 +5,17 @@ module CartoDB
       table = ::Table.new(attributes)
       table.user_id = if attributes[:user_id].nil?
         UUIDTools::UUID.timestamp_create.to_s
-        #create_user.id
       else
         attributes.delete(:user_id)
       end
+
       table.name = if attributes.keys.include?(:name) && attributes[:name] == nil
         attributes.delete(:name)
         nil
       else
         attributes[:name] || String.random(10)
       end
+      
       table
     end