Make relocator dump the whole user schema

services/relocator/relocator.rb

@@ -7,6 +7,7 @@ require_relative 'relocator/queue_consumer'
 require_relative 'relocator/dumper'
 require_relative 'relocator/tester'
 require_relative 'relocator/table_dumper'
+require_relative 'relocator/schema_dumper'
 require_relative 'relocator/trigger_loader'
 module CartoDB
   module Relocator
@@ -37,7 +38,7 @@ module CartoDB
         if config[:mode] == :relocate
           @dumper = Dumper.new(config: @config)
         else
-          @dumper = TableDumper.new(config: @config)
+          @dumper = SchemaDumper.new(config: @config)
         end
         @consumer = QueueConsumer.new(config: @config)
         @tester   = Tester.new(config: @config) 
@@ -72,6 +73,10 @@ module CartoDB
           @trigger_loader.unload_triggers
           @consumer.empty_queue
         end
+
+        if @config[:mode] == :organize
+          @dumper.remove_target_schema
+        end
       end
     end
   end

services/relocator/relocator/schema_dumper.rb

@@ -0,0 +1,67 @@
+require_relative 'utils'
+
+module CartoDB
+  module Relocator
+    class SchemaDumper
+      include CartoDB::Relocator::Connections
+      def initialize(params={})    
+        @config = params[:config]
+        #@source_db = params[:source_db] || PG.connect(@config[:source][:conn])
+        #@target_db = params[:source_db] || PG.connect(@config[:target][:conn])
+        @dbname = @config[:dbname]
+        @username = @config[:username]
+      end
+
+      def dump_command(config, schema)
+        "pg_dump -n '#{schema}' --verbose --no-tablespaces -Z 0 #{Utils.conn_string(config)}"
+      end
+
+      def restore_command(config)
+        #"pg_restore --verbose --single-transaction --no-tablespaces --disable-triggers #{Utils.conn_string(config)}"
+        "psql -v on_error_stop=1 #{Utils.conn_string(config)}"
+      end
+
+
+      def superuser_conn
+        superuser_conf = @config[:target][:conn].clone
+        superuser_conf.merge!(:user => 'postgres', :dbname => 'postgres')
+        @superuser_conn ||= PG.connect(superuser_conf)
+      end
+
+      def create_db(dbname)
+        #connect as superuser (postgres)
+        puts "Creating DB #{dbname}..."
+        begin
+          superuser_conn.query("CREATE DATABASE \"#{dbname}\"")
+        rescue PG::Error => e
+          puts "Error- Database already exists?"
+          throw e
+        end
+      end
+
+      def get_tables(conn=@config[:source])
+        @table_conn ||= PG.connect(conn[:conn])
+        query = "SELECT table_name FROM information_schema.tables WHERE table_schema='#{conn[:schema]}' AND table_type='BASE TABLE';"
+        @table_conn.query(query).to_a.collect{|t| t['table_name']}
+      end
+
+      def remove_target_schema(conn=@config[:target])
+        @drop_conn ||= PG.connect(conn[:conn])
+        @drop_conn.query("DROP SCHEMA #{conn[:schema]} CASCADE;")
+      end
+
+      def migrate
+        command = "(echo \"BEGIN TRANSACTION;\"; #{dump_command(@config[:source][:conn], @config[:target][:schema])}; echo \"COMMIT;\")| sed \"s/^CREATE SCHEMA.*;$/\-- schema removed/g\"| #{restore_command(@config[:target][:conn])}"
+        puts "Running: #{command}"
+        return_code = system(command)
+        raise "Error dumping and restoring! Please cleanup" if return_code != true
+        get_tables(@config[:target]).reject{|t| t == "spatial_ref_sys"}.each do |table|
+          puts "Cartodbfying table #{table}.."
+          @cartodbfy_conn ||= PG.connect(@config[:target][:conn])
+          puts @cartodbfy_conn.query("select cdb_cartodbfytable('#{@config[:target][:schema]}.#{table}')").to_a
+        end
+      end
+    end
+  end
+end
+

services/relocator/relocator/tester.rb

@@ -28,8 +28,8 @@ module CartoDB
 
       def get_names_of_tables(conn)
         PG.connect(conn[:conn])
-          .query("select count(*) from information_schema.tables where table_type='BASE TABLE' AND table_schema='#{conn[:schema]}' and table_name NOT IN ('spatial_ref_sys');")
-          .to_a.collect{|t| t['rel_name']}
+          .query("select * from information_schema.tables where table_type='BASE TABLE' AND table_schema='#{conn[:schema]}' and table_name NOT IN ('spatial_ref_sys');")
+          .to_a.collect{|t| t['table_name']}.sort
       end
 
       def get_statement_timeout(conn)
@@ -68,8 +68,9 @@ module CartoDB
         first = get_state(@config[:source])
         puts "Fetching stats for target database.."
         last = get_state(@config[:target])
+        p [first, last]
         first.keys.collect do |key|
-          throw "Consistency fail: #{key} (#{first[key]} != #{last[key][to_s]}) " if first[key] != last[key]
+          throw "Consistency fail: #{key} (#{first[key].to_s} != #{last[key].to_s}) " if first[key] != last[key]
           [key, first[key] == last[key]]
         end
       end

services/relocator/worker.rb

@@ -18,14 +18,43 @@ module CartoDB
         relocator = CartoDB::Relocator::Relocation.new(
           source: {conn: {host: user.database_host, port: port,
                           dbname: user.database_name,
-                          user: 'postgres'}, schema: user.database_schema},
+                          user: 'postgres'}, schema: user.username}, #we will move 'public' to this schema
           target: {conn: {host: org.owner.database_host,  port: port,
-                          dbname: org.owner.database_name, user: user.database_username}, schema: user.username},
+                          dbname: org.owner.database_name, user: 'postgres'}, schema: user.username},
           redis: {host: Cartodb.config[:redis]['host'], port: Cartodb.config[:redis]['port']},
           dbname: user.database_name, username: user.database_username, :mode => :organize,
           user_object: user
         )
         begin
+          #we first move the user to its own schema
+          case user.database_schema
+          when 'public'
+            #associate it to the organization now so it lets create the public_user
+            #on the schema.
+            user.organization = org
+            user.database_schema = user.username
+            begin
+              user.create_public_db_user
+            rescue => e
+              puts "Error #{e} while creating public user. Ignoring as it probably already existed"
+            end
+            user.set_database_search_path
+            user.grant_publicuser_in_database
+            user.set_user_privileges
+            user.organization = nil
+            User.terminate_database_connections(user.database_name, user.database_host)
+            user.in_database(as: :superuser) do |database|
+              database['ALTER SCHEMA public RENAME TO '+user.username].all
+              # An apple a day keeps PostGIS away
+              database['CREATE SCHEMA public; ALTER EXTENSION postgis SET SCHEMA public'].all
+            end
+            user.save
+            puts "Migrated to schema-powered successfully!"
+          when user.username
+            puts "User is already on its own, non-public schema."
+          else
+            raise "User is on a different schema than expected."
+          end
           user.database_host = org.owner.database_host
           user.database_name = org.owner.database_name
           user.organization = org
@@ -45,23 +74,32 @@ module CartoDB
           user.set_database_search_path
           user.grant_user_in_database
           user.set_user_privileges
+          old_user_timeout = user.user_timeout
+          user.user_timeout = 0
+          user.set_statement_timeouts
+          User.terminate_database_connections(user.database_name, user.database_host)
           relocator.migrate
+          #wipe all OIDs
+          user.tables.each{|t| t.table_id=nil; t.save}
+          user.user_timeout = old_user_timeout
           user.set_statement_timeouts
           relocator.compare
           relocator.finalize
+          user.grant_publicuser_in_database
           user.set_user_privileges
           user.rebuild_quota_trigger
           user.set_user_as_organization_member
           user.enable_remote_db_user
-          user.save
-          user.create_in_central
-          user.update_in_central
         rescue => e
           puts "Error: #{e}, #{e.backtrace}"
           puts "Rolling back in 5 secs"
           sleep 5
           relocator.rollback
+          return
         end
+        user.save
+        user.create_in_central
+        user.update_in_central
       end # organize
       
       def self.relocate(user, new_database_host, new_database_port=nil)