cartodb/script/migrate_to_uuid.rb

#!/usr/bin/env ruby

require 'pg'
require 'redis'

# RAILS_ENV=development DBNAME=carto_db_development DBHOST=127.0.0.1 DBUSER=postgres REDIS_HOST=127.0.0.1




def execution_summary()
  wait_time = 30
  puts <<-EOH
  ###
  #
  # You are running the action '#{ACTION}' which performs the next actions:
  # 
  #   #{@actions[ACTION]}
  #
  # It's highly recommended to have a database backup of the PostgreSQL databases, mainly the metadata one and Redis.
  #
  #
  # Params used within the execution of the script
  # 
  # DB Connection
  # -------------
  # Database host: #{DBHOST}
  # Database port: #{DBPORT}
  # Database name: #{DBNAME}
  # Database user: #{DBUSER}
  #
  # Redis Connection
  # ----------------
  # Redis host: #{REDIS_HOST}
  #
  ###
  EOH
  puts "###"
  puts "If there is anything wrong or you are not sure about what you are doing, you have #{wait_time} secs to cancel before the process starts"
  counter(wait_time)
end

def usage(message = nil)
  if !message.nil?
    puts ""
    puts "ERROR: #{message}"
    puts ""
  end
  puts "Usage:"
  puts "  export RAILS_ENV=<rails_env>"
  puts "  export DBNAME=<your_postgresql_database_name>"
  puts "  export DBHOST=<your_postgresql_database_host>"
  puts "  export DBPORT=<your_postgresql_database_port>"
  puts "  export DBUSER=<your_postgresql_database_user>"
  puts "  export REDIS_HOST=<your_redis_host>"
  puts ""
  puts "# Notice that you probably want to use the same values found in your database.yml and app_config.yml"
  puts ""
  puts "  #{__FILE__} <action>"
  puts ""
  puts "Actions:"
  @actions.each {|k,v| puts "  %10s   %s" % [k, v]; puts "" }
  exit 1
end

def counter(max)
  (0..max).each do |n|
    print n
    sleep 1
    print "\r"
  end
end

## MAIN
#

@actions = {
            'schema' => 'Creates a UUID column in every table with a id. Also creates a\nll the UUID dependency columns between tables. You can still rollback after this step',
            'meta' => 'Update every dependency UUID column with the proper one based on the id integer relations. This is the last step that you can rollback',
            'rollback' => 'Try to rollback previous steps',
            'data' => 'Migrate all postgresql database and users names to UUID format. Also update the user model database_name attribute. Update all redis info with UUIDs. IMPORTANT: You cannot rollback this step',
            'clean' => 'Drop old id columns. Rename new uuid colums to id. Rename all uuid dependency columns to id. Create new primary keys from UUID attributes. IMPORTANT: You cannot rollback this step'
           }

usage "You need to initialize a environment with RAILS_ENV" if (ENV['RAILS_ENV'].nil? || ENV['RAILS_ENV'].empty?)
usage "You need to set a DBNAME env" if (ENV['DBNAME'].nil? || ENV['DBNAME'].empty?)
usage "You need to set a DBHOST env" if (ENV['DBHOST'].nil? || ENV['DBHOST'].empty?)
usage "You need to set a DBPORT env" if (ENV['DBPORT'].nil? || ENV['DBPORT'].empty?)
usage "You need to set a DBUSER env" if (ENV['DBUSER'].nil? || ENV['DBUSER'].empty?)
usage "You need to set a REDIS_HOST env" if (ENV['REDIS_HOST'].nil? || ENV['REDIS_HOST'].empty?)

ENVIRONMENT = ENV['RAILS_ENV']
DBHOST = ENV['DBHOST']
DBPORT = ENV['DBPORT']
DBUSER = ENV['DBUSER']
DBNAME = ENV['DBNAME']
REDIS_HOST = ENV['REDIS_HOST']


ACTION = ARGV[0]

if ACTION.nil? || !@actions.keys.include?(ACTION)
  usage "Missing action"
end

@logs = Hash.new

tables = {
           :assets => { 
             :related => [],
             :singular => 'asset'
           },
           :automatic_geocodings => { 
             :related => ['geocodings'],
             :singular => 'automatic_geocoding'
           },
           :client_applications => {
             :related => ['oauth_tokens'],
             :singular => 'client_application'
           },
           :data_imports => {
             :related => ['user_tables'],
             :singular => 'data_import'
           },
           :geocodings => {
             :related => [],
             :singular => 'geocoding'
           },
           :layers => {
             :related => ['layers_maps', 'layers_users', 'layers_user_tables', 'visualizations'],
             :singular => 'layer',
             :relation_for => {'visualizations' => 'active_layer'}
           },
           :layers_maps => {
             :related => [],
             :singular => 'layer_map'
           },
           :layers_user_tables => {
             :related => [],
             :singular => 'layer_user_table'
           },
           :layers_users => {
             :related => [],
             :singular => 'layer_user'
           },
           :maps => {
             :related => ['user_tables', 'layers_maps', 'visualizations'],
             :singular => 'map'
           },
           :oauth_nonces => {
             :related => [],
             :singular => 'oauth_nonce'
           },
           :oauth_tokens => {
             :related => [],
             :singular => 'oauth_token'
           },
           :overlays => {
             :related => [],
             :singular => 'overlay'
           },
           :tags => {
             :related => [],
             :singular => 'tag'
           },
           :user_tables => {
             :related => ['data_imports', 'layers_user_tables', 'tags', 'automatic_geocodings', 'geocodings'],
             :singular => 'table',
             :relation_for => {'layers_user_tables' => 'user_table'}
           },
           :users => {
             :related => ['user_tables', 'maps', 'layers_users', 'assets', 'client_applications', 'oauth_tokens', 'tags', 'data_imports', 'synchronizations', 'geocodings'],
             :singular => 'user'
           },
           #:visualizations => {
           #  :related => ['overlays'],
           #  :singular => 'visualization'
           #}
         }

redis_keys = {
              :map_style => {
                :template => "map_style|USERDB|*",
                :var_position => 1,
                :separator => '|',
                :db => 0,
                :type => 'string'
              },
              :table => {
                :template => "rails:USERDB:*",
                :var_position => 1, 
                :separator => ':',
                :db => 0,
                :type => 'hash',
                :attributes => {
                  :user_id => 'USERID'
                }
              },
              :user => {
                :template => "rails:users:USERNAME",
                :no_clone => true,
                :var_position => 2,
                :separator => ':',
                :db => 5,
                :type => 'hash',
                :attributes => {
                  :database_name => 'USERDB',
                  :id => 'USERID'
                }
              }
            }

def redis_replace_from_template(template, id, username)
  if template.include?('USERDB')
    user_database(id)
  elsif template.include?('DBUSER')
    database_username(id)
  elsif template.include?('USERNAME')
    username
  elsif template.include?('USERID')
    id
  else
    '' 
  end
end

def redis_template_user_gsub(template, id, username)
  replacement = redis_replace_from_template(template, id, username)
  if template.include?('USERDB')
    template.gsub('USERDB', replacement)
  elsif template.include?('DBUSER')
    template.gsub('DBUSER', replacement)
  elsif template.include?('USERNAME')
    template.gsub('USERNAME', replacement)
  else
    '' 
  end
end

def copy_redis_keys(redis_keys, id, uuid, username)
  redis = Redis.new(:host => REDIS_HOST)
  redis_keys.each do |k,v|
    redis.select(v[:db])
    these_redis_keys = redis.keys(redis_template_user_gsub(v[:template], id, username))
    these_redis_keys.each do |trd|
        original_value = redis.dump(trd)
        new_array = trd.split(v[:separator])
        new_array[v[:var_position]] = redis_replace_from_template(v[:template], uuid, username)
        new_key = new_array.join(v[:separator])
      unless v[:no_clone]
        redis.restore(new_key, 0, original_value)
      end
      if v[:type] == 'hash'
        v[:attributes].each do |a,av|
          redis.hset(new_key, a.to_s, redis_replace_from_template(av, uuid, username))
        end
      end
    end
  end
  redis.quit
end

def alter_redis_hash(redis_key, redis_attribute, redis_value, options = {})
  redis_db = options['db'].nil? ? 0 : options['db']
  redis = Redis.new(:host => REDIS_HOST)
  redis.select(redis_db)
  redis.hset(redis_key, user_id, redis_value)
  redis.quit
end

def relation_column_name_for(tables, table, related)
  if tables[table][:related].include?(related) && tables[table][:relation_for] && tables[table][:relation_for][related]
    tables[table][:relation_for][related]
  else
    tables[table][:singular]
  end
end


def log(severity, type, msg)
  puts "    #{msg}. Ignoring.."
  @logs.merge({:severity => severity, :type => type, :msg => msg})
end

def database_username(user_id)
  "#{db_username_prefix}#{user_id}"
end #database_username

def user_database(user_id)
  "#{database_name_prefix}#{user_id}_db"
end #user_database

def db_username_prefix
  return "cartodb_user_" if ENVIRONMENT == 'production'
  return "development_cartodb_user_" if ENVIRONMENT == 'development'
  "cartodb_user_#{ENVIRONMENT}_"
end #username_prefix

def database_name_prefix
  return "cartodb_user_" if ENVIRONMENT == 'production'
  return "cartodb_dev_user_" if ENVIRONMENT == 'development'
  "cartodb_#{ENVIRONMENT}_user_"
end #database_prefix

def alter_schema(tables) 
  tables.each do |tname, tinfo|
    # Create main uuid column in every table
    puts "Creating uuid column in #{tname}"
    begin
      @conn.exec("ALTER TABLE #{tname} ADD uuid uuid UNIQUE NOT NULL DEFAULT uuid_generate_v4()")
    rescue => e
      log('C', "Creating uuid column in #{tname}", e.error.strip)
    end
    tinfo[:related].each do |rtable|
      # Create relation uuid column in a dependent table
      puts "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
      begin
        @conn.exec("ALTER TABLE #{rtable} ADD #{relation_column_name_for(tables, tname, rtable)}_uuid uuid")
      rescue => e
        log('C', "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
      end
    end
  end
end

def rollback_schema(tables)
  tables.each do |tname, tinfo|
    tinfo[:related].each do |rtable|
      # Create relation uuid column in a dependent table
      puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
      begin
        @conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_uuid")
      rescue => e
        log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
      end
    end
    # Destroy main uuid column in every table
    puts "Dropping uuid column in #{tname}"
    begin
      @conn.exec("ALTER TABLE #{tname} DROP IF EXISTS uuid")
    rescue => e
      log('C', "Dropping uuid column in #{tname}", e.error.strip)
    end
  end
end

def migrate_meta(tables)
  tables.each do |tname, tinfo|
    @conn.exec("SELECT id,uuid FROM #{tname}") do |result|
      result.each do |row|
        tinfo[:related].each do |rtable|
          puts "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}"
          begin
            @conn.exec("UPDATE #{rtable} SET #{relation_column_name_for(tables, tname, rtable)}_uuid='#{row['uuid']}' WHERE #{relation_column_name_for(tables, tname, rtable)}_id='#{row['id']}'")
          rescue => e
            log('C', "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}", e.error.strip)
          end
        end
      end
    end
  end
end


def migrate_data(redis_keys)
  sconn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: 'postgres' )
  @conn.exec("SELECT id,uuid,database_name,username FROM users") do |result|
    result.each do |row|
      puts "Renaming pg user and db for id #{row['id']}"
      begin
        sconn.exec("ALTER DATABASE \"#{row['database_name']}\" RENAME TO \"#{user_database(row['uuid'])}\"")
        sconn.exec("ALTER ROLE \"#{database_username(row['id'])}\" RENAME TO \"#{database_username(row['uuid'])}\"")
        @conn.exec("UPDATE users SET database_name='#{user_database(row['uuid'])}' WHERE id=#{row['id']} AND uuid='#{row['uuid']}'")
        @conn.exec("UPDATE user_tables SET database_name='#{user_database(row['uuid'])}' WHERE user_id='#{row['id']}' AND user_uuid='#{row['uuid']}'")
      rescue => e
        log('C', "Renaming pg user and db for id #{row['id']}", e.error.strip)
      end
      puts "Copying redis keys with uuid for id #{row['id']}"
      #begin
        copy_redis_keys(redis_keys, row['id'], row['uuid'], row['username'])
      #rescue => e
      #  log('C', "Copying redis keys with uuid for id #{row['id']}", e.error.strip)
      #end
    end
  end
  @conn.exec("SELECT token,user_id FROM oauth_tokens WHERE type='AccessToken'") do |result|
    result.each do |row|
      puts "Chaing user_id for oauth token '#{row['token']}'"
      begin
        alter_redis_hash("rails:oauth_access_tokens:#{row['token']}", 'user_id', row['user_id'], {'db' => 3})
      rescue => e
        log('C', "Changing user id to uuid in oauth token #{row['token']}", e.error.strip)
      end
    end
  end
end

def clean_db(tables) 
  tables.each do |tname, tinfo|
    tinfo[:related].each do |rtable|
      # Drop old id relation column in every table
      puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}"
      begin
        @conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_id")
      rescue => e
        log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}", e.error.strip)
      end
      # Rename new uuid relation column to id
      puts "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}"
      begin
        @conn.exec("ALTER TABLE #{rtable} RENAME #{relation_column_name_for(tables, tname, rtable)}_uuid TO #{relation_column_name_for(tables, tname, rtable)}_id")
      rescue => e
        log('C', "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}", e.error.strip)
      end
    end
    # Drop old id column in every table
    puts "Dropping old id from #{tname}"
    begin
      @conn.exec("ALTER TABLE #{tname} DROP IF EXISTS id")
    rescue => e
      log('C', "Dropping old id from #{rtable}", e.error.strip)
    end
    # Rename new uuid relation column to id
    puts "Renaming uuid to id in #{tname}"
    begin
      @conn.exec("ALTER TABLE #{tname} RENAME uuid TO id")
    rescue => e
      log('C', "Renaming uuid to id in #{tname}", e.error.strip)
    end
    # Set new id as primary key
    puts "Setting new id as primary key on #{tname}"
    begin
      @conn.exec("ALTER TABLE #{tname} ADD PRIMARY KEY (id)")
    rescue => e
      log('C', "Setting new id as primary key on #{tname}", e.error.strip)
    end
  end
end

execution_summary

@conn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: DBNAME )
@conn.exec("CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"")

if ACTION == 'schema'
  alter_schema(tables)
elsif ACTION == 'rollback'
  rollback_schema(tables)
elsif ACTION == 'meta'
  migrate_meta(tables)
elsif ACTION == 'data'
  migrate_data(redis_keys)
elsif ACTION == 'clean'
  clean_db(tables)
end

puts ""
puts "#############"
puts "#{@logs.length} errors"
puts "#############"