cartodb
/
cartodb-4.29
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'f546250111'
${ noResults }
cartodb-4.29/script/migrate_to_uuid.rb

461 lines
15 KiB
Raw Blame History

#!/usr/bin/env ruby
require 'pg'
require 'redis'
# RAILS_ENV=development DBNAME=carto_db_development DBHOST=127.0.0.1 DBUSER=postgres REDIS_HOST=127.0.0.1
def execution_summary()
wait_time = 30
puts <<-EOH
###
#
# You are running the action '#{ACTION}' which performs the next actions:
#
# #{@actions[ACTION]}
#
# It's highly recommended to have a database backup of the PostgreSQL databases, mainly the metadata one and Redis.
#
#
# Params used within the execution of the script
#
# DB Connection
# -------------
# Database host: #{DBHOST}
# Database port: #{DBPORT}
# Database name: #{DBNAME}
# Database user: #{DBUSER}
#
# Redis Connection
# ----------------
# Redis host: #{REDIS_HOST}
#
###
EOH
puts "###"
puts "If there is anything wrong or you are not sure about what you are doing, you have #{wait_time} secs to cancel before the process starts"
counter(wait_time)
end
def usage(message = nil)
if !message.nil?
puts ""
puts "ERROR: #{message}"
puts ""
end
puts "Usage:"
puts " export RAILS_ENV=<rails_env>"
puts " export DBNAME=<your_postgresql_database_name>"
puts " export DBHOST=<your_postgresql_database_host>"
puts " export DBPORT=<your_postgresql_database_port>"
puts " export DBUSER=<your_postgresql_database_user>"
puts " export REDIS_HOST=<your_redis_host>"
puts ""
puts "# Notice that you probably want to use the same values found in your database.yml and app_config.yml"
puts ""
puts " #{__FILE__} <action>"
puts ""
puts "Actions:"
@actions.each {|k,v| puts " %10s %s" % [k, v]; puts "" }
exit 1
end
def counter(max)
(0..max).each do |n|
print n
sleep 1
print "\r"
end
end
## MAIN
#
@actions = {
'schema' => 'Creates a UUID column in every table with a id. Also creates a\nll the UUID dependency columns between tables. You can still rollback after this step',
'meta' => 'Update every dependency UUID column with the proper one based on the id integer relations. This is the last step that you can rollback',
'rollback' => 'Try to rollback previous steps',
'data' => 'Migrate all postgresql database and users names to UUID format. Also update the user model database_name attribute. Update all redis info with UUIDs. IMPORTANT: You cannot rollback this step',
'clean' => 'Drop old id columns. Rename new uuid colums to id. Rename all uuid dependency columns to id. Create new primary keys from UUID attributes. IMPORTANT: You cannot rollback this step'
}
usage "You need to initialize a environment with RAILS_ENV" if (ENV['RAILS_ENV'].nil? || ENV['RAILS_ENV'].empty?)
usage "You need to set a DBNAME env" if (ENV['DBNAME'].nil? || ENV['DBNAME'].empty?)
usage "You need to set a DBHOST env" if (ENV['DBHOST'].nil? || ENV['DBHOST'].empty?)
usage "You need to set a DBPORT env" if (ENV['DBPORT'].nil? || ENV['DBPORT'].empty?)
usage "You need to set a DBUSER env" if (ENV['DBUSER'].nil? || ENV['DBUSER'].empty?)
usage "You need to set a REDIS_HOST env" if (ENV['REDIS_HOST'].nil? || ENV['REDIS_HOST'].empty?)
ENVIRONMENT = ENV['RAILS_ENV']
DBHOST = ENV['DBHOST']
DBPORT = ENV['DBPORT']
DBUSER = ENV['DBUSER']
DBNAME = ENV['DBNAME']
REDIS_HOST = ENV['REDIS_HOST']
ACTION = ARGV[0]
if ACTION.nil? || !@actions.keys.include?(ACTION)
usage "Missing action"
end
@logs = Hash.new
tables = {
:assets => {
:related => [],
:singular => 'asset'
},
:automatic_geocodings => {
:related => ['geocodings'],
:singular => 'automatic_geocoding'
},
:client_applications => {
:related => ['oauth_tokens'],
:singular => 'client_application'
},
:data_imports => {
:related => ['user_tables'],
:singular => 'data_import'
},
:geocodings => {
:related => [],
:singular => 'geocoding'
},
:layers => {
:related => ['layers_maps', 'layers_users', 'layers_user_tables', 'visualizations'],
:singular => 'layer',
:relation_for => {'visualizations' => 'active_layer'}
},
:layers_maps => {
:related => [],
:singular => 'layer_map'
},
:layers_user_tables => {
:related => [],
:singular => 'layer_user_table'
},
:layers_users => {
:related => [],
:singular => 'layer_user'
},
:maps => {
:related => ['user_tables', 'layers_maps', 'visualizations'],
:singular => 'map'
},
:oauth_nonces => {
:related => [],
:singular => 'oauth_nonce'
},
:oauth_tokens => {
:related => [],
:singular => 'oauth_token'
},
:overlays => {
:related => [],
:singular => 'overlay'
},
:tags => {
:related => [],
:singular => 'tag'
},
:user_tables => {
:related => ['data_imports', 'layers_user_tables', 'tags', 'automatic_geocodings', 'geocodings'],
:singular => 'table',
:relation_for => {'layers_user_tables' => 'user_table'}
},
:users => {
:related => ['user_tables', 'maps', 'layers_users', 'assets', 'client_applications', 'oauth_tokens', 'tags', 'data_imports', 'synchronizations', 'geocodings'],
:singular => 'user'
},
#:visualizations => {
# :related => ['overlays'],
# :singular => 'visualization'
#}
}
redis_keys = {
:map_style => {
:template => "map_style|USERDB|*",
:var_position => 1,
:separator => '|',
:db => 0,
:type => 'string'
},
:table => {
:template => "rails:USERDB:*",
:var_position => 1,
:separator => ':',
:db => 0,
:type => 'hash',
:attributes => {
:user_id => 'USERID'
}
},
:user => {
:template => "rails:users:USERNAME",
:no_clone => true,
:var_position => 2,
:separator => ':',
:db => 5,
:type => 'hash',
:attributes => {
:database_name => 'USERDB',
:id => 'USERID'
}
}
}
def redis_replace_from_template(template, id, username)
if template.include?('USERDB')
user_database(id)
elsif template.include?('DBUSER')
database_username(id)
elsif template.include?('USERNAME')
username
elsif template.include?('USERID')
id
else
''
end
end
def redis_template_user_gsub(template, id, username)
replacement = redis_replace_from_template(template, id, username)
if template.include?('USERDB')
template.gsub('USERDB', replacement)
elsif template.include?('DBUSER')
template.gsub('DBUSER', replacement)
elsif template.include?('USERNAME')
template.gsub('USERNAME', replacement)
else
''
end
end
def copy_redis_keys(redis_keys, id, uuid, username)
redis = Redis.new(:host => REDIS_HOST)
redis_keys.each do |k,v|
redis.select(v[:db])
these_redis_keys = redis.keys(redis_template_user_gsub(v[:template], id, username))
these_redis_keys.each do |trd|
original_value = redis.dump(trd)
new_array = trd.split(v[:separator])
new_array[v[:var_position]] = redis_replace_from_template(v[:template], uuid, username)
new_key = new_array.join(v[:separator])
unless v[:no_clone]
redis.restore(new_key, 0, original_value)
end
if v[:type] == 'hash'
v[:attributes].each do |a,av|
redis.hset(new_key, a.to_s, redis_replace_from_template(av, uuid, username))
end
end
end
end
redis.quit
end
def alter_redis_hash(redis_key, redis_attribute, redis_value, options = {})
redis_db = options['db'].nil? ? 0 : options['db']
redis = Redis.new(:host => REDIS_HOST)
redis.select(redis_db)
redis.hset(redis_key, user_id, redis_value)
redis.quit
end
def relation_column_name_for(tables, table, related)
if tables[table][:related].include?(related) && tables[table][:relation_for] && tables[table][:relation_for][related]
tables[table][:relation_for][related]
else
tables[table][:singular]
end
end
def log(severity, type, msg)
puts " #{msg}. Ignoring.."
@logs.merge({:severity => severity, :type => type, :msg => msg})
end
def database_username(user_id)
"#{db_username_prefix}#{user_id}"
end #database_username
def user_database(user_id)
"#{database_name_prefix}#{user_id}_db"
end #user_database
def db_username_prefix
return "cartodb_user_" if ENVIRONMENT == 'production'
return "development_cartodb_user_" if ENVIRONMENT == 'development'
"cartodb_user_#{ENVIRONMENT}_"
end #username_prefix
def database_name_prefix
return "cartodb_user_" if ENVIRONMENT == 'production'
return "cartodb_dev_user_" if ENVIRONMENT == 'development'
"cartodb_#{ENVIRONMENT}_user_"
end #database_prefix
def alter_schema(tables)
tables.each do |tname, tinfo|
# Create main uuid column in every table
puts "Creating uuid column in #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} ADD uuid uuid UNIQUE NOT NULL DEFAULT uuid_generate_v4()")
rescue => e
log('C', "Creating uuid column in #{tname}", e.error.strip)
end
tinfo[:related].each do |rtable|
# Create relation uuid column in a dependent table
puts "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} ADD #{relation_column_name_for(tables, tname, rtable)}_uuid uuid")
rescue => e
log('C', "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
end
end
end
end
def rollback_schema(tables)
tables.each do |tname, tinfo|
tinfo[:related].each do |rtable|
# Create relation uuid column in a dependent table
puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_uuid")
rescue => e
log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
end
end
# Destroy main uuid column in every table
puts "Dropping uuid column in #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} DROP IF EXISTS uuid")
rescue => e
log('C', "Dropping uuid column in #{tname}", e.error.strip)
end
end
end
def migrate_meta(tables)
tables.each do |tname, tinfo|
@conn.exec("SELECT id,uuid FROM #{tname}") do |result|
result.each do |row|
tinfo[:related].each do |rtable|
puts "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}"
begin
@conn.exec("UPDATE #{rtable} SET #{relation_column_name_for(tables, tname, rtable)}_uuid='#{row['uuid']}' WHERE #{relation_column_name_for(tables, tname, rtable)}_id='#{row['id']}'")
rescue => e
log('C', "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}", e.error.strip)
end
end
end
end
end
end
def migrate_data(redis_keys)
sconn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: 'postgres' )
@conn.exec("SELECT id,uuid,database_name,username FROM users") do |result|
result.each do |row|
puts "Renaming pg user and db for id #{row['id']}"
begin
sconn.exec("ALTER DATABASE \"#{row['database_name']}\" RENAME TO \"#{user_database(row['uuid'])}\"")
sconn.exec("ALTER ROLE \"#{database_username(row['id'])}\" RENAME TO \"#{database_username(row['uuid'])}\"")
@conn.exec("UPDATE users SET database_name='#{user_database(row['uuid'])}' WHERE id=#{row['id']} AND uuid='#{row['uuid']}'")
@conn.exec("UPDATE user_tables SET database_name='#{user_database(row['uuid'])}' WHERE user_id='#{row['id']}' AND user_uuid='#{row['uuid']}'")
rescue => e
log('C', "Renaming pg user and db for id #{row['id']}", e.error.strip)
end
puts "Copying redis keys with uuid for id #{row['id']}"
#begin
copy_redis_keys(redis_keys, row['id'], row['uuid'], row['username'])
#rescue => e
# log('C', "Copying redis keys with uuid for id #{row['id']}", e.error.strip)
#end
end
end
@conn.exec("SELECT token,user_id FROM oauth_tokens WHERE type='AccessToken'") do |result|
result.each do |row|
puts "Chaing user_id for oauth token '#{row['token']}'"
begin
alter_redis_hash("rails:oauth_access_tokens:#{row['token']}", 'user_id', row['user_id'], {'db' => 3})
rescue => e
log('C', "Changing user id to uuid in oauth token #{row['token']}", e.error.strip)
end
end
end
end
def clean_db(tables)
tables.each do |tname, tinfo|
tinfo[:related].each do |rtable|
# Drop old id relation column in every table
puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_id")
rescue => e
log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}", e.error.strip)
end
# Rename new uuid relation column to id
puts "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} RENAME #{relation_column_name_for(tables, tname, rtable)}_uuid TO #{relation_column_name_for(tables, tname, rtable)}_id")
rescue => e
log('C', "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}", e.error.strip)
end
end
# Drop old id column in every table
puts "Dropping old id from #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} DROP IF EXISTS id")
rescue => e
log('C', "Dropping old id from #{rtable}", e.error.strip)
end
# Rename new uuid relation column to id
puts "Renaming uuid to id in #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} RENAME uuid TO id")
rescue => e
log('C', "Renaming uuid to id in #{tname}", e.error.strip)
end
# Set new id as primary key
puts "Setting new id as primary key on #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} ADD PRIMARY KEY (id)")
rescue => e
log('C', "Setting new id as primary key on #{tname}", e.error.strip)
end
end
end
execution_summary
@conn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: DBNAME )
@conn.exec("CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"")
if ACTION == 'schema'
alter_schema(tables)
elsif ACTION == 'rollback'
rollback_schema(tables)
elsif ACTION == 'meta'
migrate_meta(tables)
elsif ACTION == 'data'
migrate_data(redis_keys)
elsif ACTION == 'clean'
clean_db(tables)
end
puts ""
puts "#############"
puts "#{@logs.length} errors"
puts "#############"
Reference in new issue View Git Blame Copy Permalink