461 lines
15 KiB
Ruby
461 lines
15 KiB
Ruby
|
#!/usr/bin/env ruby
|
||
|
|
|
||
|
|
require 'pg'
|
||
|
|
require 'redis'
|
||
|
|
|
||
|
|
# RAILS_ENV=development DBNAME=carto_db_development DBHOST=127.0.0.1 DBUSER=postgres REDIS_HOST=127.0.0.1
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def execution_summary()
|
||
|
|
wait_time = 30
|
||
|
|
puts <<-EOH
|
||
|
|
###
|
||
|
|
#
|
||
|
|
# You are running the action '#{ACTION}' which performs the next actions:
|
||
|
|
#
|
||
|
|
# #{@actions[ACTION]}
|
||
|
|
#
|
||
|
|
# It's highly recommended to have a database backup of the PostgreSQL databases, mainly the metadata one and Redis.
|
||
|
|
#
|
||
|
|
#
|
||
|
|
# Params used within the execution of the script
|
||
|
|
#
|
||
|
|
# DB Connection
|
||
|
|
# -------------
|
||
|
|
# Database host: #{DBHOST}
|
||
|
|
# Database port: #{DBPORT}
|
||
|
|
# Database name: #{DBNAME}
|
||
|
|
# Database user: #{DBUSER}
|
||
|
|
#
|
||
|
|
# Redis Connection
|
||
|
|
# ----------------
|
||
|
|
# Redis host: #{REDIS_HOST}
|
||
|
|
#
|
||
|
|
###
|
||
|
|
EOH
|
||
|
|
puts "###"
|
||
|
|
puts "If there is anything wrong or you are not sure about what you are doing, you have #{wait_time} secs to cancel before the process starts"
|
||
|
|
counter(wait_time)
|
||
|
|
end
|
||
|
|
|
||
|
|
def usage(message = nil)
|
||
|
|
if !message.nil?
|
||
|
|
puts ""
|
||
|
|
puts "ERROR: #{message}"
|
||
|
|
puts ""
|
||
|
|
end
|
||
|
|
puts "Usage:"
|
||
|
|
puts " export RAILS_ENV=<rails_env>"
|
||
|
|
puts " export DBNAME=<your_postgresql_database_name>"
|
||
|
|
puts " export DBHOST=<your_postgresql_database_host>"
|
||
|
|
puts " export DBPORT=<your_postgresql_database_port>"
|
||
|
|
puts " export DBUSER=<your_postgresql_database_user>"
|
||
|
|
puts " export REDIS_HOST=<your_redis_host>"
|
||
|
|
puts ""
|
||
|
|
puts "# Notice that you probably want to use the same values found in your database.yml and app_config.yml"
|
||
|
|
puts ""
|
||
|
|
puts " #{__FILE__} <action>"
|
||
|
|
puts ""
|
||
|
|
puts "Actions:"
|
||
|
|
@actions.each {|k,v| puts " %10s %s" % [k, v]; puts "" }
|
||
|
|
exit 1
|
||
|
|
end
|
||
|
|
|
||
|
|
def counter(max)
|
||
|
|
(0..max).each do |n|
|
||
|
|
print n
|
||
|
|
sleep 1
|
||
|
|
print "\r"
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
## MAIN
|
||
|
|
#
|
||
|
|
|
||
|
|
@actions = {
|
||
|
|
'schema' => 'Creates a UUID column in every table with a id. Also creates a\nll the UUID dependency columns between tables. You can still rollback after this step',
|
||
|
|
'meta' => 'Update every dependency UUID column with the proper one based on the id integer relations. This is the last step that you can rollback',
|
||
|
|
'rollback' => 'Try to rollback previous steps',
|
||
|
|
'data' => 'Migrate all postgresql database and users names to UUID format. Also update the user model database_name attribute. Update all redis info with UUIDs. IMPORTANT: You cannot rollback this step',
|
||
|
|
'clean' => 'Drop old id columns. Rename new uuid colums to id. Rename all uuid dependency columns to id. Create new primary keys from UUID attributes. IMPORTANT: You cannot rollback this step'
|
||
|
|
}
|
||
|
|
|
||
|
|
usage "You need to initialize a environment with RAILS_ENV" if (ENV['RAILS_ENV'].nil? || ENV['RAILS_ENV'].empty?)
|
||
|
|
usage "You need to set a DBNAME env" if (ENV['DBNAME'].nil? || ENV['DBNAME'].empty?)
|
||
|
|
usage "You need to set a DBHOST env" if (ENV['DBHOST'].nil? || ENV['DBHOST'].empty?)
|
||
|
|
usage "You need to set a DBPORT env" if (ENV['DBPORT'].nil? || ENV['DBPORT'].empty?)
|
||
|
|
usage "You need to set a DBUSER env" if (ENV['DBUSER'].nil? || ENV['DBUSER'].empty?)
|
||
|
|
usage "You need to set a REDIS_HOST env" if (ENV['REDIS_HOST'].nil? || ENV['REDIS_HOST'].empty?)
|
||
|
|
|
||
|
|
ENVIRONMENT = ENV['RAILS_ENV']
|
||
|
|
DBHOST = ENV['DBHOST']
|
||
|
|
DBPORT = ENV['DBPORT']
|
||
|
|
DBUSER = ENV['DBUSER']
|
||
|
|
DBNAME = ENV['DBNAME']
|
||
|
|
REDIS_HOST = ENV['REDIS_HOST']
|
||
|
|
|
||
|
|
|
||
|
|
ACTION = ARGV[0]
|
||
|
|
|
||
|
|
if ACTION.nil? || !@actions.keys.include?(ACTION)
|
||
|
|
usage "Missing action"
|
||
|
|
end
|
||
|
|
|
||
|
|
@logs = Hash.new
|
||
|
|
|
||
|
|
tables = {
|
||
|
|
:assets => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'asset'
|
||
|
|
},
|
||
|
|
:automatic_geocodings => {
|
||
|
|
:related => ['geocodings'],
|
||
|
|
:singular => 'automatic_geocoding'
|
||
|
|
},
|
||
|
|
:client_applications => {
|
||
|
|
:related => ['oauth_tokens'],
|
||
|
|
:singular => 'client_application'
|
||
|
|
},
|
||
|
|
:data_imports => {
|
||
|
|
:related => ['user_tables'],
|
||
|
|
:singular => 'data_import'
|
||
|
|
},
|
||
|
|
:geocodings => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'geocoding'
|
||
|
|
},
|
||
|
|
:layers => {
|
||
|
|
:related => ['layers_maps', 'layers_users', 'layers_user_tables', 'visualizations'],
|
||
|
|
:singular => 'layer',
|
||
|
|
:relation_for => {'visualizations' => 'active_layer'}
|
||
|
|
},
|
||
|
|
:layers_maps => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'layer_map'
|
||
|
|
},
|
||
|
|
:layers_user_tables => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'layer_user_table'
|
||
|
|
},
|
||
|
|
:layers_users => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'layer_user'
|
||
|
|
},
|
||
|
|
:maps => {
|
||
|
|
:related => ['user_tables', 'layers_maps', 'visualizations'],
|
||
|
|
:singular => 'map'
|
||
|
|
},
|
||
|
|
:oauth_nonces => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'oauth_nonce'
|
||
|
|
},
|
||
|
|
:oauth_tokens => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'oauth_token'
|
||
|
|
},
|
||
|
|
:overlays => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'overlay'
|
||
|
|
},
|
||
|
|
:tags => {
|
||
|
|
:related => [],
|
||
|
|
:singular => 'tag'
|
||
|
|
},
|
||
|
|
:user_tables => {
|
||
|
|
:related => ['data_imports', 'layers_user_tables', 'tags', 'automatic_geocodings', 'geocodings'],
|
||
|
|
:singular => 'table',
|
||
|
|
:relation_for => {'layers_user_tables' => 'user_table'}
|
||
|
|
},
|
||
|
|
:users => {
|
||
|
|
:related => ['user_tables', 'maps', 'layers_users', 'assets', 'client_applications', 'oauth_tokens', 'tags', 'data_imports', 'synchronizations', 'geocodings'],
|
||
|
|
:singular => 'user'
|
||
|
|
},
|
||
|
|
#:visualizations => {
|
||
|
|
# :related => ['overlays'],
|
||
|
|
# :singular => 'visualization'
|
||
|
|
#}
|
||
|
|
}
|
||
|
|
|
||
|
|
redis_keys = {
|
||
|
|
:map_style => {
|
||
|
|
:template => "map_style|USERDB|*",
|
||
|
|
:var_position => 1,
|
||
|
|
:separator => '|',
|
||
|
|
:db => 0,
|
||
|
|
:type => 'string'
|
||
|
|
},
|
||
|
|
:table => {
|
||
|
|
:template => "rails:USERDB:*",
|
||
|
|
:var_position => 1,
|
||
|
|
:separator => ':',
|
||
|
|
:db => 0,
|
||
|
|
:type => 'hash',
|
||
|
|
:attributes => {
|
||
|
|
:user_id => 'USERID'
|
||
|
|
}
|
||
|
|
},
|
||
|
|
:user => {
|
||
|
|
:template => "rails:users:USERNAME",
|
||
|
|
:no_clone => true,
|
||
|
|
:var_position => 2,
|
||
|
|
:separator => ':',
|
||
|
|
:db => 5,
|
||
|
|
:type => 'hash',
|
||
|
|
:attributes => {
|
||
|
|
:database_name => 'USERDB',
|
||
|
|
:id => 'USERID'
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
def redis_replace_from_template(template, id, username)
|
||
|
|
if template.include?('USERDB')
|
||
|
|
user_database(id)
|
||
|
|
elsif template.include?('DBUSER')
|
||
|
|
database_username(id)
|
||
|
|
elsif template.include?('USERNAME')
|
||
|
|
username
|
||
|
|
elsif template.include?('USERID')
|
||
|
|
id
|
||
|
|
else
|
||
|
|
''
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def redis_template_user_gsub(template, id, username)
|
||
|
|
replacement = redis_replace_from_template(template, id, username)
|
||
|
|
if template.include?('USERDB')
|
||
|
|
template.gsub('USERDB', replacement)
|
||
|
|
elsif template.include?('DBUSER')
|
||
|
|
template.gsub('DBUSER', replacement)
|
||
|
|
elsif template.include?('USERNAME')
|
||
|
|
template.gsub('USERNAME', replacement)
|
||
|
|
else
|
||
|
|
''
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def copy_redis_keys(redis_keys, id, uuid, username)
|
||
|
|
redis = Redis.new(:host => REDIS_HOST)
|
||
|
|
redis_keys.each do |k,v|
|
||
|
|
redis.select(v[:db])
|
||
|
|
these_redis_keys = redis.keys(redis_template_user_gsub(v[:template], id, username))
|
||
|
|
these_redis_keys.each do |trd|
|
||
|
|
original_value = redis.dump(trd)
|
||
|
|
new_array = trd.split(v[:separator])
|
||
|
|
new_array[v[:var_position]] = redis_replace_from_template(v[:template], uuid, username)
|
||
|
|
new_key = new_array.join(v[:separator])
|
||
|
|
unless v[:no_clone]
|
||
|
|
redis.restore(new_key, 0, original_value)
|
||
|
|
end
|
||
|
|
if v[:type] == 'hash'
|
||
|
|
v[:attributes].each do |a,av|
|
||
|
|
redis.hset(new_key, a.to_s, redis_replace_from_template(av, uuid, username))
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
redis.quit
|
||
|
|
end
|
||
|
|
|
||
|
|
def alter_redis_hash(redis_key, redis_attribute, redis_value, options = {})
|
||
|
|
redis_db = options['db'].nil? ? 0 : options['db']
|
||
|
|
redis = Redis.new(:host => REDIS_HOST)
|
||
|
|
redis.select(redis_db)
|
||
|
|
redis.hset(redis_key, user_id, redis_value)
|
||
|
|
redis.quit
|
||
|
|
end
|
||
|
|
|
||
|
|
def relation_column_name_for(tables, table, related)
|
||
|
|
if tables[table][:related].include?(related) && tables[table][:relation_for] && tables[table][:relation_for][related]
|
||
|
|
tables[table][:relation_for][related]
|
||
|
|
else
|
||
|
|
tables[table][:singular]
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
|
||
|
|
def log(severity, type, msg)
|
||
|
|
puts " #{msg}. Ignoring.."
|
||
|
|
@logs.merge({:severity => severity, :type => type, :msg => msg})
|
||
|
|
end
|
||
|
|
|
||
|
|
def database_username(user_id)
|
||
|
|
"#{db_username_prefix}#{user_id}"
|
||
|
|
end #database_username
|
||
|
|
|
||
|
|
def user_database(user_id)
|
||
|
|
"#{database_name_prefix}#{user_id}_db"
|
||
|
|
end #user_database
|
||
|
|
|
||
|
|
def db_username_prefix
|
||
|
|
return "cartodb_user_" if ENVIRONMENT == 'production'
|
||
|
|
return "development_cartodb_user_" if ENVIRONMENT == 'development'
|
||
|
|
"cartodb_user_#{ENVIRONMENT}_"
|
||
|
|
end #username_prefix
|
||
|
|
|
||
|
|
def database_name_prefix
|
||
|
|
return "cartodb_user_" if ENVIRONMENT == 'production'
|
||
|
|
return "cartodb_dev_user_" if ENVIRONMENT == 'development'
|
||
|
|
"cartodb_#{ENVIRONMENT}_user_"
|
||
|
|
end #database_prefix
|
||
|
|
|
||
|
|
def alter_schema(tables)
|
||
|
|
tables.each do |tname, tinfo|
|
||
|
|
# Create main uuid column in every table
|
||
|
|
puts "Creating uuid column in #{tname}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{tname} ADD uuid uuid UNIQUE NOT NULL DEFAULT uuid_generate_v4()")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Creating uuid column in #{tname}", e.error.strip)
|
||
|
|
end
|
||
|
|
tinfo[:related].each do |rtable|
|
||
|
|
# Create relation uuid column in a dependent table
|
||
|
|
puts "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{rtable} ADD #{relation_column_name_for(tables, tname, rtable)}_uuid uuid")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def rollback_schema(tables)
|
||
|
|
tables.each do |tname, tinfo|
|
||
|
|
tinfo[:related].each do |rtable|
|
||
|
|
# Create relation uuid column in a dependent table
|
||
|
|
puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_uuid")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
# Destroy main uuid column in every table
|
||
|
|
puts "Dropping uuid column in #{tname}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{tname} DROP IF EXISTS uuid")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Dropping uuid column in #{tname}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def migrate_meta(tables)
|
||
|
|
tables.each do |tname, tinfo|
|
||
|
|
@conn.exec("SELECT id,uuid FROM #{tname}") do |result|
|
||
|
|
result.each do |row|
|
||
|
|
tinfo[:related].each do |rtable|
|
||
|
|
puts "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}"
|
||
|
|
begin
|
||
|
|
@conn.exec("UPDATE #{rtable} SET #{relation_column_name_for(tables, tname, rtable)}_uuid='#{row['uuid']}' WHERE #{relation_column_name_for(tables, tname, rtable)}_id='#{row['id']}'")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
|
||
|
|
def migrate_data(redis_keys)
|
||
|
|
sconn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: 'postgres' )
|
||
|
|
@conn.exec("SELECT id,uuid,database_name,username FROM users") do |result|
|
||
|
|
result.each do |row|
|
||
|
|
puts "Renaming pg user and db for id #{row['id']}"
|
||
|
|
begin
|
||
|
|
sconn.exec("ALTER DATABASE \"#{row['database_name']}\" RENAME TO \"#{user_database(row['uuid'])}\"")
|
||
|
|
sconn.exec("ALTER ROLE \"#{database_username(row['id'])}\" RENAME TO \"#{database_username(row['uuid'])}\"")
|
||
|
|
@conn.exec("UPDATE users SET database_name='#{user_database(row['uuid'])}' WHERE id=#{row['id']} AND uuid='#{row['uuid']}'")
|
||
|
|
@conn.exec("UPDATE user_tables SET database_name='#{user_database(row['uuid'])}' WHERE user_id='#{row['id']}' AND user_uuid='#{row['uuid']}'")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Renaming pg user and db for id #{row['id']}", e.error.strip)
|
||
|
|
end
|
||
|
|
puts "Copying redis keys with uuid for id #{row['id']}"
|
||
|
|
#begin
|
||
|
|
copy_redis_keys(redis_keys, row['id'], row['uuid'], row['username'])
|
||
|
|
#rescue => e
|
||
|
|
# log('C', "Copying redis keys with uuid for id #{row['id']}", e.error.strip)
|
||
|
|
#end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
@conn.exec("SELECT token,user_id FROM oauth_tokens WHERE type='AccessToken'") do |result|
|
||
|
|
result.each do |row|
|
||
|
|
puts "Chaing user_id for oauth token '#{row['token']}'"
|
||
|
|
begin
|
||
|
|
alter_redis_hash("rails:oauth_access_tokens:#{row['token']}", 'user_id', row['user_id'], {'db' => 3})
|
||
|
|
rescue => e
|
||
|
|
log('C', "Changing user id to uuid in oauth token #{row['token']}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def clean_db(tables)
|
||
|
|
tables.each do |tname, tinfo|
|
||
|
|
tinfo[:related].each do |rtable|
|
||
|
|
# Drop old id relation column in every table
|
||
|
|
puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_id")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}", e.error.strip)
|
||
|
|
end
|
||
|
|
# Rename new uuid relation column to id
|
||
|
|
puts "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{rtable} RENAME #{relation_column_name_for(tables, tname, rtable)}_uuid TO #{relation_column_name_for(tables, tname, rtable)}_id")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
# Drop old id column in every table
|
||
|
|
puts "Dropping old id from #{tname}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{tname} DROP IF EXISTS id")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Dropping old id from #{rtable}", e.error.strip)
|
||
|
|
end
|
||
|
|
# Rename new uuid relation column to id
|
||
|
|
puts "Renaming uuid to id in #{tname}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{tname} RENAME uuid TO id")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Renaming uuid to id in #{tname}", e.error.strip)
|
||
|
|
end
|
||
|
|
# Set new id as primary key
|
||
|
|
puts "Setting new id as primary key on #{tname}"
|
||
|
|
begin
|
||
|
|
@conn.exec("ALTER TABLE #{tname} ADD PRIMARY KEY (id)")
|
||
|
|
rescue => e
|
||
|
|
log('C', "Setting new id as primary key on #{tname}", e.error.strip)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
execution_summary
|
||
|
|
|
||
|
|
@conn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: DBNAME )
|
||
|
|
@conn.exec("CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"")
|
||
|
|
|
||
|
|
if ACTION == 'schema'
|
||
|
|
alter_schema(tables)
|
||
|
|
elsif ACTION == 'rollback'
|
||
|
|
rollback_schema(tables)
|
||
|
|
elsif ACTION == 'meta'
|
||
|
|
migrate_meta(tables)
|
||
|
|
elsif ACTION == 'data'
|
||
|
|
migrate_data(redis_keys)
|
||
|
|
elsif ACTION == 'clean'
|
||
|
|
clean_db(tables)
|
||
|
|
end
|
||
|
|
|
||
|
|
puts ""
|
||
|
|
puts "#############"
|
||
|
|
puts "#{@logs.length} errors"
|
||
|
|
puts "#############"
|