329 lines
12 KiB
Ruby
329 lines
12 KiB
Ruby
|
require 'json'
|
||
|
|
require_dependency 'carto/export/layer_exporter'
|
||
|
|
require_dependency 'carto/export/connector_configuration_exporter'
|
||
|
|
|
||
|
|
# Not migrated
|
||
|
|
# invitations -> temporary by nature
|
||
|
|
# ldap_configurations -> not enabled in SaaS
|
||
|
|
|
||
|
|
# Version History
|
||
|
|
# 1.0.0: export organization metadata
|
||
|
|
# 1.0.1: export password expiration
|
||
|
|
# 1.0.2: export connector configurations
|
||
|
|
# 1.0.3: export oauth_app_organizations
|
||
|
|
# 1.0.4: export inherit_owner_ffs
|
||
|
|
module Carto
|
||
|
|
module OrganizationMetadataExportServiceConfiguration
|
||
|
|
CURRENT_VERSION = '1.0.4'.freeze
|
||
|
|
EXPORTED_ORGANIZATION_ATTRIBUTES = [
|
||
|
|
:id, :seats, :quota_in_bytes, :created_at, :updated_at, :name, :avatar_url, :owner_id, :website, :description,
|
||
|
|
:display_name, :discus_shortname, :twitter_username, :geocoding_quota, :map_view_quota, :auth_token,
|
||
|
|
:geocoding_block_price, :map_view_block_price, :twitter_datasource_enabled, :twitter_datasource_block_price,
|
||
|
|
:twitter_datasource_block_size, :twitter_datasource_quota, :google_maps_key, :google_maps_private_key, :color,
|
||
|
|
:default_quota_in_bytes, :whitelisted_email_domains, :admin_email, :auth_username_password_enabled,
|
||
|
|
:auth_google_enabled, :location, :here_isolines_quota, :here_isolines_block_price, :strong_passwords_enabled,
|
||
|
|
:obs_snapshot_quota, :obs_snapshot_block_price, :obs_general_quota, :obs_general_block_price,
|
||
|
|
:salesforce_datasource_enabled, :viewer_seats, :geocoder_provider, :isolines_provider, :routing_provider,
|
||
|
|
:auth_github_enabled, :engine_enabled, :mapzen_routing_quota, :mapzen_routing_block_price, :builder_enabled,
|
||
|
|
:auth_saml_configuration, :no_map_logo, :password_expiration_in_d, :inherit_owner_ffs
|
||
|
|
].freeze
|
||
|
|
|
||
|
|
def compatible_version?(version)
|
||
|
|
version.to_i == CURRENT_VERSION.split('.')[0].to_i
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
module OrganizationMetadataExportServiceImporter
|
||
|
|
include OrganizationMetadataExportServiceConfiguration
|
||
|
|
include LayerImporter
|
||
|
|
include ConnectorConfigurationImporter
|
||
|
|
|
||
|
|
def build_organization_from_json_export(exported_json_string)
|
||
|
|
build_organization_from_hash_export(JSON.parse(exported_json_string, symbolize_names: true))
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_organization_from_hash_export(exported_hash)
|
||
|
|
raise 'Wrong export version' unless compatible_version?(exported_hash[:version])
|
||
|
|
|
||
|
|
build_organization_from_hash(exported_hash[:organization])
|
||
|
|
end
|
||
|
|
|
||
|
|
private
|
||
|
|
|
||
|
|
def save_imported_organization(organization)
|
||
|
|
organization.save!
|
||
|
|
::Organization[organization.id].after_save
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_organization_from_hash(exported_organization)
|
||
|
|
organization = Organization.new(exported_organization.slice(*EXPORTED_ORGANIZATION_ATTRIBUTES - [:id]))
|
||
|
|
organization.assets = exported_organization[:assets].map { |asset| build_asset_from_hash(asset.symbolize_keys) }
|
||
|
|
organization.groups = exported_organization[:groups].map { |group| build_group_from_hash(group.symbolize_keys) }
|
||
|
|
organization.notifications = exported_organization[:notifications].map do |notification|
|
||
|
|
build_notification_from_hash(notification.symbolize_keys)
|
||
|
|
end
|
||
|
|
if exported_organization[:oauth_app_organizations]
|
||
|
|
organization.oauth_app_organizations = exported_organization[:oauth_app_organizations].map do |oao|
|
||
|
|
build_oauth_app_organization_from_hash(oao.symbolize_keys)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
organization.connector_configurations = build_connector_configurations_from_hash(
|
||
|
|
exported_organization[:connector_configurations]
|
||
|
|
)
|
||
|
|
|
||
|
|
# Must be the last one to avoid attribute assignments to try to run SQL
|
||
|
|
organization.id = exported_organization[:id]
|
||
|
|
organization
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_asset_from_hash(exported_asset)
|
||
|
|
Asset.new(
|
||
|
|
public_url: exported_asset[:public_url],
|
||
|
|
kind: exported_asset[:kind],
|
||
|
|
storage_info: exported_asset[:storage_info]
|
||
|
|
)
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_group_from_hash(exported_group)
|
||
|
|
g = Group.new_instance_without_validation(
|
||
|
|
name: exported_group[:name],
|
||
|
|
display_name: exported_group[:display_name],
|
||
|
|
database_role: exported_group[:database_role],
|
||
|
|
auth_token: exported_group[:auth_token]
|
||
|
|
)
|
||
|
|
g.users_group = exported_group[:user_ids].map { |uid| UsersGroup.new(user_id: uid) }
|
||
|
|
g.id = exported_group[:id]
|
||
|
|
|
||
|
|
g
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_notification_from_hash(notification)
|
||
|
|
Notification.new(
|
||
|
|
icon: notification[:icon],
|
||
|
|
recipients: notification[:recipients],
|
||
|
|
body: notification[:body],
|
||
|
|
created_at: notification[:created_at],
|
||
|
|
received_notifications: notification[:received_by].map do |received_notification|
|
||
|
|
build_received_notification_from_hash(received_notification.symbolize_keys)
|
||
|
|
end
|
||
|
|
)
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_received_notification_from_hash(received_notification)
|
||
|
|
ReceivedNotification.new(
|
||
|
|
user_id: received_notification[:user_id],
|
||
|
|
received_at: received_notification[:received_at],
|
||
|
|
read_at: received_notification[:read_at]
|
||
|
|
)
|
||
|
|
end
|
||
|
|
|
||
|
|
def build_oauth_app_organization_from_hash(oao_hash)
|
||
|
|
Carto::OauthAppOrganization.new(
|
||
|
|
oauth_app_id: oao_hash[:oauth_app_id],
|
||
|
|
organization_id: oao_hash[:organization_id],
|
||
|
|
seats: oao_hash[:seats],
|
||
|
|
created_at: oao_hash[:created_at],
|
||
|
|
updated_at: oao_hash[:updated_at]
|
||
|
|
)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
module OrganizationMetadataExportServiceExporter
|
||
|
|
include OrganizationMetadataExportServiceConfiguration
|
||
|
|
include LayerExporter
|
||
|
|
include ConnectorConfigurationExporter
|
||
|
|
|
||
|
|
def export_organization_json_string(organization)
|
||
|
|
export_organization_json_hash(organization).to_json
|
||
|
|
end
|
||
|
|
|
||
|
|
def export_organization_json_hash(organization)
|
||
|
|
{
|
||
|
|
version: CURRENT_VERSION,
|
||
|
|
organization: export(organization)
|
||
|
|
}
|
||
|
|
end
|
||
|
|
|
||
|
|
private
|
||
|
|
|
||
|
|
def export(organization)
|
||
|
|
organization_hash = EXPORTED_ORGANIZATION_ATTRIBUTES.map { |att| [att, organization.attributes[att.to_s]] }.to_h
|
||
|
|
|
||
|
|
organization_hash[:assets] = organization.assets.map { |a| export_asset(a) }
|
||
|
|
organization_hash[:groups] = organization.groups.map { |g| export_group(g) }
|
||
|
|
organization_hash[:notifications] = organization.notifications.map { |n| export_notification(n) }
|
||
|
|
organization_hash[:connector_configurations] = organization.connector_configurations.map do |cc|
|
||
|
|
export_connector_configuration(cc)
|
||
|
|
end
|
||
|
|
organization_hash[:oauth_app_organizations] = organization.oauth_app_organizations.map do |oao|
|
||
|
|
export_oauth_app_organization(oao)
|
||
|
|
end
|
||
|
|
|
||
|
|
organization_hash
|
||
|
|
end
|
||
|
|
|
||
|
|
def export_asset(asset)
|
||
|
|
{
|
||
|
|
public_url: asset.public_url,
|
||
|
|
kind: asset.kind,
|
||
|
|
storage_info: asset.storage_info
|
||
|
|
}
|
||
|
|
end
|
||
|
|
|
||
|
|
def export_group(group)
|
||
|
|
{
|
||
|
|
id: group.id,
|
||
|
|
name: group.name,
|
||
|
|
display_name: group.display_name,
|
||
|
|
database_role: group.database_role,
|
||
|
|
auth_token: group.auth_token,
|
||
|
|
user_ids: group.users.map(&:id)
|
||
|
|
}
|
||
|
|
end
|
||
|
|
|
||
|
|
def export_notification(notification)
|
||
|
|
{
|
||
|
|
icon: notification.icon,
|
||
|
|
recipients: notification.recipients,
|
||
|
|
body: notification.body,
|
||
|
|
created_at: notification.created_at,
|
||
|
|
received_by: notification.received_notifications.map { |rn| export_received_notification(rn) }
|
||
|
|
}
|
||
|
|
end
|
||
|
|
|
||
|
|
def export_received_notification(received_notification)
|
||
|
|
{
|
||
|
|
user_id: received_notification.user_id,
|
||
|
|
received_at: received_notification.received_at,
|
||
|
|
read_at: received_notification.read_at
|
||
|
|
}
|
||
|
|
end
|
||
|
|
|
||
|
|
def export_oauth_app_organization(oao)
|
||
|
|
{
|
||
|
|
oauth_app_id: oao.oauth_app_id,
|
||
|
|
organization_id: oao.organization_id,
|
||
|
|
seats: oao.seats,
|
||
|
|
created_at: oao.created_at,
|
||
|
|
updated_at: oao.updated_at
|
||
|
|
}
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
class OrganizationAlreadyExists < RuntimeError; end
|
||
|
|
# Both String and Hash versions are provided because `deep_symbolize_keys` won't symbolize through arrays
|
||
|
|
# and having separated methods make handling and testing much easier.
|
||
|
|
class OrganizationMetadataExportService
|
||
|
|
include OrganizationMetadataExportServiceImporter
|
||
|
|
include OrganizationMetadataExportServiceExporter
|
||
|
|
|
||
|
|
def export_to_directory(organization, path)
|
||
|
|
root_dir = Pathname.new(path)
|
||
|
|
|
||
|
|
# Export organization
|
||
|
|
organization_json = export_organization_json_string(organization)
|
||
|
|
root_dir.join("organization_#{organization.id}.json").open('w') { |file| file.write(organization_json) }
|
||
|
|
|
||
|
|
redis_json = Carto::RedisExportService.new.export_organization_json_string(organization)
|
||
|
|
root_dir.join("redis_organization_#{organization.id}.json").open('w') { |file| file.write(redis_json) }
|
||
|
|
|
||
|
|
# Export users
|
||
|
|
organization.users.each do |user|
|
||
|
|
Carto::UserMetadataExportService.new.export_to_directory(user, root_dir.join("user_#{user.id}"))
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def import_from_directory(meta_path)
|
||
|
|
# Import organization
|
||
|
|
organization = load_organization_from_directory(meta_path)
|
||
|
|
raise OrganizationAlreadyExists.new if ::Carto::Organization.exists?(id: organization.id)
|
||
|
|
|
||
|
|
organization_redis_file = redis_filename(meta_path)
|
||
|
|
Carto::RedisExportService.new.restore_redis_from_json_export(File.read(organization_redis_file))
|
||
|
|
|
||
|
|
# Groups and notifications must be saved after users
|
||
|
|
groups = organization.groups.map(&:clone)
|
||
|
|
organization.groups.clear
|
||
|
|
notifications = organization.notifications.map(&:clone)
|
||
|
|
organization.notifications.clear
|
||
|
|
oauth_app_organizations = organization.oauth_app_organizations.map(&:clone)
|
||
|
|
organization.oauth_app_organizations.clear
|
||
|
|
|
||
|
|
save_imported_organization(organization)
|
||
|
|
|
||
|
|
user_list = get_user_list(meta_path)
|
||
|
|
|
||
|
|
# In order to get permissions right, we first import all users, then all datasets and finally, all maps
|
||
|
|
organization.users = user_list.map do |user_path|
|
||
|
|
Carto::UserMetadataExportService.new.import_from_directory(user_path)
|
||
|
|
end
|
||
|
|
|
||
|
|
organization.groups = groups
|
||
|
|
organization.notifications = notifications
|
||
|
|
organization.oauth_app_organizations = oauth_app_organizations
|
||
|
|
organization.save
|
||
|
|
|
||
|
|
organization
|
||
|
|
end
|
||
|
|
|
||
|
|
def rollback_import_from_directory(meta_path)
|
||
|
|
organization_redis_file = redis_filename(meta_path)
|
||
|
|
Carto::RedisExportService.new.remove_redis_from_json_export(File.read(organization_redis_file))
|
||
|
|
organization = load_organization_from_directory(meta_path)
|
||
|
|
|
||
|
|
user_list = organization.non_owner_users + [organization.owner]
|
||
|
|
user_list.map do |user|
|
||
|
|
Carto::UserMetadataExportService.new.rollback_import_from_directory("#{meta_path}/user_#{user.id}")
|
||
|
|
end
|
||
|
|
return unless Carto::Organization.exists?(organization.id)
|
||
|
|
|
||
|
|
organization = Carto::Organization.find(organization.id)
|
||
|
|
organization.groups.delete
|
||
|
|
organization.notifications.delete
|
||
|
|
organization.oauth_app_organizations.delete
|
||
|
|
organization.assets.map(&:delete)
|
||
|
|
organization.users.delete
|
||
|
|
organization.delete
|
||
|
|
end
|
||
|
|
|
||
|
|
def get_user_list(meta_path)
|
||
|
|
Dir["#{meta_path}/user_*"]
|
||
|
|
end
|
||
|
|
|
||
|
|
def redis_filename(meta_path)
|
||
|
|
Dir["#{meta_path}/redis_organization_*.json"].first
|
||
|
|
end
|
||
|
|
|
||
|
|
def load_organization_from_directory(meta_path)
|
||
|
|
organization_file = Dir["#{meta_path}/organization_*.json"].first
|
||
|
|
build_organization_from_json_export(File.read(organization_file))
|
||
|
|
end
|
||
|
|
|
||
|
|
def import_metadata_from_directory(organization, path)
|
||
|
|
organization.users.each do |user|
|
||
|
|
Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
|
||
|
|
user, Carto::Visualization::TYPE_REMOTE, "#{path}/user_#{user.id}"
|
||
|
|
)
|
||
|
|
|
||
|
|
Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
|
||
|
|
user, Carto::Visualization::TYPE_CANONICAL, "#{path}/user_#{user.id}"
|
||
|
|
)
|
||
|
|
end
|
||
|
|
|
||
|
|
# Derived must be the last because of shared canonicals
|
||
|
|
organization.users.each do |user|
|
||
|
|
Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
|
||
|
|
user, Carto::Visualization::TYPE_DERIVED, "#{path}/user_#{user.id}"
|
||
|
|
)
|
||
|
|
end
|
||
|
|
|
||
|
|
organization.users.each do |user|
|
||
|
|
Carto::UserMetadataExportService.new.import_search_tweets_from_directory(user, "#{path}/user_#{user.id}")
|
||
|
|
end
|
||
|
|
|
||
|
|
organization
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|