cartodb-4.42/app/services/carto/organization_metadata_export_service.rb

require 'json'
require_dependency 'carto/export/layer_exporter'
require_dependency 'carto/export/connector_configuration_exporter'

# Not migrated
# invitations -> temporary by nature
# ldap_configurations -> not enabled in SaaS

# Version History
# 1.0.0: export organization metadata
# 1.0.1: export password expiration
# 1.0.2: export connector configurations
# 1.0.3: export oauth_app_organizations
# 1.0.4: export inherit_owner_ffs
module Carto
  module OrganizationMetadataExportServiceConfiguration
    CURRENT_VERSION = '1.0.4'.freeze
    EXPORTED_ORGANIZATION_ATTRIBUTES = [
      :id, :seats, :quota_in_bytes, :created_at, :updated_at, :name, :avatar_url, :owner_id, :website, :description,
      :display_name, :discus_shortname, :twitter_username, :geocoding_quota, :map_view_quota, :auth_token,
      :geocoding_block_price, :map_view_block_price, :twitter_datasource_enabled, :twitter_datasource_block_price,
      :twitter_datasource_block_size, :twitter_datasource_quota, :google_maps_key, :google_maps_private_key, :color,
      :default_quota_in_bytes, :whitelisted_email_domains, :admin_email, :auth_username_password_enabled,
      :auth_google_enabled, :location, :here_isolines_quota, :here_isolines_block_price, :strong_passwords_enabled,
      :obs_snapshot_quota, :obs_snapshot_block_price, :obs_general_quota, :obs_general_block_price,
      :salesforce_datasource_enabled, :viewer_seats, :geocoder_provider, :isolines_provider, :routing_provider,
      :auth_github_enabled, :engine_enabled, :mapzen_routing_quota, :mapzen_routing_block_price, :builder_enabled,
      :auth_saml_configuration, :no_map_logo, :password_expiration_in_d, :inherit_owner_ffs
    ].freeze

    def compatible_version?(version)
      version.to_i == CURRENT_VERSION.split('.')[0].to_i
    end
  end

  module OrganizationMetadataExportServiceImporter
    include OrganizationMetadataExportServiceConfiguration
    include LayerImporter
    include ConnectorConfigurationImporter

    def build_organization_from_json_export(exported_json_string)
      build_organization_from_hash_export(JSON.parse(exported_json_string, symbolize_names: true))
    end

    def build_organization_from_hash_export(exported_hash)
      raise 'Wrong export version' unless compatible_version?(exported_hash[:version])

      build_organization_from_hash(exported_hash[:organization])
    end

    private

    def save_imported_organization(organization)
      organization.save!
      ::Organization[organization.id].after_save
    end

    def build_organization_from_hash(exported_organization)
      organization = Organization.new(exported_organization.slice(*EXPORTED_ORGANIZATION_ATTRIBUTES - [:id]))
      organization.assets = exported_organization[:assets].map { |asset| build_asset_from_hash(asset.symbolize_keys) }
      organization.groups = exported_organization[:groups].map { |group| build_group_from_hash(group.symbolize_keys) }
      organization.notifications = exported_organization[:notifications].map do |notification|
        build_notification_from_hash(notification.symbolize_keys)
      end
      if exported_organization[:oauth_app_organizations]
        organization.oauth_app_organizations = exported_organization[:oauth_app_organizations].map do |oao|
          build_oauth_app_organization_from_hash(oao.symbolize_keys)
        end
      end

      organization.connector_configurations = build_connector_configurations_from_hash(
        exported_organization[:connector_configurations]
      )

      # Must be the last one to avoid attribute assignments to try to run SQL
      organization.id = exported_organization[:id]
      organization
    end

    def build_asset_from_hash(exported_asset)
      Asset.new(
        public_url: exported_asset[:public_url],
        kind: exported_asset[:kind],
        storage_info: exported_asset[:storage_info]
      )
    end

    def build_group_from_hash(exported_group)
      g = Group.new_instance_without_validation(
        name: exported_group[:name],
        display_name: exported_group[:display_name],
        database_role: exported_group[:database_role],
        auth_token: exported_group[:auth_token]
      )
      g.users_group = exported_group[:user_ids].map { |uid| UsersGroup.new(user_id: uid) }
      g.id = exported_group[:id]

      g
    end

    def build_notification_from_hash(notification)
      Notification.new(
        icon: notification[:icon],
        recipients: notification[:recipients],
        body: notification[:body],
        created_at: notification[:created_at],
        received_notifications: notification[:received_by].map do |received_notification|
          build_received_notification_from_hash(received_notification.symbolize_keys)
        end
      )
    end

    def build_received_notification_from_hash(received_notification)
      ReceivedNotification.new(
        user_id: received_notification[:user_id],
        received_at: received_notification[:received_at],
        read_at: received_notification[:read_at]
      )
    end

    def build_oauth_app_organization_from_hash(oao_hash)
      Carto::OauthAppOrganization.new(
        oauth_app_id: oao_hash[:oauth_app_id],
        organization_id: oao_hash[:organization_id],
        seats: oao_hash[:seats],
        created_at: oao_hash[:created_at],
        updated_at: oao_hash[:updated_at]
      )
    end
  end

  module OrganizationMetadataExportServiceExporter
    include OrganizationMetadataExportServiceConfiguration
    include LayerExporter
    include ConnectorConfigurationExporter

    def export_organization_json_string(organization)
      export_organization_json_hash(organization).to_json
    end

    def export_organization_json_hash(organization)
      {
        version: CURRENT_VERSION,
        organization: export(organization)
      }
    end

    private

    def export(organization)
      organization_hash = EXPORTED_ORGANIZATION_ATTRIBUTES.map { |att| [att, organization.attributes[att.to_s]] }.to_h

      organization_hash[:assets] = organization.assets.map { |a| export_asset(a) }
      organization_hash[:groups] = organization.groups.map { |g| export_group(g) }
      organization_hash[:notifications] = organization.notifications.map { |n| export_notification(n) }
      organization_hash[:connector_configurations] = organization.connector_configurations.map do |cc|
        export_connector_configuration(cc)
      end
      organization_hash[:oauth_app_organizations] = organization.oauth_app_organizations.map do |oao|
        export_oauth_app_organization(oao)
      end

      organization_hash
    end

    def export_asset(asset)
      {
        public_url: asset.public_url,
        kind: asset.kind,
        storage_info: asset.storage_info
      }
    end

    def export_group(group)
      {
        id: group.id,
        name: group.name,
        display_name: group.display_name,
        database_role: group.database_role,
        auth_token: group.auth_token,
        user_ids: group.users.map(&:id)
      }
    end

    def export_notification(notification)
      {
        icon: notification.icon,
        recipients: notification.recipients,
        body: notification.body,
        created_at: notification.created_at,
        received_by: notification.received_notifications.map { |rn| export_received_notification(rn) }
      }
    end

    def export_received_notification(received_notification)
      {
        user_id: received_notification.user_id,
        received_at: received_notification.received_at,
        read_at: received_notification.read_at
      }
    end

    def export_oauth_app_organization(oao)
      {
        oauth_app_id: oao.oauth_app_id,
        organization_id: oao.organization_id,
        seats: oao.seats,
        created_at: oao.created_at,
        updated_at: oao.updated_at
      }
    end
  end

  class OrganizationAlreadyExists < RuntimeError; end
  # Both String and Hash versions are provided because `deep_symbolize_keys` won't symbolize through arrays
  # and having separated methods make handling and testing much easier.
  class OrganizationMetadataExportService
    include OrganizationMetadataExportServiceImporter
    include OrganizationMetadataExportServiceExporter

    def export_to_directory(organization, path)
      root_dir = Pathname.new(path)

      # Export organization
      organization_json = export_organization_json_string(organization)
      root_dir.join("organization_#{organization.id}.json").open('w') { |file| file.write(organization_json) }

      redis_json = Carto::RedisExportService.new.export_organization_json_string(organization)
      root_dir.join("redis_organization_#{organization.id}.json").open('w') { |file| file.write(redis_json) }

      # Export users
      organization.users.each do |user|
        Carto::UserMetadataExportService.new.export_to_directory(user, root_dir.join("user_#{user.id}"))
      end
    end

    def import_from_directory(meta_path)
      # Import organization
      organization = load_organization_from_directory(meta_path)
      raise OrganizationAlreadyExists.new if ::Carto::Organization.exists?(id: organization.id)

      organization_redis_file = redis_filename(meta_path)
      Carto::RedisExportService.new.restore_redis_from_json_export(File.read(organization_redis_file))

      # Groups and notifications must be saved after users
      groups = organization.groups.map(&:clone)
      organization.groups.clear
      notifications = organization.notifications.map(&:clone)
      organization.notifications.clear
      oauth_app_organizations = organization.oauth_app_organizations.map(&:clone)
      organization.oauth_app_organizations.clear

      save_imported_organization(organization)

      user_list = get_user_list(meta_path)

      # In order to get permissions right, we first import all users, then all datasets and finally, all maps
      organization.users = user_list.map do |user_path|
        Carto::UserMetadataExportService.new.import_from_directory(user_path)
      end

      organization.groups = groups
      organization.notifications = notifications
      organization.oauth_app_organizations = oauth_app_organizations
      organization.save

      organization
    end

    def rollback_import_from_directory(meta_path)
      organization_redis_file = redis_filename(meta_path)
      Carto::RedisExportService.new.remove_redis_from_json_export(File.read(organization_redis_file))
      organization = load_organization_from_directory(meta_path)

      user_list = organization.non_owner_users + [organization.owner]
      user_list.map do |user|
        Carto::UserMetadataExportService.new.rollback_import_from_directory("#{meta_path}/user_#{user.id}")
      end
      return unless Carto::Organization.exists?(organization.id)

      organization = Carto::Organization.find(organization.id)
      organization.groups.delete
      organization.notifications.delete
      organization.oauth_app_organizations.delete
      organization.assets.map(&:delete)
      organization.users.delete
      organization.delete
    end

    def get_user_list(meta_path)
      Dir["#{meta_path}/user_*"]
    end

    def redis_filename(meta_path)
      Dir["#{meta_path}/redis_organization_*.json"].first
    end

    def load_organization_from_directory(meta_path)
      organization_file = Dir["#{meta_path}/organization_*.json"].first
      build_organization_from_json_export(File.read(organization_file))
    end

    def import_metadata_from_directory(organization, path)
      organization.users.each do |user|
        Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
          user, Carto::Visualization::TYPE_REMOTE, "#{path}/user_#{user.id}"
        )

        Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
          user, Carto::Visualization::TYPE_CANONICAL, "#{path}/user_#{user.id}"
        )
      end

      # Derived must be the last because of shared canonicals
      organization.users.each do |user|
        Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
          user, Carto::Visualization::TYPE_DERIVED, "#{path}/user_#{user.id}"
        )
      end

      organization.users.each do |user|
        Carto::UserMetadataExportService.new.import_search_tweets_from_directory(user, "#{path}/user_#{user.id}")
      end

      organization
    end
  end
end