cartodb-4.42/app/models/carto/user_migration_export.rb

require 'active_record'
require 'fileutils'
require_relative '../../../services/user-mover/export_user'
require_dependency 'file_upload'
require_dependency 'resque/user_migration_jobs'
require_dependency 'carto/ghost_tables_manager'

module Carto
  class UserMigrationExport < ::ActiveRecord::Base
    belongs_to :organization, class_name: Carto::Organization
    belongs_to :user, class_name: Carto::User
    belongs_to :log, class_name: Carto::Log

    STATE_PENDING = 'pending'.freeze
    STATE_EXPORTING = 'exporting'.freeze
    STATE_UPLOADING = 'uploading'.freeze
    STATE_COMPLETE = 'complete'.freeze
    STATE_FAILURE = 'failure'.freeze
    VALID_STATES = [STATE_PENDING, STATE_EXPORTING, STATE_UPLOADING, STATE_COMPLETE, STATE_FAILURE].freeze

    after_initialize :set_defaults

    validates :state, inclusion: { in: VALID_STATES }
    validate  :user_or_organization_present
    validate  :validate_export_data

    def run_export
      check_valid_user(user) if user && export_metadata
      check_valid_organization(organization) if organization && export_metadata
      check_custom_plpython2_functions

      log.append("=== Exporting #{organization ? 'user' : 'org'} data ===")
      update_attributes(state: STATE_EXPORTING)

      package = if backup
                  UserMigrationPackage.for_backup("backup_#{user.username}_#{Time.now.iso8601}", log)
                else
                  UserMigrationPackage.for_export(id, log)
                end

      export_job = CartoDB::DataMover::ExportJob.new(export_job_arguments(package.data_dir)) if export_data?

      run_metadata_export(package.meta_dir) if export_metadata?

      log.append("=== Uploading ===")
      update_attributes(
        state: STATE_UPLOADING,
        json_file: export_data? ? "#{id}/#{export_job.json_file}" : 'no_data_exported'
      )
      uploaded_path = package.upload

      state = uploaded_path.present? ? STATE_COMPLETE : STATE_FAILURE
      log.append("=== Finishing. State: #{state}. File: #{uploaded_path} ===")
      update_attributes(state: state, exported_file: uploaded_path)

      true
    rescue StandardError => e
      log.append_exception('Exporting', exception: e)
      log_error(exception: e, message: 'Error exporting user data', error_detail: inspect)
      update_attributes(state: STATE_FAILURE)

      false
    ensure
      package.try(:cleanup)
    end

    def enqueue
      Resque.enqueue(Resque::UserMigrationJobs::Export, export_id: id)
    end

    private

    # TODO: delete this once migration to plpython3 is completed
    def check_custom_plpython2_functions
      target_user = user || organization.owner

      pg_result = target_user.in_database(as: :superuser).execute(%{
        select
          nspname, proname
        from
          pg_catalog.pg_proc p
          join pg_catalog.pg_language l on p.prolang = l.oid
          join pg_namespace on p.pronamespace = pg_namespace.oid
        where
          lanname in ('plpythonu', 'plpython2u') and
          nspname not in ('cartodb', 'cdb_dataservices_client', 'cdb_crankshaft') and
          (nspname = 'public' and proname not in ('cdb_invalidate_varnish', 'update_timestamp'))
      })

      raise "Can't migrate custom plpython2 functions" if pg_result.ntuples > 0
    end

    def check_valid_user(user)
      Carto::GhostTablesManager.new(user.id).link_ghost_tables_synchronously
    end

    def check_valid_organization(organization)
      organization.users.each do |user|
        check_valid_user(user)
      end
    end

    def run_metadata_export(meta_dir)
      if organization
        Carto::OrganizationMetadataExportService.new.export_to_directory(organization, meta_dir)
      elsif user
        Carto::UserMetadataExportService.new.export_to_directory(user, meta_dir)
      else
        raise 'Unrecognized export type for exporting metadata'
      end
    end

    def export_job_arguments(data_dir)
      args = if user.present?
               {
                 id: user.username,
                 schema_mode: false
               }
             else
               {
                 organization_name: organization.name,
                 schema_mode: true,
                 split_user_schemas: false
               }
             end

      args.merge(
        path: data_dir,
        job_uuid: id,
        export_job_logger: log.logger,
        logger: log.logger,
        metadata: false,
        data: export_data?
      )
    end

    def user_or_organization_present
      unless (user.present? && organization.blank?) || (organization.present? && user.blank?)
        errors.add(:user, 'exactly one user or organization required')
      end
    end

    def validate_export_data
      errors.add(:export_metadata, 'needs to be true if export_data is set to false') if !export_data? && !export_metadata?
    end

    def set_defaults
      self.log = Carto::Log.create(type: 'user_migration_export') unless log
      self.state = STATE_PENDING unless state
      save
    end
  end
end