require_relative '../../spec_helper_min'
require_relative '../../../app/models/carto/user_migration_import'
require_relative '../../../app/models/carto/user_migration_export'
require_relative '../../support/factories/tables'
require_relative '../../factories/organizations_contexts'
require 'helpers/database_connection_helper'
require 'factories/carto_visualizations'

describe 'UserMigration' do
  include Carto::Factories::Visualizations
  include CartoDB::Factories
  include DatabaseConnectionHelper

  let(:records) do
    [
      { name: 'carto', description: 'awesome' },
      { name: 'user-mover', description: 'insanity' }
    ]
  end

  let(:agg_ds_config) do
    {
      aggregation_tables: {
        'host' => 'localhost',
        'port' => '5432',
        'dbname' => 'test_migration',
        'username' => 'geocoder_api',
        'password' => '',
        'tables' => {
          'admin0' => 'ne_admin0_v3',
          'admin1' => 'global_province_polygons'
        }
      },
      geocoder: {
        'api' => {
          'host' => 'localhost',
          'port' => '5432',
          'dbname' => 'test_migration',
          'user' => 'geocoder_api'
        }
      }
    }
  end

  shared_examples_for 'migrating metadata' do |migrate_metadata|

    before :each do
      @user = FactoryGirl.build(:valid_user).save
      @carto_user = Carto::User.find(@user.id)
      @user_attributes = @carto_user.attributes

      @table1 = create_table(user_id: @user.id)
      records.each { |row| @table1.insert_row!(row) }
      create_database('test_migration', @user) if migrate_metadata
    end

    after :each do
      if migrate_metadata
        @user.destroy_cascade
        drop_database('test_migration', @user)
      else
        @user.destroy
      end
    end

    it "exports and reimports a user #{migrate_metadata ? 'with' : 'without'} metadata" do
      CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)

      export = Carto::UserMigrationExport.create(
        user: @carto_user,
        export_metadata: migrate_metadata
      )
      export.run_export

      puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

      @carto_user.client_applications.each(&:destroy)
      @table1.table_visualization.layers.each(&:destroy)
      @table1.destroy
      expect { @table1.records }.to raise_error

      migrate_metadata ? @user.destroy : drop_user_database(@user)

      Cartodb.with_config(agg_ds_config) do
        # Do not depend on dataservices_client to be installed
        CartoDB::UserModule::DBService.any_instance.stubs(:install_geocoder_api_extension)

        import = Carto::UserMigrationImport.create(
          exported_file: export.exported_file,
          database_host: @user_attributes['database_host'],
          org_import: false,
          json_file: export.json_file,
          import_metadata: migrate_metadata
        )
        import.run_import

        puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
        expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

        @carto_user = Carto::User.find(@user_attributes['id'])

        if migrate_metadata
          attributes_to_test(@user_attributes).each do |attribute|
            expect(@carto_user.attributes[attribute]).to eq(@user_attributes[attribute])
          end
          @user.in_database(as: :superuser) do |db|
            ds_config = db.fetch("SELECT * from cdb_conf where key = 'geocoder_server_config'").first[:value]
            fdws_config = db.fetch("SELECT * from cdb_conf where key = 'fdws'").first[:value]
            expect(ds_config).to match /dbname=test_migration/
            expect(fdws_config).to match /\"dbname\":\"test_migration\"/
          end
        else
          expect(@carto_user.attributes).to eq(@user_attributes)
        end

        records.each.with_index { |row, index| @table1.record(index + 1).should include(row) }

      end
    end
  end

  it_should_behave_like 'migrating metadata', true
  it_should_behave_like 'migrating metadata', false

  describe 'failing user imports should rollback' do
    before :each do
      @user = create_user_with_visualizations
      @carto_user = Carto::User.find(@user.id)
      @user_attributes = @carto_user.attributes

      @export = Carto::UserMigrationExport.create(
        user: @carto_user,
        export_metadata: true
      )
      @export.run_export
      destroy_user
    end

    after :each do
      @carto_user.destroy
    end

    it 'import failing in import_metadata should rollback' do
      Carto::RedisExportService.any_instance.stubs(:restore_redis_from_hash_export).raises('Some exception')

      imp = import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      Carto::RedisExportService.any_instance.unstub(:restore_redis_from_hash_export)

      import.run_import.should eq true
    end

    it 'import failing in JobImport#run!' do
      CartoDB::DataMover::ImportJob.any_instance.stubs(:grant_user_role).raises('Some exception')

      imp = import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      CartoDB::DataMover::ImportJob.any_instance.unstub(:grant_user_role)

      import.run_import.should eq true
    end

    it 'import failing creating user database and roles' do
      CartoDB::DataMover::ImportJob.any_instance.stubs(:import_pgdump).raises('Some exception')

      imp = import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      CartoDB::DataMover::ImportJob.any_instance.unstub(:import_pgdump)

      import.run_import.should eq true
    end

    it 'import failing importing visualizations' do
      Carto::UserMetadataExportService.any_instance.stubs(:import_search_tweets_from_directory).raises('Some exception')

      imp = import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      Carto::UserMetadataExportService.any_instance.unstub(:import_search_tweets_from_directory)

      import.run_import.should eq true
    end

    it 'fails importing an already existing user' do
      import.run_import.should eq true
      import.run_import.should eq false
    end

    it 'should continue with rollback if data import rollback fails' do
      CartoDB::DataMover::ImportJob.any_instance.stubs(:grant_user_role).raises('Some exception')
      CartoDB::DataMover::ImportJob.any_instance.stubs(:rollback_user).raises('Some exception')
      import.run_import.should eq false
      CartoDB::DataMover::ImportJob.any_instance.unstub(:grant_user_role)
      CartoDB::DataMover::ImportJob.any_instance.unstub(:rollback_user)
      import.run_import.should eq true
    end

    it 'should not remove user if already exists while importing' do
      import.run_import.should eq true
      import.run_import.should eq false
      Carto::User.exists?(@user.id).should eq true
    end

    it 'import record should exist if import_data fails and rollbacks' do
      Carto::UserMigrationImport.any_instance.stubs(:do_import_data).raises('Some exception')

      imp = import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      Carto::UserMigrationImport.where(id: imp.id).should_not be_empty
      Carto::User.where(username: @carto_user.username).should be_empty

      Carto::UserMigrationImport.any_instance.unstub(:do_import_data)
    end

    it 'import failing importing visualizations does not remove assets' do
      Carto::UserMetadataExportService.any_instance.stubs(:import_search_tweets_from_directory).raises('Some exception')
      Asset.any_instance.stubs(:use_s3?).returns(false)
      asset = Asset.create(asset_file: Rails.root + 'spec/support/data/cartofante_blue.png', user: @user)
      local_url = CGI.unescape(asset.public_url.gsub(/(http:)?\/\/#{CartoDB.account_host}/, ''))
      imp = import

      imp.run_import.should eq false
      imp.state.should eq 'failure'
      File.exists?((asset.public_uploaded_assets_path + local_url).gsub('/uploads/uploads/', '/uploads/')).should eq true
    end
  end

  describe 'failing organization organizations should rollback' do
    include_context 'organization with users helper'
    before :all do
      owner = @carto_organization.owner
      filepath = "#{Rails.root}/services/importer/spec/fixtures/visualization_export_with_two_tables.carto"
      data_import = DataImport.create(
        user_id: owner.id,
        data_source: filepath,
        updated_at: Time.now.utc,
        append: false,
        create_visualization: true
      )
      data_import.values[:data_source] = filepath

      data_import.run_import!
      data_import.success.should eq true

      @export = Carto::UserMigrationExport.create(
        organization: @carto_organization,
        export_metadata: true
      )
      @export.run_export
      @organization.destroy_cascade
    end

    after :each do
      begin
        @organization.reload
        @organization.destroy_cascade
      rescue
      end
    end

    it 'import failing in import_metadata should rollback' do
      Carto::RedisExportService.any_instance.stubs(:restore_redis_from_hash_export).raises('Some exception')

      imp = org_import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      Carto::RedisExportService.any_instance.unstub(:restore_redis_from_hash_export)

      org_import.run_import.should eq true
    end

    it 'import failing in JobImport#run!' do
      CartoDB::DataMover::ImportJob.any_instance.stubs(:grant_user_role).raises('Some exception')

      imp = org_import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      CartoDB::DataMover::ImportJob.any_instance.unstub(:grant_user_role)

      org_import.run_import.should eq true
    end

    it 'import failing creating user database and roles' do
      CartoDB::DataMover::ImportJob.any_instance.stubs(:import_pgdump).raises('Some exception')

      imp = org_import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      CartoDB::DataMover::ImportJob.any_instance.unstub(:import_pgdump)

      org_import.run_import.should eq true
    end

    it 'import failing importing visualizations' do
      Carto::UserMetadataExportService.any_instance.stubs(:import_search_tweets_from_directory).raises('Some exception')

      imp = org_import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      Carto::UserMetadataExportService.any_instance.unstub(:import_search_tweets_from_directory)

      org_import.run_import.should eq true
    end

    it 'import failing import visualizations with metadata_only option' do
      Carto::UserMetadataExportService.any_instance.stubs(:import_search_tweets_from_directory).raises('Some exception')

      imp = org_import
      imp.import_data = false
      imp.save!
      imp.run_import.should eq false
      imp.state.should eq 'failure'
      imp.reload
    end

    it 'should fail if importing an already existing organization with metadata' do
      org_import.run_import.should eq true
      imp = org_import
      imp.run_import.should eq false
      imp.state.should eq 'failure'
    end

    it 'import record should exist if import_data fails and rollbacks' do
      Carto::UserMigrationImport.any_instance.stubs(:do_import_data).raises('Some exception')

      imp = org_import
      imp.run_import.should eq false
      imp.state.should eq 'failure'

      Carto::UserMigrationImport.where(id: imp.id).should_not be_empty
      Carto::Organization.where(id: @carto_organization.id).should be_empty

      Carto::UserMigrationImport.any_instance.unstub(:do_import_data)
    end

    it 'import failing importing visualizations does not remove assets' do
      Carto::StorageOptions::S3.stubs(:enabled?).returns(false)
      Carto::UserMetadataExportService.any_instance.stubs(:import_search_tweets_from_directory).raises('Some exception')
      asset = Carto::Asset.for_organization(
        organization: @carto_organization,
        resource: File.open(Rails.root + 'spec/support/data/cartofante_blue.png')
      )
      imp = org_import

      imp.run_import.should eq false
      imp.state.should eq 'failure'
      File.exists?(asset.storage_info[:identifier]).should eq true
    end
  end

  it 'exports and imports a user with a data import with two tables' do
    CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)

    user = create_user_with_visualizations

    carto_user = Carto::User.find(user.id)
    user_attributes = carto_user.attributes

    source_visualizations = carto_user.visualizations.map(&:name).sort

    export = Carto::UserMigrationExport.create(
      user: carto_user,
      export_metadata: true
    )
    export.run_export

    puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
    expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

    carto_user.client_applications.each(&:destroy)
    user.destroy

    import = Carto::UserMigrationImport.create(
      exported_file: export.exported_file,
      database_host: user_attributes['database_host'],
      org_import: false,
      json_file: export.json_file,
      import_metadata: true,
      dry: false
    )
    import.stubs(:assert_organization_does_not_exist)
    import.stubs(:assert_user_does_not_exist)
    import.run_import

    puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
    expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

    carto_user = Carto::User.find(user_attributes['id'])

    attributes_to_test(user_attributes).each do |attribute|
      expect(carto_user.attributes[attribute]).to eq(user_attributes[attribute])
    end
    expect(carto_user.visualizations.map(&:name).sort).to eq(source_visualizations)

    user.destroy_cascade
  end

  it 'does not export duplicated vizs' do
    user = create_user_with_visualizations
    carto_user = Carto::User.find(user.id)
    user_attributes = carto_user.attributes
    source_vis = carto_user.visualizations.where(type: Carto::Visualization::TYPE_CANONICAL).first
    carto_user.visualizations.count.should eq 3
    map, _, table_visualization, visualization = create_full_visualization(carto_user)
    table_visualization.update_column(:name, source_vis.name)
    table_visualization.update_column(:map_id, source_vis.map.id)
    table_visualization.update_column(:updated_at, source_vis.updated_at - 1.minute)
    map.destroy
    visualization.destroy

    carto_user.visualizations.count.should eq 4

    export = Carto::UserMigrationExport.create(
      user: carto_user,
      export_metadata: true
    )
    export.run_export

    puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
    expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

    user.destroy_cascade

    import = Carto::UserMigrationImport.create(
      exported_file: export.exported_file,
      database_host: user_attributes['database_host'],
      org_import: false,
      json_file: export.json_file,
      import_metadata: true,
      dry: false
    )
    import.stubs(:assert_organization_does_not_exist)
    import.stubs(:assert_user_does_not_exist)
    import.run_import

    puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
    expect(export.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
    imported_user = Carto::User.find(user_attributes['id'])
    imported_user.visualizations.count.should eq 3
    expect { imported_user.visualizations.find(table_visualization.id) }.to raise_error(ActiveRecord::RecordNotFound)

    user.destroy_cascade
  end

  it 'exporting and then importing to the same DB host fails but DB is not deleted (#c1945)' do
    CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)

    user = create_user_with_visualizations

    carto_user = Carto::User.find(user.id)
    user_attributes = carto_user.attributes

    export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: false)
    export.run_export

    expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

    import = Carto::UserMigrationImport.create(
      exported_file: export.exported_file,
      database_host: user_attributes['database_host'],
      org_import: false,
      json_file: export.json_file,
      import_metadata: false,
      dry: false
    )
    import.run_import

    expect(import.state).to eq(Carto::UserMigrationImport::STATE_FAILURE)
    expect(import.log.entries).to include('DB already exists at DB host')

    # DB exists, otherwise this would fail
    user.in_database.run("select 1;")

    user.destroy_cascade
  end

  it 'exports users with datasets without a physical table if metadata export is requested (see #13721)' do
    CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)

    user = FactoryGirl.build(:valid_user).save
    carto_user = Carto::User.find(user.id)

    @map, @table, @table_visualization, @visualization = create_full_visualization(carto_user)

    carto_user.tables.exists?(name: @table.name).should be
    user.in_database.execute("DROP TABLE #{@table.name}")
    # The table is still registered after the deletion
    carto_user.reload

    carto_user.tables.exists?(name: @table.name).should be

    export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
    export.run_export

    export.log.entries.should_not include("Cannot export if tables aren't synched with db. Please run ghost tables.")
    expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)
    export.destroy

    user.destroy
  end

  it 'does export users with a canonical viz without user table if metadata export is requested (see #12588)' do
    CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)

    user = FactoryGirl.build(:valid_user).save
    carto_user = Carto::User.find(user.id)

    @map, @table, @table_visualization, @visualization = create_full_visualization(carto_user)

    @table.delete
    user.in_database.execute("DROP TABLE #{@table.name}")
    # The canonical visualization is still registered after the deletion
    @table_visualization.reload
    @table_visualization.should be

    export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
    export.run_export

    expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

    user.destroy
  end

  it 'exports and imports a user with raster overviews because exporting skips them' do
    CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
    user = FactoryGirl.build(:valid_user).save
    carto_user = Carto::User.find(user.id)
    user_attributes = carto_user.attributes
    user.in_database.execute('CREATE TABLE i_hate_raster(rast raster)')
    user.in_database.execute('INSERT INTO i_hate_raster VALUES(ST_MakeEmptyRaster(100, 100, 0, 0, 100, 100, 0, 0, 2274))')
    user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
    user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
    user.in_database.execute("SELECT AddRasterConstraints('i_hate_raster', 'rast')")
    user.in_database.execute("SELECT ST_CreateOverview('i_hate_raster'::regclass, 'rast', 2)")
    user.in_database.execute('DROP TABLE i_hate_raster')
    export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
    export.run_export
    user.destroy

    import = Carto::UserMigrationImport.create(
      exported_file: export.exported_file,
      database_host: user_attributes['database_host'],
      org_import: false,
      json_file: export.json_file,
      import_metadata: true,
      dry: false
    )
    import.run_import

    expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
  end

  describe 'legacy functions' do
    before :each do
      @legacy_functions = CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS
    end

    after :each do
      CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS = @legacy_functions
    end

    it 'loads legacy functions' do
      CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS.count.should eq 2503
    end

    it 'matches functions with attributes qualified with namespace' do
      class DummyTester
        include CartoDB::DataMover::LegacyFunctions
      end
      line = '1880; 1255 5950507 FUNCTION asbinary("geometry", "pg_catalog"."text") postgres'
      DummyTester.new.remove_line?(line).should be true
      line2 = '8506; 2753 18284 OPERATOR FAMILY public btree_geography_ops postgres'
      DummyTester.new.remove_line?(line2).should be true
      line3 = '18305; 0 0 ACL public st_wkbtosql("bytea") postgres'
      DummyTester.new.remove_line?(line3).should be true
      line4 = '18333; 0 0 ACL public st_countagg("raster", integer, boolean, double precision) postgres'
      DummyTester.new.remove_line?(line4).should be false
      line5 = '541; 1259 735510 FOREIGN TABLE aggregation agg_admin1 postgres'
      DummyTester.new.remove_line?(line5).should be false
      line6 = '242; 1255 148122 FUNCTION org00000001-admin st_text(boolean)'
      DummyTester.new.remove_line?(line6).should be true
      line7 = '5003; 0 0 ACL org000001-admin FUNCTION "st_text"(boolean)'
      DummyTester.new.remove_line?(line7).should be true
    end

    it 'skips importing legacy functions using fixture' do
      CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
      CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS = ["FUNCTION increment(integer)", "FUNCTION sumita(integer,integer)"].freeze
      user = FactoryGirl.build(:valid_user).save
      carto_user = Carto::User.find(user.id)
      user_attributes = carto_user.attributes
      user.in_database.execute('CREATE OR REPLACE FUNCTION increment(i INT) RETURNS INT AS $$
      BEGIN
        RETURN i + 1;
      END;
      $$ LANGUAGE plpgsql;')

      user.in_database.execute('CREATE OR REPLACE FUNCTION sumita(i1 INT, i2 INT) RETURNS INT AS $$
      BEGIN
        RETURN i1 + i2;
      END;
      $$ LANGUAGE plpgsql;')

      export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
      export.run_export
      user.destroy

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        dry: false
      )
      import.run_import

      user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'increment'").should eq 0
      user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'sumita'").should eq 0
      user.destroy
    end

    it 'imports functions and tables that are not on the legacy list using fixture' do
      CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
      user = FactoryGirl.build(:valid_user).save
      carto_user = Carto::User.find(user.id)
      user_attributes = carto_user.attributes
      user.in_database.execute('CREATE OR REPLACE FUNCTION st_text(b boolean) RETURNS INT AS $$
      BEGIN
        RETURN 1;
      END;
      $$ LANGUAGE plpgsql;')

      user.in_database.execute('CREATE OR REPLACE FUNCTION increment(i INT) RETURNS INT AS $$
      BEGIN
        RETURN i + 1;
      END;
      $$ LANGUAGE plpgsql;')

      user.in_database.execute('CREATE TABLE layer_wadus(number INT)')
      user.in_database.execute('INSERT INTO layer_wadus VALUES (\'1\')')

      export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
      export.run_export
      user.destroy

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        dry: false
      )
      import.run_import

      user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'st_text'").should eq 0
      user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'increment'").should eq 1
      user.in_database.execute('select count(*) from layer_wadus').should eq 1
      user.destroy
    end

    describe 'with organization' do
      include_context 'organization with users helper'

      let(:org_attributes) { @carto_organization.attributes }
      let(:owner_attributes) { @carto_org_user_owner.attributes }

      it 'should not import acl over deprecated functions' do
        user1 = @carto_organization.users.first
        user2 = @carto_organization.users.last
        user1.in_database.execute('CREATE OR REPLACE FUNCTION st_text(boolean) RETURNS INT AS $$
        BEGIN
          RETURN 1;
        END;
        $$ LANGUAGE plpgsql;')

        user1.in_database.execute("GRANT ALL ON FUNCTION st_text TO \"#{user2.service.database_public_username}\"")

        export = Carto::UserMigrationExport.create(organization: @carto_organization, export_metadata: true)
        export.run_export
        @organization.destroy_cascade

        import = Carto::UserMigrationImport.create(
          exported_file: export.exported_file,
          database_host: owner_attributes['database_host'],
          org_import: true,
          json_file: export.json_file,
          import_metadata: true,
          import_data: true,
          dry: false
        )
        import.run_import

        import.state.should eq 'complete'
        Organization[@organization.id].users.first.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'st_text'").should eq 0

      end
    end
  end

  describe 'with organization' do
    include_context 'organization with users helper'

    let(:org_attributes) { @carto_organization.attributes }
    let(:owner_attributes) { @carto_org_user_owner.attributes }

    shared_examples_for 'migrating metadata' do |migrate_metadata|
      before(:each) do
        @table1 = create_table(user_id: @carto_org_user_1.id)
        records.each { |row| @table1.insert_row!(row) }
        create_database('test_migration', @organization.owner) if migrate_metadata
        @owner_api_key = Carto::ApiKey.create_regular_key!(user: @carto_org_user_owner, name: unique_name('api_key'),
                                                           grants: [{ type: "apis", apis: ["maps", "sql"] }])
        @user1_api_key = Carto::ApiKey.create_regular_key!(user: @carto_org_user_1, name: unique_name('api_key'),
                                                           grants: [{ type: "apis", apis: ["maps", "sql"] }])
        @carto_organization.reload
      end

      after(:each) do
        drop_database('test_migration', @organization.owner) if migrate_metadata
        @owner_api_key.destroy
        @user1_api_key.destroy
      end

      it "exports and reimports an organization #{migrate_metadata ? 'with' : 'without'} metadata" do
        export = Carto::UserMigrationExport.create(organization: @carto_organization, export_metadata: migrate_metadata)
        export.run_export

        export.state.should eq Carto::UserMigrationExport::STATE_COMPLETE

        migrate_metadata ? @organization.destroy_cascade : drop_user_database(@organization.owner)

        Cartodb.with_config(agg_ds_config) do
          # Do not depend on dataservices_client to be installed
          CartoDB::UserModule::DBService.any_instance.stubs(:install_geocoder_api_extension)

          import = Carto::UserMigrationImport.create(
            exported_file: export.exported_file,
            database_host: owner_attributes['database_host'],
            org_import: true,
            json_file: export.json_file,
            import_metadata: migrate_metadata
          )
          import.stubs(:assert_organization_does_not_exist)
          import.stubs(:assert_user_does_not_exist)
          import.run_import

          puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
          import.state.should eq Carto::UserMigrationImport::STATE_COMPLETE

          new_organization = Carto::Organization.find(org_attributes['id'])
          attributes_to_test(new_organization.attributes).should eq attributes_to_test(org_attributes)
          new_organization.users.count.should eq 3
          attributes_to_test(new_organization.owner.attributes).should eq attributes_to_test(owner_attributes)
          records.each.with_index { |row, index| @table1.record(index + 1).should include(row) }
          if migrate_metadata
            new_organization.owner.in_database(as: :superuser) do |db|
              ds_config = db.exec_query("SELECT * from cdb_conf where key = 'geocoder_server_config'").first['value']
              fdws_config = db.exec_query("SELECT * from cdb_conf where key = 'fdws'").first['value']
              expect(ds_config).to match /dbname=test_migration/
              expect(fdws_config).to match /\"dbname\":\"test_migration\"/
            end
          end
        end
      end
    end

    it_should_behave_like 'migrating metadata', true
    it_should_behave_like 'migrating metadata', false

    it 'exports orgs with datasets without physical table if metadata export is requested (see #13721)' do
      @map, @table, @table_visualization, @visualization = create_full_visualization(@carto_org_user_1)

      @carto_org_user_1.tables.exists?(name: @table.name).should be
      @org_user_1.in_database.execute("DROP TABLE #{@table.name}")
      # The table is still registered after the deletion
      @carto_org_user_1.reload
      @carto_org_user_1.tables.exists?(name: @table.name).should be

      export = Carto::UserMigrationExport.create(organization: @carto_organization, export_metadata: true)
      export.run_export
      export.log.entries.should_not include("Cannot export if tables aren't synched with db. Please run ghost tables.")
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)
      export.destroy
    end
  end

  it '#run_import does not modify database_host with dry' do
    user = create_user_with_visualizations
    carto_user = Carto::User.find(user.id)
    database_host = carto_user.database_host

    export = Carto::UserMigrationExport.create(
      user: carto_user,
      export_metadata: true
    )

    export.run_export

    drop_user_database(user)

    # Let's fake the column to check that dry doesn't fix it
    carto_user.update_column(:database_host, 'wadus')

    import = Carto::UserMigrationImport.create(
      exported_file: export.exported_file,
      database_host: database_host,
      org_import: false,
      json_file: export.json_file,
      import_metadata: false,
      dry: true
    )

    import.run_import
    import.state.should eq 'complete'

    carto_user2 = Carto::User.find(user.id)
    carto_user2.attributes.should eq carto_user.attributes
  end

  describe 'api keys import and exports' do
    before :each do
      @user = FactoryGirl.build(:valid_user)
      @user.save
      @carto_user = Carto::User.find(@user.id)
      @master_api_key = @carto_user.api_keys.master.first
      @map, @table, @table_visualization, @visualization = create_full_visualization(@carto_user)
      @regular_api_key = Carto::ApiKey.create_regular_key!(user: @carto_user,
                                                           name: 'Some ApiKey',
                                                           grants: [
                                                             {
                                                               type: "apis",
                                                               apis: ["maps", "sql"]
                                                             },
                                                             {
                                                               type: 'database',
                                                               tables: [
                                                                 {
                                                                   schema: @carto_user.database_schema,
                                                                   name: @table.name,
                                                                   permissions: ['select']
                                                                 }
                                                               ],
                                                               schemas: [
                                                                 {
                                                                   name: @carto_user.database_schema,
                                                                   permissions: ['create']
                                                                 }
                                                               ]
                                                             }
                                                           ])
    end

    after :each do
      @user.destroy
    end

    it 'api keys are in redis and db roles are created' do
      user_attributes = @carto_user.attributes
      export = Carto::UserMigrationExport.create(
        user: @carto_user,
        export_metadata: true
      )
      export.run_export

      puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

      username = @user.username
      $users_metadata.hmget("api_keys:#{username}:#{@master_api_key.token}", 'user')[0].should eq username
      $users_metadata.hmget("api_keys:#{username}:#{@regular_api_key.token}", 'user')[0].should eq username

      @carto_user.client_applications.each(&:destroy)
      @master_api_key.destroy
      @table.destroy
      @map.destroy
      @table_visualization.destroy
      @visualization.destroy
      @carto_user.destroy
      @regular_api_key.destroy
      drop_user_database(@user)

      $users_metadata.hmget("api_keys:#{username}:#{@master_api_key.token}", 'user')[0].should be nil
      $users_metadata.hmget("api_keys:#{username}:#{@regular_api_key.token}", 'user')[0].should be nil

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        dry: false
      )

      import.stubs(:assert_organization_does_not_exist)
      import.stubs(:assert_user_does_not_exist)
      import.run_import

      puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
      expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

      $users_metadata.hmget("api_keys:#{username}:#{@master_api_key.token}", 'user')[0].should eq username
      $users_metadata.hmget("api_keys:#{username}:#{@regular_api_key.token}", 'user')[0].should eq username

      user = Carto::User.find(user_attributes['id'])
      user.should be
      user.api_keys.each(&:table_permissions_from_db) # to make sure DB can be queried without exceptions
      user.api_keys.select { |a| a.type == 'master' }.first.table_permissions_from_db.count.should be > 0
    end

    it 'keeps roles for oauth api keys with schemas grants and you can drop tables after migration' do
      Cartodb::Central.stubs(:sync_data_with_cartodb_central?).returns(false)
      oauth_app = Carto::OauthApp.create!(name: 'test',
                                          user_id: @carto_user.id,
                                          redirect_uris: ['https://example.com'],
                                          website_url: 'http://localhost',
                                          icon_url: 'https://example.com')
      oauth_app_user = oauth_app.oauth_app_users.create!(user_id: @carto_user.id)
      access_token = oauth_app_user.oauth_access_tokens.create!(scopes: ['schemas:c'])

      api_key = access_token.api_key
      with_connection_from_api_key(api_key) do |connection|
        connection.execute("create table test_table(id INT)")
        connection.execute("insert into test_table values (999)")
        connection.execute("select id from test_table") do |result|
          result[0]['id'].should eq '999'
        end
      end

      user_attributes = @carto_user.attributes
      export = Carto::UserMigrationExport.create(
        user: @carto_user,
        export_metadata: true
      )
      export.run_export

      puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

      @carto_user.client_applications.each(&:destroy)
      @master_api_key.destroy
      @table.destroy
      @map.destroy
      @table_visualization.destroy
      @visualization.destroy

      # oauth_app must exist in the destination
      # so we remove the user_id to avoid it being cascade deleted with the user
      oauth_app.stubs(:sync_with_central?).returns(false)
      oauth_app.stubs(:central_enabled?).returns(true)
      oauth_app.user_id = nil
      oauth_app.save!

      @carto_user.destroy
      @regular_api_key.destroy
      drop_user_database(@user)

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        dry: false
      )

      import.stubs(:assert_organization_does_not_exist)
      import.stubs(:assert_user_does_not_exist)
      import.run_import
      puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
      expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

      user = Carto::User.find(user_attributes['id'])
      user.should be
      user.api_keys.each(&:table_permissions_from_db) # to make sure DB can be queried without exceptions
      user.api_keys.select { |a| a.type == 'master' }.first.table_permissions_from_db.count.should be > 0

      expect(user.oauth_app_users.count).to eq 1
      oauth_app_user = user.oauth_app_users.first
      expect(oauth_app_user.exists_ownership_role?).to be_true
      expect(oauth_app_user.oauth_access_tokens.count).to eq 1

      access_token = oauth_app_user.oauth_access_tokens.first
      api_key = access_token.api_key
      with_connection_from_api_key(api_key) do |connection|
        connection.execute("drop table test_table")
        connection.execute("select relname from pg_class where relname = 'test_table'") do |result|
          result.count eq 0
        end
        connection.execute("create table test_table(id INT)")
      end

      check_cdb_conf_query = "SELECT value->>'ownership_role_name' as c from cdb_conf where key = 'api_keys_' || '#{api_key.db_role}';"
      result = user.in_database(as: :superuser).execute(check_cdb_conf_query)
      expect(result.count).to eq 1
      Cartodb::Central.unstub(:sync_data_with_cartodb_central?)
      oauth_app.destroy!
    end

    it 'api keys keeps the grants and you can drop tables after migration' do
      regular_api_key = @carto_user.api_keys.regular.first
      with_connection_from_api_key(regular_api_key) do |connection|
        connection.execute("create table test_table(id INT)")
        connection.execute("insert into test_table values (999)")
        connection.execute("select id from test_table") do |result|
          result[0]['id'].should eq '999'
        end
      end

      user_attributes = @carto_user.attributes
      export = Carto::UserMigrationExport.create(
        user: @carto_user,
        export_metadata: true
      )
      export.run_export

      puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

      @carto_user.client_applications.each(&:destroy)
      @master_api_key.destroy
      @table.destroy
      @map.destroy
      @table_visualization.destroy
      @visualization.destroy
      @carto_user.destroy
      @regular_api_key.destroy
      drop_user_database(@user)

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        dry: false
      )

      import.stubs(:assert_organization_does_not_exist)
      import.stubs(:assert_user_does_not_exist)
      import.run_import

      puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
      expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

      user = Carto::User.find(user_attributes['id'])
      user.should be
      user.api_keys.each(&:table_permissions_from_db) # to make sure DB can be queried without exceptions
      user.api_keys.select { |a| a.type == 'master' }.first.table_permissions_from_db.count.should be > 0

      with_connection_from_api_key(user.api_keys.master.first) do |connection|
        connection.execute("drop table test_table")
        connection.execute("select relname from pg_class where relname = 'test_table'") do |result|
          result.count eq 0
        end
      end
    end
  end

  include CartoDB::DataMover::Utils
  describe 'database version' do
    before(:each) do
      @conn_mock = Object.new
      @conn_mock.stubs(:query).returns(['version' => 'PostgreSQL 9.5.2 on x86_64-pc-linux-gnu...'])
    end

    it 'should get proper database version for pg_* binaries' do
      get_database_version_for_binaries(@conn_mock).should eq '9.5'

      @conn_mock.stubs(:query).returns(['version' => 'PostgreSQL 10.1 on x86_64-pc-linux-gnu...'])
      get_database_version_for_binaries(@conn_mock).should eq '10'
    end

    it 'should get proper binary paths version for pg_dump and pg_restore' do
      get_pg_dump_bin_path(@conn_mock).should include 'pg_dump'
      get_pg_restore_bin_path(@conn_mock).should include 'pg_restore'
    end

    it 'raises exception if cannot get dump database version' do
      expect { get_dump_database_version(@conn_mock, '123') }.to raise_error
    end

    it 'retrieves dump database version from stubbed dump file name' do
      @conn_mock.stubs(:query).returns(['version' => 'PostgreSQL 10.1 on x86_64-pc-linux-gnu...'])
      status_mock = Object.new
      status_mock.stubs(:success?).returns(true)
      Open3.stubs(:capture3).returns([';     Dumped by pg_dump version: 9.5.2', '', status_mock])
      get_dump_database_version(@conn_mock, '/tmp/test.dump').should eq '9.5'
    end
  end

  describe 'export_data being false' do
    it 'exports and imports user with viz' do
      user = create_user_with_visualizations

      carto_user = Carto::User.find(user.id)
      user_attributes = carto_user.attributes

      source_visualizations = carto_user.visualizations.map(&:name).sort

      export = Carto::UserMigrationExport.create(
        user: carto_user,
        export_metadata: true,
        export_data: false
      )
      export.run_export

      puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

      remove_user(carto_user)

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        import_data: false,
        dry: false
      )

      import.run_import

      puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
      expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

      carto_user = Carto::User.find(user_attributes['id'])

      attributes_to_test(user_attributes).each do |attribute|
        expect(carto_user.attributes[attribute]).to eq(user_attributes[attribute])
      end
      expect(carto_user.visualizations.map(&:name).sort).to eq(source_visualizations)

      Carto::GhostTablesManager.new(user.id).user_tables_synced_with_db?.should eq true

      user.destroy_cascade
    end

    it 'export and imports keeping import when import visualizations fails' do
      user = create_user_with_visualizations

      carto_user = Carto::User.find(user.id)
      user_attributes = carto_user.attributes

      export = Carto::UserMigrationExport.create(
        user: carto_user,
        export_metadata: true,
        export_data: false
      )
      export.run_export

      import = Carto::UserMigrationImport.create!(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        import_data: false,
        dry: false
      )
      Carto::UserMetadataExportService.any_instance.stubs(:import_metadata_from_directory).raises('Something went bad')

      import.run_import

      import.reload.persisted?.should eq true

      user.destroy_cascade
    end

    it 'does not remove database when visuaization import fails' do
      user = create_user_with_visualizations

      carto_user = Carto::User.find(user.id)
      user_attributes = carto_user.attributes

      export = Carto::UserMigrationExport.create(
        user: carto_user,
        export_metadata: true,
        export_data: false
      )
      export.run_export

      puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
      expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

      remove_user(carto_user)

      import = Carto::UserMigrationImport.create(
        exported_file: export.exported_file,
        database_host: user_attributes['database_host'],
        org_import: false,
        json_file: export.json_file,
        import_metadata: true,
        import_data: false,
        dry: false
      )

      Carto::UserMigrationImport.any_instance.stubs(:import_visualizations).raises('wadus')

      import.run_import

      config = CartoDB::DataMover::Config.config
      PG.connect(host: user_attributes['database_host'],
                 user: config[:dbuser],
                 dbname: user_attributes['database_name'],
                 port: config[:dbport],
                 connect_timeout: config[:connect_timeout])
    end

    describe 'with orgs' do
      include_context 'organization with users helper'
      it 'exports and imports org with users with viz' do
        CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
        export = Carto::UserMigrationExport.create(
          organization: @carto_organization,
          export_metadata: true,
          export_data: false
        )
        export.run_export

        puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
        expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

        database_host = @carto_organization.owner.database_host

        @carto_organization.users.each { |u| remove_user(u) }
        @carto_organization.delete

        import = Carto::UserMigrationImport.create(
          exported_file: export.exported_file,
          database_host: database_host,
          org_import: true,
          json_file: export.json_file,
          import_metadata: true,
          import_data: false,
          dry: false
        )

        import.run_import

        puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
        expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)

        @carto_organization.users.each do |u|
          Carto::GhostTablesManager.new(u.id).user_tables_synced_with_db?.should eq true
        end
      end

      it 'does not drop database if visualizations import fails' do
        CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
        export = Carto::UserMigrationExport.create(
          organization: @carto_organization,
          export_metadata: true,
          export_data: false
        )
        export.run_export

        puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
        expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)

        database_host = @carto_organization.owner.database_host

        db_host = @carto_organization.owner.database_host
        db_name = @carto_organization.database_name

        @carto_organization.users.each { |u| remove_user(u) }
        @carto_organization.delete

        import = Carto::UserMigrationImport.create(
          exported_file: export.exported_file,
          database_host: database_host,
          org_import: true,
          json_file: export.json_file,
          import_metadata: true,
          import_data: false,
          dry: false
        )

        Carto::UserMigrationImport.any_instance.stubs(:import_visualizations).raises('wadus')

        import.run_import

        config = CartoDB::DataMover::Config.config
        PG.connect(host: db_host,
                   user: config[:dbuser],
                   dbname: db_name,
                   port: config[:dbport],
                   connect_timeout: config[:connect_timeout])
      end
    end

    def remove_user(carto_user)
      Carto::Visualization.where(user_id: carto_user.id).each do |v|
        v.overlays.each(&:delete)
        v.user_table.delete if v.user_table # Delete user_table since it can conflict by name
        v.delete
      end
      gum = CartoDB::GeocoderUsageMetrics.new(carto_user.username)
      $users_metadata.DEL(gum.send(:user_key_prefix, :geocoder_here, :success_responses, Time.now))
      ::User[carto_user.id].client_application.oauth_tokens.each(&:destroy)
      ::User[carto_user.id].client_application.destroy
      carto_user.delete
    end
  end

  def drop_user_database(user)
    conn = user.in_database(as: :cluster_admin)
    user.db_service.drop_database_and_user(conn)
    user.db_service.drop_user(conn)
  end

  def create_database(name, user)
    conn = user.in_database(as: :cluster_admin)
    sql = "CREATE DATABASE \"#{name}\"
    WITH TEMPLATE = template_postgis
    ENCODING = 'UTF8'
    CONNECTION LIMIT=-1"
    conn.run(sql) rescue conn.exec_query(sql)
  end

  def drop_database(name, user)
    conn = user.in_database(as: :cluster_admin)
    sql = "DROP DATABASE \"#{name}\""
    conn.run(sql) rescue conn.exec_query(sql)
  end

  def attributes_to_test(user_attributes)
    user_attributes.keys - %w(created_at updated_at period_end_date)
  end

  private

  def create_user_with_visualizations
    user = FactoryGirl.build(:valid_user).save

    filepath = "#{Rails.root}/services/importer/spec/fixtures/visualization_export_with_two_tables.carto"
    data_import = DataImport.create(
      user_id: user.id,
      data_source: filepath,
      updated_at: Time.now.utc,
      append: false,
      create_visualization: true
    )
    data_import.values[:data_source] = filepath

    data_import.run_import!
    data_import.success.should eq true
    user
  end

  def org_import
    imp = Carto::UserMigrationImport.create(
      exported_file: @export.exported_file,
      database_host: @carto_organization.owner.attributes['database_host'],
      org_import: true,
      json_file: @export.json_file,
      import_metadata: true,
      dry: false
    )

    imp.stubs(:assert_organization_does_not_exist)
    imp.stubs(:assert_user_does_not_exist)
    imp
  end

  def import
    imp = Carto::UserMigrationImport.create(
      exported_file: @export.exported_file,
      database_host: @user_attributes['database_host'],
      org_import: false,
      json_file: @export.json_file,
      import_metadata: true,
      dry: false
    )
    imp.stubs(:assert_organization_does_not_exist)
    imp.stubs(:assert_user_does_not_exist)
    imp
  end

  def destroy_user
    @carto_user.client_applications.each(&:destroy)
    @user.destroy
  end
end