cartodb-4.42/spec/models/carto/user_migration_api_key_spec.rb
2024-04-06 05:25:13 +00:00

595 lines
24 KiB
Ruby

require_relative '../../spec_helper_min'
require_relative '../../../app/models/carto/user_migration_import'
require_relative '../../../app/models/carto/user_migration_export'
require_relative '../../support/factories/tables'
require_relative '../../factories/organizations_contexts'
require_relative './helpers/user_migration_helper'
require 'helpers/database_connection_helper'
require 'factories/carto_visualizations'
describe 'UserMigration' do
include Carto::Factories::Visualizations
include CartoDB::Factories
include DatabaseConnectionHelper
include UserMigrationHelper
it 'exports and imports a user with raster overviews because exporting skips them' do
CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
user = FactoryGirl.build(:valid_user).save
next unless user.in_database.table_exists?('raster_overviews')
carto_user = Carto::User.find(user.id)
user_attributes = carto_user.attributes
user.in_database.execute('CREATE TABLE i_hate_raster(rast raster)')
user.in_database.execute('INSERT INTO i_hate_raster VALUES(ST_MakeEmptyRaster(100, 100, 0, 0, 100, 100, 0, 0, 2274))')
user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
user.in_database.execute("SELECT AddRasterConstraints('i_hate_raster', 'rast')")
user.in_database.execute("SELECT ST_CreateOverview('i_hate_raster'::regclass, 'rast', 2)")
user.in_database.execute('DROP TABLE i_hate_raster')
export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
export.run_export
user.destroy
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.run_import
expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
end
describe 'legacy functions' do
before(:all) do
class DummyTester
include CartoDB::DataMover::LegacyFunctions
end
@dummy_tester = DummyTester.new
end
before :each do
@legacy_functions = CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS
end
after :each do
CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS = @legacy_functions
end
it 'loads legacy functions' do
CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS.count.should eq 2511
end
it 'matches functions with attributes qualified with namespace' do
line = '1880; 1255 5950507 FUNCTION asbinary("geometry", "pg_catalog"."text") postgres'
@dummy_tester.remove_line?(line).should be true
line2 = '8506; 2753 18284 OPERATOR FAMILY public btree_geography_ops postgres'
@dummy_tester.remove_line?(line2).should be true
line3 = '18305; 0 0 ACL public st_wkbtosql("bytea") postgres'
@dummy_tester.remove_line?(line3).should be true
line4 = '18333; 0 0 ACL public st_countagg("raster", integer, boolean, double precision) postgres'
@dummy_tester.remove_line?(line4).should be false
line5 = '541; 1259 735510 FOREIGN TABLE aggregation agg_admin1 postgres'
@dummy_tester.remove_line?(line5).should be false
line6 = '242; 1255 148122 FUNCTION org00000001-admin st_text(boolean)'
@dummy_tester.remove_line?(line6).should be true
line7 = '5003; 0 0 ACL org000001-admin FUNCTION "st_text"(boolean)'
@dummy_tester.remove_line?(line7).should be true
end
it 'matches functions with attributes qualified with namespace and name' do
line = '5785; 0 0 ACL public FUNCTION "st_asgeojson"'\
'("geog" "public"."geography", "maxdecimaldigits" integer, "options" integer) postgres'
expect(@dummy_tester.remove_line?(line)).to be_true
end
it 'skips importing legacy functions using fixture' do
CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
CartoDB::DataMover::LegacyFunctions::LEGACY_FUNCTIONS = ["FUNCTION increment(integer)", "FUNCTION sumita(integer,integer)"].freeze
user = FactoryGirl.build(:valid_user).save
carto_user = Carto::User.find(user.id)
user_attributes = carto_user.attributes
user.in_database.execute('CREATE OR REPLACE FUNCTION increment(i INT) RETURNS INT AS $$
BEGIN
RETURN i + 1;
END;
$$ LANGUAGE plpgsql;')
user.in_database.execute('CREATE OR REPLACE FUNCTION sumita(i1 INT, i2 INT) RETURNS INT AS $$
BEGIN
RETURN i1 + i2;
END;
$$ LANGUAGE plpgsql;')
export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
export.run_export
user.destroy
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.run_import
user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'increment'").should eq 0
user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'sumita'").should eq 0
user.destroy
end
it 'imports functions and tables that are not on the legacy list using fixture' do
CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
user = FactoryGirl.build(:valid_user).save
carto_user = Carto::User.find(user.id)
user_attributes = carto_user.attributes
user.in_database.execute('CREATE OR REPLACE FUNCTION st_text(b boolean) RETURNS INT AS $$
BEGIN
RETURN 1;
END;
$$ LANGUAGE plpgsql;')
user.in_database.execute('CREATE OR REPLACE FUNCTION increment(i INT) RETURNS INT AS $$
BEGIN
RETURN i + 1;
END;
$$ LANGUAGE plpgsql;')
user.in_database.execute('CREATE TABLE layer_wadus(number INT)')
user.in_database.execute('INSERT INTO layer_wadus VALUES (\'1\')')
export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
export.run_export
user.destroy
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.run_import
user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'st_text'").should eq 0
user.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'increment'").should eq 1
user.in_database.execute('select count(*) from layer_wadus').should eq 1
user.destroy
end
describe 'with organization' do
include_context 'organization with users helper'
let(:org_attributes) { @carto_organization.attributes }
let(:owner_attributes) { @carto_org_user_owner.attributes }
it 'should not import acl over deprecated functions' do
user1 = @carto_organization.users.first
user2 = @carto_organization.users.last
user1.in_database.execute('CREATE OR REPLACE FUNCTION st_text(boolean) RETURNS INT AS $$
BEGIN
RETURN 1;
END;
$$ LANGUAGE plpgsql;')
user1.in_database.execute("GRANT ALL ON FUNCTION st_text TO \"#{user2.service.database_public_username}\"")
export = Carto::UserMigrationExport.create(organization: @carto_organization, export_metadata: true)
export.run_export
@organization.destroy_cascade
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: owner_attributes['database_host'],
org_import: true,
json_file: export.json_file,
import_metadata: true,
import_data: true,
dry: false
)
import.run_import
import.state.should eq 'complete'
Organization[@organization.id].users.first.in_database.execute("SELECT prosrc FROM pg_proc WHERE proname = 'st_text'").should eq 0
end
end
end
describe 'with organization' do
include_context 'organization with users helper'
records =
[
{ name: 'carto', description: 'awesome' },
{ name: 'user-mover', description: 'insanity' }
]
agg_ds_config =
{
aggregation_tables: {
'host' => 'localhost',
'port' => '5432',
'dbname' => 'test_migration',
'username' => 'geocoder_api',
'password' => '',
'tables' => {
'admin0' => 'ne_admin0_v3',
'admin1' => 'global_province_polygons'
}
},
geocoder: {
'api' => {
'host' => 'localhost',
'port' => '5432',
'dbname' => 'test_migration',
'user' => 'geocoder_api'
}
}
}
let(:org_attributes) { @carto_organization.attributes }
let(:owner_attributes) { @carto_org_user_owner.attributes }
shared_examples_for 'migrating metadata' do |migrate_metadata|
before(:each) do
@table1 = create_table(user_id: @carto_org_user_1.id)
records.each { |row| @table1.insert_row!(row) }
create_database('test_migration', @organization.owner) if migrate_metadata
@owner_api_key = Carto::ApiKey.create_regular_key!(user: @carto_org_user_owner, name: unique_name('api_key'),
grants: [{ type: "apis", apis: ["maps", "sql"] }])
@user1_api_key = Carto::ApiKey.create_regular_key!(user: @carto_org_user_1, name: unique_name('api_key'),
grants: [{ type: "apis", apis: ["maps", "sql"] }])
@carto_organization.reload
end
after(:each) do
drop_database('test_migration', @organization.owner) if migrate_metadata
@owner_api_key.destroy
@user1_api_key.destroy
end
it "exports and reimports an organization #{migrate_metadata ? 'with' : 'without'} metadata" do
export = Carto::UserMigrationExport.create(organization: @carto_organization, export_metadata: migrate_metadata)
export.run_export
export.state.should eq Carto::UserMigrationExport::STATE_COMPLETE
migrate_metadata ? @organization.destroy_cascade : drop_user_database(@organization.owner)
Cartodb.with_config(agg_ds_config) do
# Do not depend on dataservices_client to be installed
CartoDB::UserModule::DBService.any_instance.stubs(:install_geocoder_api_extension)
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: owner_attributes['database_host'],
org_import: true,
json_file: export.json_file,
import_metadata: migrate_metadata
)
import.stubs(:assert_organization_does_not_exist)
import.stubs(:assert_user_does_not_exist)
import.run_import
puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
import.state.should eq Carto::UserMigrationImport::STATE_COMPLETE
new_organization = Carto::Organization.find(org_attributes['id'])
attributes_to_test(new_organization.attributes).should eq attributes_to_test(org_attributes)
new_organization.users.count.should eq 3
attributes_to_test(new_organization.owner.attributes).should eq attributes_to_test(owner_attributes)
records.each.with_index { |row, index| @table1.record(index + 1).should include(row) }
if migrate_metadata
new_organization.owner.in_database(as: :superuser) do |db|
db.exec_query("SELECT cartodb.cdb_extension_reload()")
ds_config = db.exec_query("SELECT * from cdb_conf where key = 'geocoder_server_config'").first['value']
fdws_config = db.exec_query("SELECT * from cdb_conf where key = 'fdws'").first['value']
expect(ds_config).to match /dbname=test_migration/
expect(fdws_config).to match /\"dbname\":\"test_migration\"/
end
end
end
end
end
it_should_behave_like 'migrating metadata', true
it_should_behave_like 'migrating metadata', false
it 'exports orgs with datasets without physical table if metadata export is requested (see #13721)' do
@map, @table, @table_visualization, @visualization = create_full_visualization(@carto_org_user_1)
@carto_org_user_1.tables.exists?(name: @table.name).should be
@org_user_1.in_database.execute("DROP TABLE #{@table.name}")
# The table is still registered after the deletion
@carto_org_user_1.reload
@carto_org_user_1.tables.exists?(name: @table.name).should be
export = Carto::UserMigrationExport.create(organization: @carto_organization, export_metadata: true)
export.run_export
export.log.entries.should_not include("Cannot export if tables aren't synched with db. Please run ghost tables.")
expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)
export.destroy
end
end
it '#run_import does not modify database_host with dry' do
user = create_user_with_visualizations
carto_user = Carto::User.find(user.id)
database_host = carto_user.database_host
export = Carto::UserMigrationExport.create(
user: carto_user,
export_metadata: true
)
export.run_export
drop_user_database(user)
# Let's fake the column to check that dry doesn't fix it
carto_user.update_column(:database_host, 'wadus')
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: database_host,
org_import: false,
json_file: export.json_file,
import_metadata: false,
dry: true
)
import.run_import
import.state.should eq 'complete'
carto_user2 = Carto::User.find(user.id)
carto_user2.attributes.should eq carto_user.attributes
end
describe 'api keys import and exports' do
before :each do
@user = FactoryGirl.build(:valid_user)
@user.save
@carto_user = Carto::User.find(@user.id)
@master_api_key = @carto_user.api_keys.master.first
@map, @table, @table_visualization, @visualization = create_full_visualization(@carto_user)
@regular_api_key = Carto::ApiKey.create_regular_key!(user: @carto_user,
name: 'Some ApiKey',
grants: [
{
type: "apis",
apis: ["maps", "sql"]
},
{
type: 'database',
tables: [
{
schema: @carto_user.database_schema,
name: @table.name,
permissions: ['select']
}
],
schemas: [
{
name: @carto_user.database_schema,
permissions: ['create']
}
]
}
])
end
after :each do
@user.destroy
end
it 'api keys are in redis and db roles are created' do
user_attributes = @carto_user.attributes
export = Carto::UserMigrationExport.create(
user: @carto_user,
export_metadata: true
)
export.run_export
puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)
username = @user.username
$users_metadata.hmget("api_keys:#{username}:#{@master_api_key.token}", 'user')[0].should eq username
$users_metadata.hmget("api_keys:#{username}:#{@regular_api_key.token}", 'user')[0].should eq username
@carto_user.client_applications.each(&:destroy)
@master_api_key.destroy
@table.destroy
@map.destroy
@table_visualization.destroy
@visualization.destroy
@carto_user.destroy
@regular_api_key.destroy
drop_user_database(@user)
$users_metadata.hmget("api_keys:#{username}:#{@master_api_key.token}", 'user')[0].should be nil
$users_metadata.hmget("api_keys:#{username}:#{@regular_api_key.token}", 'user')[0].should be nil
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.stubs(:assert_organization_does_not_exist)
import.stubs(:assert_user_does_not_exist)
import.run_import
puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
$users_metadata.hmget("api_keys:#{username}:#{@master_api_key.token}", 'user')[0].should eq username
$users_metadata.hmget("api_keys:#{username}:#{@regular_api_key.token}", 'user')[0].should eq username
user = Carto::User.find(user_attributes['id'])
user.should be
user.api_keys.each(&:table_permissions_from_db) # to make sure DB can be queried without exceptions
user.api_keys.select { |a| a.type == 'master' }.first.table_permissions_from_db.count.should be > 0
end
it 'keeps roles for oauth api keys with schemas grants and you can drop tables after migration' do
Cartodb::Central.stubs(:sync_data_with_cartodb_central?).returns(false)
oauth_app = Carto::OauthApp.create!(name: 'test',
user_id: @carto_user.id,
redirect_uris: ['https://example.com'],
website_url: 'http://localhost',
icon_url: 'https://example.com')
oauth_app_user = oauth_app.oauth_app_users.create!(user_id: @carto_user.id)
access_token = oauth_app_user.oauth_access_tokens.create!(scopes: ['schemas:c'])
api_key = access_token.api_key
with_connection_from_api_key(api_key) do |connection|
connection.execute("create table test_table(id INT)")
connection.execute("insert into test_table values (999)")
connection.execute("select id from test_table") do |result|
result[0]['id'].should eq '999'
end
end
user_attributes = @carto_user.attributes
export = Carto::UserMigrationExport.create(
user: @carto_user,
export_metadata: true
)
export.run_export
puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)
@carto_user.client_applications.each(&:destroy)
@master_api_key.destroy
@table.destroy
@map.destroy
@table_visualization.destroy
@visualization.destroy
# oauth_app must exist in the destination
# so we remove the user_id to avoid it being cascade deleted with the user
oauth_app.stubs(:sync_with_central?).returns(false)
oauth_app.stubs(:central_enabled?).returns(true)
oauth_app.user_id = nil
oauth_app.save!
@carto_user.destroy
@regular_api_key.destroy
drop_user_database(@user)
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.stubs(:assert_organization_does_not_exist)
import.stubs(:assert_user_does_not_exist)
import.run_import
puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
user = Carto::User.find(user_attributes['id'])
user.should be
user.api_keys.each(&:table_permissions_from_db) # to make sure DB can be queried without exceptions
user.api_keys.select { |a| a.type == 'master' }.first.table_permissions_from_db.count.should be > 0
expect(user.oauth_app_users.count).to eq 1
oauth_app_user = user.oauth_app_users.first
expect(oauth_app_user.exists_ownership_role?).to be_true
expect(oauth_app_user.oauth_access_tokens.count).to eq 1
access_token = oauth_app_user.oauth_access_tokens.first
api_key = access_token.api_key
with_connection_from_api_key(api_key) do |connection|
connection.execute("drop table test_table")
connection.execute("select relname from pg_class where relname = 'test_table'") do |result|
result.count eq 0
end
connection.execute("create table test_table(id INT)")
end
check_cdb_conf_query = "SELECT value->>'ownership_role_name' as c from cdb_conf where key = 'api_keys_' || '#{api_key.db_role}';"
user.in_database(as: :superuser).execute("SELECT cartodb.cdb_extension_reload()")
result = user.in_database(as: :superuser).execute(check_cdb_conf_query)
expect(result.count).to eq 1
Cartodb::Central.unstub(:sync_data_with_cartodb_central?)
oauth_app.destroy!
end
it 'api keys keeps the grants and you can drop tables after migration' do
regular_api_key = @carto_user.api_keys.regular.first
with_connection_from_api_key(regular_api_key) do |connection|
connection.execute("create table test_table(id INT)")
connection.execute("insert into test_table values (999)")
connection.execute("select id from test_table") do |result|
result[0]['id'].should eq '999'
end
end
user_attributes = @carto_user.attributes
export = Carto::UserMigrationExport.create(
user: @carto_user,
export_metadata: true
)
export.run_export
puts export.log.entries if export.state != Carto::UserMigrationExport::STATE_COMPLETE
expect(export.state).to eq(Carto::UserMigrationExport::STATE_COMPLETE)
@carto_user.client_applications.each(&:destroy)
@master_api_key.destroy
@table.destroy
@map.destroy
@table_visualization.destroy
@visualization.destroy
@carto_user.destroy
@regular_api_key.destroy
drop_user_database(@user)
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.stubs(:assert_organization_does_not_exist)
import.stubs(:assert_user_does_not_exist)
import.run_import
puts import.log.entries if import.state != Carto::UserMigrationImport::STATE_COMPLETE
expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
user = Carto::User.find(user_attributes['id'])
user.should be
user.api_keys.each(&:table_permissions_from_db) # to make sure DB can be queried without exceptions
user.api_keys.select { |a| a.type == 'master' }.first.table_permissions_from_db.count.should be > 0
with_connection_from_api_key(user.api_keys.master.first) do |connection|
connection.execute("drop table test_table")
connection.execute("select relname from pg_class where relname = 'test_table'") do |result|
result.count eq 0
end
end
end
end
end