diff --git a/app/services/carto/redis_export_service.rb b/app/services/carto/redis_export_service.rb index 4c93b07964..22ba8dd5d2 100644 --- a/app/services/carto/redis_export_service.rb +++ b/app/services/carto/redis_export_service.rb @@ -16,18 +16,18 @@ module Carto module RedisExportServiceImporter include RedisExportServiceConfiguration - def restore_redis_from_json_export(exported_json_string, user = nil) - restore_redis_from_hash_export(JSON.parse(exported_json_string).deep_symbolize_keys, user) + def restore_redis_from_json_export(exported_json_string) + restore_redis_from_hash_export(JSON.parse(exported_json_string).deep_symbolize_keys) end def remove_redis_from_json_export(exported_json_string) remove_redis_from_hash_export(JSON.parse(exported_json_string).deep_symbolize_keys) end - def restore_redis_from_hash_export(exported_hash, user) + def restore_redis_from_hash_export(exported_hash) raise 'Wrong export version' unless compatible_version?(exported_hash[:version]) - restore_redis(exported_hash[:redis], user) + restore_redis(exported_hash[:redis]) end def remove_redis_from_hash_export(exported_hash) @@ -36,12 +36,18 @@ module Carto remove_redis(exported_hash[:redis]) end + def restore_redis_do_subscriptions_from_json_export(exported_json_string, user) + exported_hash = JSON.parse(exported_json_string).deep_symbolize_keys + raise 'Wrong export version' unless compatible_version?(exported_hash[:version]) + + restore_do_subscriptions($users_metadata, exported_hash[:redis][:do_subscriptions], user) + end + private - def restore_redis(redis_export, user) + def restore_redis(redis_export) restore_keys($users_metadata, redis_export[:users_metadata]) restore_named_maps($tables_metadata, redis_export[:tables_metadata]) - restore_do_subscriptions($users_metadata, redis_export[:do_subscriptions], user) end def remove_redis(redis_export) diff --git a/app/services/carto/user_metadata_export_service.rb b/app/services/carto/user_metadata_export_service.rb index c3213a3a84..570f5b2afa 100644 --- a/app/services/carto/user_metadata_export_service.rb +++ b/app/services/carto/user_metadata_export_service.rb @@ -533,7 +533,7 @@ module Carto raise UserAlreadyExists.new if ::Carto::User.exists?(id: user.id) save_imported_user(user) - Carto::RedisExportService.new.restore_redis_from_json_export(redis_user_file(path), user) + Carto::RedisExportService.new.restore_redis_from_json_export(redis_user_file(path)) user end @@ -573,6 +573,11 @@ module Carto import_user_visualizations_from_directory(user, Carto::Visualization::TYPE_DERIVED, meta_path) import_search_tweets_from_directory(user, meta_path) + + Carto::RedisExportService.new.restore_redis_do_subscriptions_from_json_export( + redis_user_file(meta_path), + user + ) end def import_search_tweets_from_directory(user, meta_path) diff --git a/spec/services/carto/redis_export_service_spec.rb b/spec/services/carto/redis_export_service_spec.rb index ff90e36df4..e4ee56a158 100644 --- a/spec/services/carto/redis_export_service_spec.rb +++ b/spec/services/carto/redis_export_service_spec.rb @@ -31,7 +31,7 @@ describe Carto::RedisExportService do table_visualization.save! visualization.save! - yield(visualization) + yield(visualization, table_visualization) ensure map.destroy table.destroy @@ -48,6 +48,33 @@ describe Carto::RedisExportService do $tables_metadata.del("map_tpl|#{visualization.user.username}") end + def with_synchronization(visualization) + synchronization = create(:carto_synchronization, visualization: visualization) + + yield(synchronization) + ensure + synchronization.destroy + end + + def with_do_subscription(synchronization) + redis_key = "do:#{@user.username}:datasets" + $users_metadata.hset(redis_key, 'bq', [{ + dataset_id: 'dataset-id', + status: 'active', + available_in: ['bq', 'bq-sample'], + license_type: 'full-access', + type: 'dataset', + sync_status: 'synced', + sync_table: synchronization.visualization.user_table.name, + sync_table_id: synchronization.visualization.user_table.id, + synchronization_id: synchronization.id + }].to_json) + + yield + ensure + $users_metadata.del(redis_key) + end + def check_export(export, prefix) expect(export[:redis][:users_metadata].keys.sort).to eq(["#{prefix}:hash", "#{prefix}:set", "#{prefix}:string"]) end @@ -57,6 +84,13 @@ describe Carto::RedisExportService do expect(export[:redis][:tables_metadata]["map_tpl|#{visualization.user.username}"]['custom_named']).to eq(Base64.encode64("{:c=>\"custom\"}")) end + def check_do_subscriptions(export, synchronization) + expect(export[:redis][:do_subscriptions].keys).to eq(["do:#{@user.username}:datasets"]) + expect( + JSON.parse(export[:redis][:do_subscriptions]["do:#{@user.username}:datasets"])['synchronization_id'] + ).to eq(synchronization.id) + end + def check_redis(prefix) expect($users_metadata.get("#{prefix}:string")).to eq('something') expect($users_metadata.zrange("#{prefix}:set", 0, -1, withscores: true)).to eq([['set_key', 5.0]]) @@ -67,6 +101,13 @@ describe Carto::RedisExportService do expect($tables_metadata.hgetall("map_tpl|#{@user.username}")). to eq("custom_named" => "{:c=>\"custom\"}") end + def check_redis_do_subscriptions + expect($users_metadata.hget("do:#{@user.username}:datasets", 'bq').present?).to be_true + expect( + JSON.parse($users_metadata.hget("do:#{@user.username}:datasets", 'bq')).first['dataset_id'] + ).to eq('dataset-id') + end + let(:service) { Carto::RedisExportService.new } describe '#export' do @@ -93,13 +134,24 @@ describe Carto::RedisExportService do end it 'includes non-cartodb-managed named maps' do - with_visualization do |visualization| + with_visualization do |visualization, _| with_named_maps(visualization) do export = service.export_user_json_hash(@user) check_named_maps(export, visualization) end end end + + it 'includes DO subscriptions' do + with_visualization do |_, table_visualization| + with_synchronization(table_visualization) do |synchronization| + with_do_subscription(synchronization) do + export = service.export_user_json_hash(@user) + check_do_subscriptions(export, synchronization) + end + end + end + end end describe '#export + import' do @@ -111,7 +163,7 @@ describe Carto::RedisExportService do end it 'copies all keys under user:username for users' do - with_visualization do |visualization| + with_visualization do |visualization, _| prefix = "user:#{@user.username}" export = with_redis_keys(prefix) do with_named_maps(visualization) do @@ -123,5 +175,18 @@ describe Carto::RedisExportService do check_redis_named_maps end end + + it 'copies DO subscriptions' do + with_visualization do |_, table_visualization| + with_synchronization(table_visualization) do |synchronization| + export = with_do_subscription(synchronization) do + service.export_user_json_hash(@user) + end + + service.restore_redis_do_subscriptions_from_json_export(export.to_json, @user) + check_redis_do_subscriptions + end + end + end end end diff --git a/spec/services/carto/user_metadata_export_service_spec.rb b/spec/services/carto/user_metadata_export_service_spec.rb index f7d67557df..a6ece0181c 100644 --- a/spec/services/carto/user_metadata_export_service_spec.rb +++ b/spec/services/carto/user_metadata_export_service_spec.rb @@ -103,6 +103,19 @@ describe Carto::UserMetadataExportService do create(:oauth_access_tokens, oauth_app_user: oauth_app_user, api_key: api_key) create(:oauth_refresh_tokens, oauth_app_user: oauth_app_user, scopes: ['offline']) + # DO subscriptions + $users_metadata.hset("do:#{@user.username}:datasets", 'bq', [{ + dataset_id: 'dataset-id', + status: 'active', + available_in: ['bq', 'bq-sample'], + license_type: 'full-access', + type: 'dataset', + sync_status: 'synced', + sync_table: @table.name, + sync_table_id: @table.id, + synchronization_id: sync.id + }].to_json) + @user.reload end @@ -497,6 +510,12 @@ describe Carto::UserMetadataExportService do def expect_redis_restored(user) expect(CartoDB::GeocoderUsageMetrics.new(user.username).get(:geocoder_here, :success_responses)).to eq(1) + + expect(user.subscriptions.count).to eq(1) + expect(user.subscriptions.first['sync_table']).to eq(@table.name) + if user.tables.exists?(name: user.subscriptions.first['sync_table']) + expect(user.subscriptions.first['sync_table_id']).not_to eq(@table.id) + end end def expect_export_matches_search_tweet(exported_search_tweet, search_tweet) @@ -745,6 +764,9 @@ describe Carto::UserMetadataExportService do $tables_metadata.del(Carto::Visualization::V2_VISUALIZATIONS_REDIS_KEY) expect(@visualization.uses_vizjson2?).to be_false + # Clean redis DO subscriptions + $tables_metadata.del("do:#{@user.username}:datasets", 'bq') + @imported_user = service.import_from_directory(path) service.import_metadata_from_directory(@imported_user, path)