diff --git a/app/services/carto/data_library_service.rb b/app/services/carto/data_library_service.rb index 1207fd4e07..d03a48045e 100644 --- a/app/services/carto/data_library_service.rb +++ b/app/services/carto/data_library_service.rb @@ -40,6 +40,26 @@ module Carto visualization end + def load_datasets!(carto_api_client, + source_username:, source_api_key:, + target_username:, granted_api_key:) + api_keys = carto_api_client.get_api_keys_v3(username: source_username, params: { api_key: granted_api_key }) + api_keys[:result].each do |api_key| + database_grants = api_key[:grants].select { |g| g[:type] == 'database' } + database_grants.each do |database_grant| + tables = database_grant[:tables] + tables.each do |table| + if table[:permissions].include?('select') + load_dataset!(carto_api_client, + source_dataset: table[:name], + source_username: source_username, source_api_key: source_api_key, + target_username: target_username, granted_api_key: granted_api_key) + end + end + end + end + end + private def display_name(remote_visualization) diff --git a/lib/carto_api/json_client.rb b/lib/carto_api/json_client.rb index 9363cef57b..b2c0035a53 100644 --- a/lib/carto_api/json_client.rb +++ b/lib/carto_api/json_client.rb @@ -24,6 +24,10 @@ module CartoAPI get(url) end + def get_api_keys_v3(username:, params: {}) + parse(get(carto_url(username, '/api/v3/api_keys', params: params)).body) + end + def base_url(username) "#{username}.#{@base_domain}" end diff --git a/lib/tasks/remote_tables_maintenance.rake b/lib/tasks/remote_tables_maintenance.rake index bdcfa88827..049188b8ad 100644 --- a/lib/tasks/remote_tables_maintenance.rake +++ b/lib/tasks/remote_tables_maintenance.rake @@ -69,10 +69,23 @@ namespace :cartodb do puts DateTime.now end - load_in_data_library_args = [:scheme, :base_domain, :port, - :source_dataset, :source_username, - :source_api_key, - :target_username, :granted_api_key] + load_in_data_library_from_api_key_args = [:scheme, :base_domain, :port, :source_username, + :source_api_key, :target_username, :granted_api_key] + # Example: rake cartodb:remotes:load_in_data_library_from_api_key[https,carto.com,80,s_user,s_user_key,t_user,g_key] + desc 'Loads all datasets for a API key in a Data Library. `granted_api_key` will be stored for importing.' + task :load_in_data_library_from_api_key, load_in_data_library_from_api_key_args => [:environment] do |_, args| + raise "All arguments are mandatory" unless load_in_data_library_from_api_key_args.all? { |a| args[a].present? } + + client = CartoAPI::JsonClient.new( + scheme: args['scheme'], base_domain: args['base_domain'], port: args['port'].to_i + ) + service = Carto::DataLibraryService.new + service.load_datasets!(client, + source_username: args['source_username'], source_api_key: args['source_api_key'], + target_username: args['target_username'], granted_api_key: args['granted_api_key']) + end + + load_in_data_library_args = load_in_data_library_from_api_key_args + ['source_dataset'] # Example: rake cartodb:remotes:load_in_data_library[https,carto.com,80,s_data,s_user,s_user_key,t_user,g_key] desc 'Loads a dataset in a Data Library. `granted_api_key` is the key that will be stored for importing.' task :load_in_data_library, load_in_data_library_args => [:environment] do |_, args| diff --git a/spec/services/carto/data_library_service_spec.rb b/spec/services/carto/data_library_service_spec.rb index 20a7b44e54..161e04d984 100644 --- a/spec/services/carto/data_library_service_spec.rb +++ b/spec/services/carto/data_library_service_spec.rb @@ -32,7 +32,7 @@ describe Carto::DataLibraryService do "\"size\":53248,\"row_count\":124}}" end - describe '#load_dataset' do + describe '#load_dataset!' do it 'loads a remote dataset into a Data Library' do client_p = { scheme: 'https', base_domain: 'waduscarto.com', port: 666 } client = CartoAPI::JsonClient.new(**client_p) @@ -77,4 +77,56 @@ describe Carto::DataLibraryService do visualization.destroy end end + + let(:api_keys_response) do + { + total: 1, + count: 1, + result: [ + { + name: "test_ro_untitled_table_39", + user: { username: "juanignaciosl" }, + type: "regular", + token: "z17KQqbcdbDfIYcD4FTw", + grants: [ + { type: "apis", apis: ["maps", "sql"] }, + { type: "database", tables: [ + { schema: "juanignaciosl", name: "untitled_table_30", permissions: ["select"] } + ] } + ], + created_at: "2018-03-23 14:45:00 +0000", + updated_at: "2018-03-23 14:45:00 +0000", + _links: { :self => "https://juanignaciosl.wcarto.com/api/v3/api_keys/test_ro_untitled_table_39" } + } + ], + _links: { + first: { :href => "https://juanignaciosl.wcarto.com/api/v3/api_keys?order=updated_at&page=1&per_page=100000" }, + last: { :href => "https://juanignaciosl.wcarto.com/api/v3/api_keys?order=updated_at&page=1&per_page=100000" } + } + } + end + + describe '#load_datasets!' do + it 'loads remote datasets from an API key into a Data Library' do + client_p = { scheme: 'https', base_domain: 'wcarto.com', port: 666 } + client = CartoAPI::JsonClient.new(**client_p) + source_dataset = api_keys_response[:result][0][:grants][1][:tables][0][:name] + params = { + source_username: 'wadus-username', + source_api_key: 'rewadus-api_key', + target_username: @carto_user1.username, + granted_api_key: 'wadus-api_key' + } + + client.expects(:get_api_keys_v3) + .with(username: params[:source_username], + params: { api_key: params[:granted_api_key] }) + .returns(api_keys_response) + + service = Carto::DataLibraryService.new + params_load_one = { source_dataset: source_dataset }.merge(params) + service.expects(:load_dataset!).once.with(client, **params_load_one) + service.load_datasets!(client, **params) + end + end end