From 91c0abc74cdb45890f012588343fde35f4ecb1ce Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 4 Feb 2020 16:29:21 +0100 Subject: [PATCH] Adapt connector tests --- services/importer/spec/doubles/connector.rb | 94 + .../spec/unit/connector_runner_spec.rb | 142 +- services/importer/spec/unit/connector_spec.rb | 2286 +---------------- services/importer/spec/unit/sql_helper.rb | 126 - .../carto/api/connectors_controller_spec.rb | 95 +- 5 files changed, 223 insertions(+), 2520 deletions(-) create mode 100644 services/importer/spec/doubles/connector.rb delete mode 100644 services/importer/spec/unit/sql_helper.rb diff --git a/services/importer/spec/doubles/connector.rb b/services/importer/spec/doubles/connector.rb new file mode 100644 index 0000000000..981e7129b0 --- /dev/null +++ b/services/importer/spec/doubles/connector.rb @@ -0,0 +1,94 @@ +require 'carto/connector/provider' + +class DummyConnectorProvider < Carto::Connector::Provider + metadata id: 'dummy', name: 'Dummy' + required_parameters %I(table req1 req2) + optional_parameters %I(opt1 opt2) + + @copies = [] + def self.copies + @copies + end + + @error_message = nil + def self.failing_with(msg) + prev = @error_message + @error_message = msg + yield + @error_message = prev + end + def self.error_message + @error_message + end + + DEFAULT_FEATURES = { + 'sql_queries': false, + 'list_databases': false, + 'list_tables': true, + 'preview_table': false, + 'dry_run': false, + 'list_projects': false + } + + def copy_table(schema_name:, table_name:, limits: {}) + validate! + raise self.class.error_message if self.class.error_message + self.class.copies << [schema_name, table_name, limits] + @connector_context.execute_in_user_database "CREATE TABLE #{schema_name}.#{table_name}()" + end + + def check_connection + self.class.error_message ? false : true + end + + def table_name + @params[:table] + end + + def remote_data_updated? + true + end + + def list_tables(limits: {}) + must_be_defined_in_derived_class unless features_information[:list_tables] + [{schema:'s1', name: 't1'}, {schema:'s2', name: 't2'}] + end + + def list_databases() + must_be_defined_in_derived_class unless features_information[:list_databases] + ['db1', 'db2'] + end + + def list_projects() + must_be_defined_in_derived_class unless features_information[:list_projects] + [{id: 'project-1', friendly_name: 'Project 1'}, {id: 'project-2', friendly_name: 'Project 2'}] + end + + def list_project_datasets(project_id) + must_be_defined_in_derived_class unless features_information[:list_projects] + [{id: 'data-1', qualified_name: "#{project_id}.data-1"}, {id: 'data-2', qualified_name: "#{project_id}.data-2"}] + end + + def list_project_dataset_tables(project_id, dataset_id) + must_be_defined_in_derived_class unless features_information[:list_projects] + [{id: 't-1', qualified_name: "#{project_id}.#{dataset_id}.t-1"}, {id: 't-2', qualified_name: "#{project_id}.#{dataset_id}.t-2"}] + end + + def dry_run + must_be_defined_in_derived_class unless features_information[:dry_run] + raise self.class.error_message if self.class.error_message + {dry_run_results: '...'} + end + + def features_information + DEFAULT_FEATURES + end +end + +def dummy_connector_provider_with_id(id, name=nil, features=DummyConnectorProvider::DEFAULT_FEATURES) + Class.new(DummyConnectorProvider) do + metadata id: id, name: name || id + @copies = [] + define_method(:features_information){ features } + end +end \ No newline at end of file diff --git a/services/importer/spec/unit/connector_runner_spec.rb b/services/importer/spec/unit/connector_runner_spec.rb index 56743a1332..361f817e06 100644 --- a/services/importer/spec/unit/connector_runner_spec.rb +++ b/services/importer/spec/unit/connector_runner_spec.rb @@ -8,9 +8,8 @@ require_relative '../doubles/log' require_relative '../doubles/indexer' require_relative '../factories/pg_connection' require_relative '../doubles/user' -require_relative '../doubles/input_file_size_limit' -require_relative '../doubles/table_row_count_limit' require_relative '../../../../spec/helpers/feature_flag_helper' +require_relative '../doubles/connector' describe CartoDB::Importer2::ConnectorRunner do before(:all) do @@ -19,8 +18,9 @@ describe CartoDB::Importer2::ConnectorRunner do @pg_options = @user.db_service.db_configuration_for @feature_flag = FactoryGirl.create(:feature_flag, name: 'carto-connectors', restricted: true) @fake_log = CartoDB::Importer2::Doubles::Log.new(@user) - @providers = %w(mysql postgres sqlserver hive) + @providers = %w(dummy) @fake_log.clear + Carto::Connector::PROVIDERS << DummyConnectorProvider Carto::Connector.providers(all: true).keys.each do |provider_name| Carto::ConnectorProvider.create! name: provider_name end @@ -37,30 +37,23 @@ describe CartoDB::Importer2::ConnectorRunner do Carto::Connector.providers(all: true).keys.each do |provider_name| Carto::ConnectorProvider.find_by_name(provider_name).destroy end + Carto::Connector::PROVIDERS.delete DummyConnectorProvider + end + + after(:each) do + DummyConnectorProvider.copies.clear end include FeatureFlagHelper describe 'with working connectors' do - before(:all) do - # Simulate connector success by ignoring all db opeartions - Carto::Connector::Context.any_instance.stubs(:execute_as_superuser).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute_as_superuser_with_timeout).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute_with_timeout).returns(nil) - end - it "Succeeds if parameters are correct" do with_feature_flag @user, 'carto-connectors', true do parameters = { - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding' + req1: 'a', + req2: 'b', + opt1: 'c' } options = { pg: @pg_options, @@ -76,20 +69,18 @@ describe CartoDB::Importer2::ConnectorRunner do connector.provider_name.should eq provider end end + DummyConnectorProvider.copies.size.should eq 1 + DummyConnectorProvider.copies[0][0].should eq 'cdb_importer' + DummyConnectorProvider.copies[0][1].should match /\Aimporter_/ end end it "Fails if parameters are invalid" do with_feature_flag @user, 'carto-connectors', true do parameters = { - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding', + req1: 'a', + req2: 'b', invalid_parameter: 'xyz' } options = { @@ -110,17 +101,37 @@ describe CartoDB::Importer2::ConnectorRunner do end end + it "Fails if parameters are missing" do + with_feature_flag @user, 'carto-connectors', true do + parameters = { + table: 'thetable', + req1: 'a', + opt1: 'c' + } + options = { + pg: @pg_options, + log: @fake_log, + user: @user + } + @providers.each do |provider| + config = { provider => { 'enabled' => true } } + Cartodb.with_config connectors: config do + connector = CartoDB::Importer2::ConnectorRunner.new(parameters.merge(provider: provider).to_json, options) + connector.run + connector.success?.should be false + connector.provider_name.should eq provider + @fake_log.to_s.should match /Error Missing required parameters req2/m + end + end + end + end + it "Fails without the feature flag" do with_feature_flag @user, 'carto-connectors', false do parameters = { - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding' + req1: 'a', + req2: 'b' } options = { pg: @pg_options, @@ -142,14 +153,9 @@ describe CartoDB::Importer2::ConnectorRunner do it "Fails if provider is not available" do with_feature_flag @user, 'carto-connectors', true do parameters = { - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding' + req1: 'a', + req2: 'b', } options = { pg: @pg_options, @@ -170,25 +176,12 @@ describe CartoDB::Importer2::ConnectorRunner do end describe 'with failing connectors' do - before(:all) do - # Simulate connector success when executing non-privileged SQL - Carto::Connector::Context.any_instance.stubs(:execute_as_superuser).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute_as_superuser_with_timeout).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute_with_timeout).raises("SQL EXECUTION ERROR") - end - it "Always fails" do with_feature_flag @user, 'carto-connectors', true do parameters = { - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding' + req1: 'a', + req2: 'b' } options = { pg: @pg_options, @@ -196,13 +189,15 @@ describe CartoDB::Importer2::ConnectorRunner do user: @user } @providers.each do |provider| - config = { provider => { 'enabled' => true } } - Cartodb.with_config connectors: config do - connector = CartoDB::Importer2::ConnectorRunner.new(parameters.merge(provider: provider).to_json, options) - connector.run - connector.success?.should be false - connector.provider_name.should eq provider - @fake_log.to_s.should match /SQL EXECUTION ERROR/m + Carto::Connector.provider_class(provider).failing_with('COPY ERROR') do + config = { provider => { 'enabled' => true } } + Cartodb.with_config connectors: config do + connector = CartoDB::Importer2::ConnectorRunner.new(parameters.merge(provider: provider).to_json, options) + connector.run + connector.success?.should be false + connector.provider_name.should eq provider + @fake_log.to_s.should match /COPY ERROR/m + end end end end @@ -210,24 +205,14 @@ describe CartoDB::Importer2::ConnectorRunner do end describe 'with invalid provider' do - Carto::Connector::Context.any_instance.stubs(:execute_as_superuser).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute_as_superuser_with_timeout).returns(nil) - Carto::Connector::Context.any_instance.stubs(:execute_with_timeout).returns(nil) - it "Fails at creation" do with_feature_flag @user, 'carto-connectors', true do parameters = { provider: 'invalid_provider', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding' + req1: 'a', + req2: 'b' } options = { pg: @pg_options, @@ -254,15 +239,10 @@ describe CartoDB::Importer2::ConnectorRunner do with_feature_flag @user, 'carto-connectors', true do parameters = { - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, table: 'thetable', - encoding: 'theencoding' - } + req1: 'a', + req2: 'b' + } options = { pg: @pg_options, log: @fake_log, diff --git a/services/importer/spec/unit/connector_spec.rb b/services/importer/spec/unit/connector_spec.rb index 6bad6fe6f4..f39fd9912d 100644 --- a/services/importer/spec/unit/connector_spec.rb +++ b/services/importer/spec/unit/connector_spec.rb @@ -4,101 +4,17 @@ require_relative '../../../../spec/spec_helper' require_relative '../doubles/importer_stats' require_relative '../doubles/loader' require_relative '../doubles/log' -require_relative '../doubles/indexer' -require_relative '../factories/pg_connection' require_relative '../doubles/user' -require_relative '../doubles/input_file_size_limit' -require_relative '../doubles/table_row_count_limit' - -require_relative 'sql_helper' - -class TestConnectorContext < Carto::Connector::Context - def initialize(executed_commands, options) - @executed_commands = executed_commands - super options - end - - def execute_as_superuser(command) - @executed_commands << [:superuser, command, @user.username] - execute_results command - end - - def execute_as_superuser_with_timeout(command, timeout) - @executed_commands << [:superuser, command, @user.username] - execute_results command - end - - def execute(command) - @executed_commands << [:user, command, @user.username] - execute_results command - end - - def execute_with_timeout(command, timeout) - @executed_commands << [:user, command, @user.username] - execute_results command - end - - private - - def execute_results(command) - if command =~ /\A\s*SELECT\s+\*\s+FROM\s+ODBCTablesList/mi - [{ schema: 'abc', name: 'xyz' }] - elsif command =~ /SELECT\s+n.nspname\s+AS\s+schema,\s*c.relname\s+AS\s+name/mi - [{ schema: 'def', name: 'uvw' }] - else - [] - end - end -end - -class FailingTestConnectorContext < TestConnectorContext - def execute_with_timeout(command, timeout) - if match_sql(command).first[:command] == :create_table_as_select - raise "SQL EXECUTION ERROR" - end - super - end -end - -class TestCountConnectorContext < TestConnectorContext - def initialize(count, *args) - @test_count = count - super *args - end - - def execute(command) - if command =~ /\A\s*SELECT\s+count\(\*\)\s+AS\s+num_rows/mi - [{ 'num_rows' => @test_count }] - else - super - end - end -end - -def expect_executed_command(cmd, expected = {}) - if expected - mode, sql, user = cmd - mode.should eq expected[:mode] if expected.key?(:mode) - user.should eq expected[:user] if expected.key?(:user) - expect_sql sql, expected[:sql] if expected.key?(:sql) - end -end - -def expect_executed_commands(executed_commands, *expected_commands) - executed_commands.zip(expected_commands).each do |executed_command, expected_command| - expect_executed_command executed_command, expected_command - end -end - -# Multiple hashes are passed to `expect_executed_commands` -# and omiting the braces of the last one is would be inconvenient, so: -# rubocop:disable Style/BracesAroundHashParameters +require_relative '../doubles/connector' describe Carto::Connector do before(:all) do @user = create_user @user.save @fake_log = CartoDB::Importer2::Doubles::Log.new(@user) + Carto::Connector::PROVIDERS << DummyConnectorProvider + Carto::Connector::PROVIDERS << dummy_connector_provider_with_id('another_dummy') + Carto::Connector::PROVIDERS << dummy_connector_provider_with_id('third_dummy') Carto::Connector.providers.keys.each do |provider_name| Carto::ConnectorProvider.create! name: provider_name end @@ -106,7 +22,6 @@ describe Carto::Connector do before(:each) do CartoDB::Stats::Aggregator.stubs(:read_config).returns({}) - @executed_commands = [] end around(:each) do |example| @@ -121,2205 +36,32 @@ describe Carto::Connector do end it "Should list providers available for a user with default configuration" do - default_config = { 'mysql' => { 'enabled' => true }, 'postgres' => { 'enabled' => false } } + default_config = { 'dummy' => { 'enabled' => true }, 'another_dummy' => { 'enabled' => false } } Cartodb.with_config connectors: default_config do Carto::Connector.providers(user: @user).should == { - "postgres" => { name: "PostgreSQL", enabled: false, description: nil }, - "mysql" => { name: "MySQL", enabled: true, description: nil }, - "sqlserver" => { name: "Microsoft SQL Server", enabled: false, description: nil }, - "hive" => { name: "Hive", enabled: false, description: nil } + "dummy" => { name: "Dummy", enabled: true, description: nil }, + "another_dummy" => { name: "another_dummy", enabled: false, description: nil }, + "third_dummy" => { name: "third_dummy", enabled: false, description: nil } } end end it "Should list providers available for a user with specific configuration" do - default_config = { 'mysql' => { 'enabled' => true }, 'postgres' => { 'enabled' => false } } - postgres = Carto::ConnectorProvider.find_by_name('postgres') + default_config = { 'dummy' => { 'enabled' => true }, 'another_dummy' => { 'enabled' => false } } + dummy = Carto::ConnectorProvider.find_by_name('dummy') user_config = Carto::ConnectorConfiguration.create!( - connector_provider_id: postgres.id, + connector_provider_id: dummy.id, user_id: @user.id, enabled: true ) Cartodb.with_config connectors: default_config do Carto::Connector.providers(user: @user).should == { - "postgres" => { name: "PostgreSQL", enabled: true, description: nil }, - "mysql" => { name: "MySQL", enabled: true, description: nil }, - "sqlserver" => { name: "Microsoft SQL Server", enabled: false, description: nil }, - "hive" => { name: "Hive", enabled: false, description: nil } + "dummy" => { name: "Dummy", enabled: true, description: nil }, + "another_dummy" => { name: "another_dummy", enabled: false, description: nil }, + "third_dummy" => { name: "third_dummy", enabled: false, description: nil } } end user_config.destroy end - - describe 'mysql' do - it 'Executes expected odbc_fdw SQL commands to copy a table' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'can import an external table under a different name' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - import_as: 'theimportedtable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - connector.table_name.should == 'theimportedtable' - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'can import a query' do - sql_query = 'SELECT 1000 AS value' - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - sql_query: sql_query, - import_as: 'theimportedtable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_theimportedtable"} - user_name = @user.username - user_role = @user.database_username - connector.table_name.should == 'theimportedtable' - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "sql_query" => sql_query, - "table" => 'theimportedtable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'can can use the legacy table parameter to name a query' do - # this is for backwards compatibility, but deprecated - sql_query = 'SELECT 1000 AS value' - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - sql_query: sql_query, - table: 'theimportedtable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_theimportedtable"} - user_name = @user.username - user_role = @user.database_username - connector.table_name.should == 'theimportedtable' - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "sql_query" => sql_query, - "table" => 'theimportedtable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Should quote ODBC parameters that require it' do - parameters = { - provider: 'mysql', - connection: { - server: 'the;server', - username: 'theuser', - password: 'the;password', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => '{the;server}', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => '{the;password}' } - }, { - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => '{the;password}' } - }] - } - ) - end - - it 'Fails when parameters are not valid' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding', - invalid_parameter: 'xyz' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - expect { - connector.copy_table schema_name: 'xyz', table_name: 'abc' - }.to raise_error(Carto::Connector::InvalidParametersError) - - # When parameters are not valid nothing should be executed in the database - @executed_commands.should be_empty - end - - it 'Fails gracefully when copy errs' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = FailingTestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - expect { - connector.copy_table schema_name: 'xyz', table_name: 'abc' - }.to raise_error('SQL EXECUTION ERROR') - - # When something fails during table copy the foreign table, user mappings and server should be cleaned up - @executed_commands.size.should eq 8 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Limits the number of rows copied from a table' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - config = { 'mysql' => { 'enabled' => true, 'max_rows' => 10 } } - Cartodb.with_config connectors: config do - context = TestCountConnectorContext.new(5, @executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - result = connector.copy_table schema_name: 'xyz', table_name: 'abc' - result.should be_empty - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/, - limit: '10' - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - end - - it 'Limits the number of rows and warns if limit is reached' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - config = { 'mysql' => { 'enabled' => true, 'max_rows' => 10 } } - Cartodb.with_config connectors: config do - context = TestCountConnectorContext.new(10, @executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - result = connector.copy_table schema_name: 'xyz', table_name: 'abc' - result[:max_rows_per_connection].should eq 10 - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'thedatabase', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/, - limit: '10' - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - end - - it 'Executes expected odbc_fdw SQL commands to list tables' do - parameters = { - provider: 'mysql', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - } - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - tables = connector.list_tables - - tables.should == [{ schema: 'abc', name: 'xyz' }] - - @executed_commands.size.should eq 7 - - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'MySQL', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase', - 'odbc_port' => '3306', - "odbc_option" => '0', - "odbc_prefetch" => '0', - "odbc_no_ssps" => '0', - "odbc_can_handle_exp_pwd" => '0' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # FETCH TABLES LIST - mode: :user, - user: user_name, - sql: [{ - command: :select_all, - from: /ODBCTablesList\('#{Regexp.escape server_name}'\s*,\d+\s*\)/ - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'requires connection parameters in order to list tables' do - parameters = { - provider: 'mysql' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - expect { - connector.copy_table schema_name: 'xyz', table_name: 'abc' - }.to raise_error(Carto::Connector::InvalidParametersError) - - # When parameters are not valid nothing should be executed in the database - @executed_commands.should be_empty - end - - it 'checks connection paramters in order to list tables' do - parameters = { - provider: 'mysql', - connection: { - # missing server - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - } - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - expect { - connector.copy_table schema_name: 'xyz', table_name: 'abc' - }.to raise_error(Carto::Connector::InvalidParametersError) - - # When parameters are not valid nothing should be executed in the database - @executed_commands.should be_empty - end - - it 'Should provide connector metadata' do - Carto::Connector.information('mysql').should == { - features: { - 'list_tables': true, - 'list_databases': false, - 'sql_queries': true, - 'preview_table': false, - 'dry_run': false, - 'list_projects': false - }, - parameters: { - 'connection' => { - 'username' => { required: true }, - 'password' => { required: true }, - 'server' => { required: true }, - 'port' => { required: false }, - 'database' => { required: false } - }, - 'table' => { required: false }, - 'import_as' => { required: false }, - 'schema' => { required: false }, - 'sql_query' => { required: false }, - 'sql_count' => { required: false }, - 'encoding' => { required: false }, - 'columns' => { required: false } - } - } - end - end - - describe 'postgresql' do - it 'Executes expected odbc_fdw SQL commands to copy a table' do - parameters = { - provider: 'postgres', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'PostgreSQL Unicode', - 'odbc_Server' => 'theserver', - 'odbc_Port' => '5432', - 'odbc_Database' => 'thedatabase', - "odbc_BoolsAsChar" => '0', - "odbc_ByteaAsLongVarBinary" => '1', - "odbc_MaxVarcharSize" => '256', - "odbc_SSLmode" => 'require' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_UID' => 'theuser', 'odbc_PWD' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_UID' => 'theuser', 'odbc_PWD' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - remote_schema_name: 'public', - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'public', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Should provide connector metadata' do - Carto::Connector.information('postgres').should == { - features: { - 'list_tables': true, - 'list_databases': false, - 'sql_queries': true, - 'preview_table': false, - 'dry_run': false, - 'list_projects': false - }, - parameters: { - 'connection' => { - 'username' => { required: true }, - 'password' => { required: false }, - 'server' => { required: true }, - 'port' => { required: false }, - 'database' => { required: true }, - 'sslmode' => { required: false } - }, - 'table' => { required: false }, - 'import_as' => { required: false }, - 'schema' => { required: false }, - 'sql_query' => { required: false }, - 'sql_count' => { required: false }, - 'encoding' => { required: false }, - 'columns' => { required: false } - } - } - end - end - - describe 'sqlserver' do - it 'Executes expected odbc_fdw SQL commands to copy a table' do - parameters = { - provider: 'sqlserver', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'FreeTDS', - 'odbc_Server' => 'theserver', - 'odbc_Port' => '1433', - 'odbc_Database' => 'thedatabase', - "odbc_AppicationIntent" => 'ReadOnly' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_UID' => 'theuser', 'odbc_PWD' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_UID' => 'theuser', 'odbc_PWD' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - remote_schema_name: 'dbo', - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'dbo', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Should provide connector metadata' do - Carto::Connector.information('sqlserver').should == { - features: { - 'list_tables': true, - 'list_databases': false, - 'sql_queries': true, - 'preview_table': false, - 'dry_run': false, - 'list_projects': false - }, - parameters: { - 'connection' => { - 'username' => { required: true }, - 'password' => { required: true }, - 'server' => { required: true }, - 'port' => { required: false }, - 'database' => { required: true } - }, - 'table' => { required: false }, - 'import_as' => { required: false }, - 'schema' => { required: false }, - 'sql_query' => { required: false }, - 'sql_count' => { required: false }, - 'encoding' => { required: false }, - 'columns' => { required: false } - } - } - end - end - - describe 'hive' do - it 'Executes expected odbc_fdw SQL commands to copy a table' do - parameters = { - provider: 'hive', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_Driver' => 'Hortonworks Hive ODBC Driver 64-bit', - 'odbc_HOST' => 'theserver', - 'odbc_PORT' => '10000', - "odbc_Schema" => 'default' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_UID' => 'theuser', 'odbc_PWD' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_UID' => 'theuser', 'odbc_PWD' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - remote_schema_name: 'default', - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "schema" => 'default', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Should provide connector metadata' do - Carto::Connector.information('hive').should == { - features: { - 'list_tables': true, - 'list_databases': false, - 'sql_queries': true, - 'preview_table': false, - 'dry_run': false, - 'list_projects': false - }, - parameters: { - 'connection' => { - 'username' => { required: false }, - 'password' => { required: false }, - 'server' => { required: true }, - 'port' => { required: false }, - 'database' => { required: false } - }, - 'table' => { required: false }, - 'import_as' => { required: false }, - 'schema' => { required: false }, - 'sql_query' => { required: false }, - 'sql_count' => { required: false }, - 'encoding' => { required: false }, - 'columns' => { required: false } - } - } - end - end - - describe 'invalid_provider' do - it 'Fails' do - parameters = { - provider: 'not_a_provider', - connection: { - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - expect { - Carto::Connector.new(parameters, options) - }.to raise_error(Carto::Connector::InvalidParametersError) - end - - it 'Should not provide metadata' do - expect { - Carto::Connector.information('not_a_provider') - }.to raise_error(Carto::Connector::InvalidParametersError) - end - end - - describe 'generic odbc provider' do - it 'Executes expected odbc_fdw SQL commands to copy a table' do - parameters = { - provider: 'odbc', - connection: { - driver: 'thedriver', - server: 'theserver', - uid: 'theuser', - pwd: 'thepassword', - database: 'thedatabase', - # anything can actually go here - aaa: 'aaa_value', - bbb: 'bbb_value', - ccc: 'ccc_value' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_driver' => 'thedriver', - 'odbc_server' => 'theserver', - 'odbc_database' => 'thedatabase' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "odbc_aaa" => 'aaa_value', - "odbc_bbb" => 'bbb_value', - "odbc_ccc" => 'ccc_value', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Should admit quoted parameters' do - parameters = { - provider: 'odbc', - connection: { - driver: 'thedriver', - server: '{the;server}', - uid: 'theuser', - pwd: '{the;password}', - database: 'thedatabase', - # anything can actually go here - aaa: 'aaa_value', - bbb: 'bbb_value', - ccc: 'ccc_value' - }, - table: 'thetable', - encoding: 'theencoding' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - foreign_table_name = %{"cdb_importer"."#{server_name}_thetable"} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'odbc_fdw', - options: { - 'odbc_driver' => 'thedriver', - 'odbc_server' => '{the;server}', - 'odbc_database' => 'thedatabase' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => '{the;password}' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'odbc_uid' => 'theuser', 'odbc_pwd' => '{the;password}' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema, - server_name: server_name, - schema_name: 'cdb_importer', - options: { - "odbc_aaa" => 'aaa_value', - "odbc_bbb" => 'bbb_value', - "odbc_ccc" => 'ccc_value', - "table" => 'thetable', - "encoding" => 'theencoding', - "prefix" => "#{server_name}_" - } - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - end - - class PgProvider < Carto::Connector::PgFdwProvider - metadata id: 'pg', name: 'PostgreSQL FDW', public?: true - end - - describe 'Non odbc provider' do - before(:each) do - Carto::Connector::PROVIDERS << PgProvider - end - - after(:each) do - Carto::Connector::PROVIDERS.delete PgProvider - end - - it 'Executes expected odbc_fdw SQL commands to copy a table' do - parameters = { - provider: 'pg', - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase', - table: 'thetable' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - connector.copy_table schema_name: 'xyz', table_name: 'abc' - - @executed_commands.size.should eq 9 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - unqualified_foreign_table_name = %{"#{server_name}_thetable"} - foreign_table_name = %{"cdb_importer".#{unqualified_foreign_table_name}} - user_name = @user.username - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'postgres_fdw', - options: { - 'host' => 'theserver', - 'dbname' => 'thedatabase' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'user' => 'theuser', 'password' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'user' => 'theuser', 'password' => 'thepassword' } - }] - }, { - # IMPORT FOREIGN SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema_limited, - server_name: server_name, - limited_to: 'thetable', - schema_name: 'cdb_importer', - remote_schema_name: 'public' - }, { - command: :rename_foreign_table, - table_name: %{"cdb_importer"."thetable"}, - new_name: unqualified_foreign_table_name - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # CREATE TABLE AS SELECT - mode: :user, - user: user_name, - sql: [{ - command: :create_table_as_select, - table_name: %{"xyz"."abc"}, - select: /\s*\*\s+FROM\s+#{Regexp.escape foreign_table_name}/ - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Fails when parameters are not valid' do - parameters = { - provider: 'pg', - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase', - table: 'thetable', - invalid_param: 'xyz' - } - options = { - logger: @fake_log, - user: @user - } - context = TestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - expect { - connector.copy_table schema_name: 'xyz', table_name: 'abc' - }.to raise_error(Carto::Connector::InvalidParametersError) - - # When parameters are not valid nothing should be executed in the database - @executed_commands.should be_empty - end - - it 'Fails gracefully when copy errs' do - parameters = { - provider: 'pg', - server: 'theserver', - username: 'theuser', - password: 'thepassword', - database: 'thedatabase', - table: 'thetable' - } - options = { - logger: @fake_log, - user: @user - } - context = FailingTestConnectorContext.new(@executed_commands = [], options) - connector = Carto::Connector.new(parameters, context) - expect { - connector.copy_table schema_name: 'xyz', table_name: 'abc' - }.to raise_error('SQL EXECUTION ERROR') - - # When something fails during table copy the foreign table, user mappings and server should be cleaned up - @executed_commands.size.should eq 8 - server_name = match_sql_command(@executed_commands[0][1])[:server_name] - unqualified_foreign_table_name = %{"#{server_name}_thetable"} - foreign_table_name = %{"cdb_importer".#{unqualified_foreign_table_name}} - user_role = @user.database_username - - expect_executed_commands( - @executed_commands, - { - # CREATE SERVER - mode: :superuser, - sql: [{ - command: :create_server, - fdw_name: 'postgres_fdw', - options: { - 'host' => 'theserver', - 'dbname' => 'thedatabase' - } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: user_role, - options: { 'user' => 'theuser', 'password' => 'thepassword' } - }] - }, { - # CREATE USER MAPPING - mode: :superuser, - sql: [{ - command: :create_user_mapping, - server_name: server_name, - user_name: 'postgres', - options: { 'user' => 'theuser', 'password' => 'thepassword' } - }] - }, { - # IMPORT FOREIGH SCHEMA; GRANT SELECT - mode: :superuser, - sql: [{ - command: :import_foreign_schema_limited, - server_name: server_name, - limited_to: 'thetable', - schema_name: 'cdb_importer', - remote_schema_name: 'public' - }, { - command: :rename_foreign_table, - table_name: %{"cdb_importer"."thetable"}, - new_name: unqualified_foreign_table_name - }, { - command: :grant_select, - table_name: foreign_table_name, - user_name: user_role - }] - }, { - # DROP FOREIGN TABLE - mode: :superuser, - sql: [{ - command: :drop_foreign_table_if_exists, - table_name: foreign_table_name - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: 'postgres' - }] - }, { - # DROP USER MAPPING - mode: :superuser, - sql: [{ - command: :drop_usermapping_if_exists, - server_name: server_name, - user_name: user_role - }] - }, { - # DROP SERVER - mode: :superuser, - sql: [{ - command: :drop_server_if_exists, - server_name: server_name - }] - } - ) - end - - it 'Should provide connector metadata' do - Carto::Connector.information('pg').should == { - features: { - 'list_tables': true, - 'list_databases': false, - 'sql_queries': false, - 'preview_table': false, - 'dry_run': false, - 'list_projects': false - }, - parameters: { - 'table' => { required: true }, - 'schema' => { required: false }, - 'username' => { required: true }, - 'password' => { required: true }, - 'server' => { required: true }, - 'port' => { required: false }, - 'database' => { required: true } - } - } - end - end end -# rubocop:enable Style/BracesAroundHashParameters diff --git a/services/importer/spec/unit/sql_helper.rb b/services/importer/spec/unit/sql_helper.rb deleted file mode 100644 index 6bc9fa5d86..0000000000 --- a/services/importer/spec/unit/sql_helper.rb +++ /dev/null @@ -1,126 +0,0 @@ -def match_sql_command(sql) - options_pattern = %q{ - (?:\s+OPTIONS\s*\((? - (?: - \s* - (?:\"(?:[^\"]+)\"|(?:[^\s]+)) - \s+ - (?:\'(?:[^\']*)\'|(?:[^\'].+)) - )* - \s* - )\))? - } - patterns = { - create_server: %r{ - CREATE\s+SERVER\s+(?[^\s]+) - \s+ - FOREIGN\s+DATA\s+WRAPPER\s+(?[^\s]+) - #{options_pattern} - }xi, - create_user_mapping: %r{ - CREATE\s+USER\s+MAPPING\s+FOR\s+\"?(?[^\s\"]+)\"? - \s+ - SERVER\s+(?[^\s]+) - #{options_pattern} - }xi, - import_foreign_schema: %r{ - IMPORT\s+FOREIGN\s+SCHEMA\s+\"?(?[^\s\"]+)\"? - \s+ - FROM\s+SERVER\s+(?[^\s]+) - \s+ - INTO\s+\"?(?[^\s\"]+)\"? - #{options_pattern} - }xi, - import_foreign_schema_limited: %r{ - IMPORT\s+FOREIGN\s+SCHEMA\s+\"?(?[^\s\"]+)\"? - \s+ - LIMIT\s+TO\s+\((?.+)\) - \s+ - FROM\s+SERVER\s+(?[^\s]+) - \s+ - INTO\s+\"?(?[^\s\"]+)\"? - #{options_pattern} - }xi, - create_foreign_table: %r{ - CREATE\+FOREIGN\+TABLE\s+(?.+)\s*\((?.+)\) - \s+ - SERVER\s+(?[^\s]+) - #{options_pattern} - }xi, - grant_select: %r{ - GRANT\s+SELECT\s+ON\s+(?[^\s]+)\s+TO\s+\"?(?[^\s\"]+)\"? - }xi, - create_table_as_select: %r{ - CREATE\s+TABLE\s+(?[^\s]+)\s+AS\s+SELECT\s+(?