Ahg add db connectors in builds (#16202)

* checkout db-connectors

* Remove shared state in user exports spec

* bump private

Co-authored-by: Alberto Miedes Garcés <albertomg994@gmail.com>

private

@@ -1 +1 @@
-Subproject commit 0d530a06c2998c9ecaaae2236639c4f7d654de27
+Subproject commit d0e6e878514c1df283c2ee07c5d1d3bb03722142

script/ci/cloudbuild-build-master.yaml

@@ -55,6 +55,14 @@ steps:
   - name: 'ssh'
     path: /root/.ssh
 
+# Checkout db-connectors
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'sh'
+  args: ['-c', 'git clone git@github.com:CartoDB/db-connectors --depth 1 private_gears/db-connectors']
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
 # Copy files to the right directories to run the tests
 - name: ubuntu
   entrypoint: /bin/bash

script/ci/cloudbuild-build-pr-branch.yaml

@@ -55,6 +55,14 @@ steps:
   - name: 'ssh'
     path: /root/.ssh
 
+# Checkout db-connectors
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'sh'
+  args: ['-c', 'git clone git@github.com:CartoDB/db-connectors --depth 1 private_gears/db-connectors']
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
 # Copy files to the right directories to run the tests
 - name: ubuntu
   entrypoint: /bin/bash

script/ci/cloudbuild-tests-pr-pg11.yaml

@@ -1,97 +0,0 @@
-steps:
-
-# Cancel previous job on the same branch
-- name: gcr.io/cloud-builders/gcloud
-  entrypoint: /bin/bash
-  args:
-  - '-c'
-  - 'gcloud builds list --ongoing --filter="buildTriggerId=056221c7-f5ef-4857-9706-3bc1054799a3 AND substitutions.BRANCH_NAME=${BRANCH_NAME} AND id!=${BUILD_ID}" --format="get(ID)" > jobs_to_cancel'
-
-- name: gcr.io/cloud-builders/gcloud
-  entrypoint: /bin/bash
-  args:
-  - '-c'
-  - 'gcloud builds cancel $(cat jobs_to_cancel | xargs) || true'
-
-# Decrypt github key
-- name: gcr.io/cloud-builders/gcloud
-  args:
-  - kms
-  - decrypt
-  - --ciphertext-file=script/ci/.id_github_cartofante.enc
-  - --plaintext-file=/root/.ssh/id_rsa
-  - --location=global
-  - --keyring=keyring-github-ci
-  - --key=github-key
-  volumes:
-  - name: 'ssh'
-    path: /root/.ssh
-
-# Set up git with key and domain
-- name: 'gcr.io/cloud-builders/git'
-  entrypoint: 'bash'
-  args:
-  - '-c'
-  - |
-    chmod 600 /root/.ssh/id_rsa
-    cat <<EOF >/root/.ssh/config
-    Hostname github.com
-    IdentityFile /root/.ssh/id_rsa
-    EOF
-    mv script/ci/known_hosts /root/.ssh/known_hosts
-  volumes:
-  - name: 'ssh'
-    path: /root/.ssh
-
-# Clean working directory
-- name: alpine
-  args: ['sh', '-c', 'rm -rf .* | true && rm -rf *']
-
-# Checkout repo with submodules
-- name: 'gcr.io/cloud-builders/git'
-  entrypoint: 'sh'
-  args: ['-c', 'git clone git@github.com:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .']
-  volumes:
-  - name: 'ssh'
-    path: /root/.ssh
-
-# Copy files to the right directories to run the tests
-- name: ubuntu
-  entrypoint: /bin/bash
-  args:
-    - -c
-    - |
-      cp private/script/ci/* .
-      cp private/Dockerfile .
-      cp script/ci/* .
-      cp config/app_config.yml.sample config/app_config.yml
-      cp config/database.yml.sample config/database.yml
-      cp lib/assets/javascripts/cdb/secrets.example.json lib/assets/javascripts/cdb/secrets.json
-
-- name: gcr.io/cloud-builders/docker
-  entrypoint: /bin/bash
-  args:
-    - -cx
-    - |
-      docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}
-      if [ $? -ne 0 ]
-      then
-          docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest
-          docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder:latest gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}
-      fi
-      docker build --build-arg COMPILE_ASSETS=false --build-arg BUNDLE_JOBS=16 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:current -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} .
-
-# Run tests
-- name: 'docker/compose:1.22.0'
-  args: ['-f', 'docker-compose-pg11.yml', 'up', '--build', '-d']
-  timeout: 900s
-
-- name: gcr.io/cloud-builders/docker
-  args: ['exec', '-i', 'builder_1', 'bash', '-c', '/cartodb/runParallelTests.sh 24' ]
-  timeout: 2700s
-
-substitutions: 
-    _BRANCH_TAG: ${BRANCH_NAME//\//-}
-options:
-    machineType: 'N1_HIGHCPU_32'
-timeout: 3600s

script/ci/cloudbuild-tests-pr-pg12.yaml

@@ -55,6 +55,14 @@ steps:
   - name: 'ssh'
     path: /root/.ssh
 
+# Checkout db-connectors
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'sh'
+  args: ['-c', 'git clone git@github.com:CartoDB/db-connectors --depth 1 private_gears/db-connectors']
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
 # Copy files to the right directories to run the tests
 - name: ubuntu
   entrypoint: /bin/bash

services/importer/spec/unit/connection_manager_spec.rb

@@ -3,6 +3,7 @@ require_relative '../doubles/connector'
 
 describe Carto::ConnectionManager do
 
+  include_context 'with MessageBroker stubs'
   let(:user) { create(:carto_user_light) }
   let(:connection_manager) { Carto::ConnectionManager.new(user) }
   let(:other_user) { create(:carto_user_light) }

spec/services/carto/user_metadata_export_service_spec.rb

@@ -11,7 +11,8 @@ describe Carto::UserMetadataExportService do
   include AccountTypesHelper
   include Carto::Factories::Visualizations
 
-  before(:all) do
+  before do
+    Cartodb::Central.any_instance.stubs(:update_user).returns(true)
     bypass_named_maps
     @feature_flag = create(:feature_flag)
     @connector_provider = FactoryGirl.create(:connector_provider)
@@ -19,13 +20,16 @@ describe Carto::UserMetadataExportService do
     @oauth_app = FactoryGirl.create(:oauth_app, user: user)
   end
 
-  after(:all) do
-    @feature_flag.destroy
-    @connector_provider.destroy
-  end
-
-  before(:each) do
-    Cartodb::Central.any_instance.stubs(:update_user).returns(true)
+  after do
+    Carto::FeatureFlagsUser.delete_all
+    Carto::FeatureFlag.delete_all
+    Carto::OauthToken.delete_all
+    Carto::OauthApp.delete_all
+    Carto::User.delete_all
+    Carto::SearchTweet.delete_all
+    Carto::AccountType.delete_all
+    Carto::RateLimit.delete_all
+    Carto::ClientApplication.delete_all
   end
 
   def create_user_with_basemaps_assets_visualizations
@@ -141,15 +145,11 @@ describe Carto::UserMetadataExportService do
   let(:service) { Carto::UserMetadataExportService.new }
 
   describe '#export' do
-    before(:all) do
+    before do
       Cartodb::Central.any_instance.stubs(:update_user).returns(true)
       create_user_with_basemaps_assets_visualizations
     end
 
-    after(:all) do
-      destroy_user
-    end
-
     it 'exports' do
       export = service.export_user_json_hash(@user)
 
@@ -167,26 +167,6 @@ describe Carto::UserMetadataExportService do
   end
 
   describe '#user import' do
-    after :each do
-      if @search_tweets
-        @search_tweets.each do |st|
-          st.data_import.try(:destroy)
-          st.destroy
-        end
-      end
-
-      @user.oauth_app_users.each do |oau|
-        unless oau.oauth_access_tokens.blank?
-          oau.oauth_access_tokens.each do |oat|
-            oat.api_key.skip_role_setup = true
-            oat.api_key.skip_cdb_conf_info = true
-          end
-        end
-      end
-
-      @user.destroy if @user
-    end
-
     def test_import_user_from_export(export)
       @user = service.build_user_from_hash_export(export)
       create_account_type_fg('FREE')
@@ -316,10 +296,6 @@ describe Carto::UserMetadataExportService do
   end
 
   describe '#user export + import' do
-    after :each do
-      destroy_user
-    end
-
     it 'export + import latest' do
       create_user_with_basemaps_assets_visualizations
       export_import(@user)
@@ -327,10 +303,6 @@ describe Carto::UserMetadataExportService do
   end
 
   describe '#full export + import (user and visualizations)' do
-    after :each do
-      destroy_user(@imported_user)
-    end
-
     it 'export + import user with rate limit and visualizations' do
       Dir.mktmpdir do |path|
         create_user_with_basemaps_assets_visualizations