diff --git a/script/ci/cloudbuild-build-master.yaml b/script/ci/cloudbuild-build-master.yaml index 1046796d9e..ef95183589 100644 --- a/script/ci/cloudbuild-build-master.yaml +++ b/script/ci/cloudbuild-build-master.yaml @@ -50,7 +50,7 @@ steps: # Checkout repo with submodules - name: 'gcr.io/cloud-builders/git' entrypoint: 'sh' - args: ['-c', 'git clone git@github.com:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .'] + args: ['-c', 'git clone git@github.com:CartoDB/${_REPO} -b "${BRANCH_NAME}" --depth 1 --recursive .'] volumes: - name: 'ssh' path: /root/.ssh @@ -81,16 +81,15 @@ steps: args: - -cx - | - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} if [ $? -ne 0 ] then - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder:latest gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:latest fi - - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --build-arg COMPILE_ASSETS=true --build-arg BUNDLE_JOBS=4 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} . - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.resque . - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.subscriber . + set -e + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --build-arg COMPILE_ASSETS=true --build-arg BUNDLE_JOBS=4 -t ${_DOCKER_IMAGE_NAME}:latest -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} --cache-from ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} --cache-from ${_DOCKER_IMAGE_NAME}:latest . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-resque:latest -t ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-resque:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.resque . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-subscriber:latest -t ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-subscriber:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.subscriber . # Checkout onprem licensing gear - name: 'gcr.io/cloud-builders/git' @@ -106,7 +105,7 @@ steps: args: - -cx - | - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --label=com.carto.onprem.revision=$(git -C private_gears/onpremises_licensing_gear/ rev-parse HEAD) -f private/Dockerfile.onprem --build-arg BUNDLE_JOBS=4 -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG}--${SHORT_SHA} . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --label=com.carto.onprem.revision=$(git -C private_gears/onpremises_licensing_gear/ rev-parse HEAD) -f private/Dockerfile.onprem --build-arg BUNDLE_JOBS=4 -t ${_DOCKER_IMAGE_NAME}-onprem:latest -t ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-onprem:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG}--${SHORT_SHA} . # Push image - name: gcr.io/cloud-builders/docker @@ -115,46 +114,51 @@ steps: - -cx - | echo 'Pushing builder image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} + docker push ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} echo 'Pushing builder-onprem image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG}--${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG} + docker push ${_DOCKER_IMAGE_NAME}-onprem:${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG}--${SHORT_SHA} echo 'Pushing builder-resque image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG}--${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG} + docker push ${_DOCKER_IMAGE_NAME}-resque:${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG}--${SHORT_SHA} echo 'Pushing builder-subscriber image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG}--${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG} + docker push ${_DOCKER_IMAGE_NAME}-subscriber:${SHORT_SHA} + docker push ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG}--${SHORT_SHA} if [ "${_BRANCH_TAG}" == 'master' ] then - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:latest - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:latest - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:latest - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:latest + docker push ${_DOCKER_IMAGE_NAME}:latest + docker push ${_DOCKER_IMAGE_NAME}-onprem:latest + docker push ${_DOCKER_IMAGE_NAME}-resque:latest + docker push ${_DOCKER_IMAGE_NAME}-subscriber:latest fi if [ ! -z "${TAG_NAME}" ] then echo "Tagging image with git tag: $TAG_NAME ..." - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} gcr.io/cartodb-on-gcp-main-artifacts/builder:${TAG_NAME} - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${SHORT_SHA} gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${TAG_NAME} - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA} gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${TAG_NAME} - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA} gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${TAG_NAME} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${TAG_NAME} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${TAG_NAME} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${TAG_NAME} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${TAG_NAME} + docker tag ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} ${_DOCKER_IMAGE_NAME}:${TAG_NAME} + docker tag ${_DOCKER_IMAGE_NAME}-onprem:${SHORT_SHA} ${_DOCKER_IMAGE_NAME}-onprem:${TAG_NAME} + docker tag ${_DOCKER_IMAGE_NAME}-resque:${SHORT_SHA} ${_DOCKER_IMAGE_NAME}-resque:${TAG_NAME} + docker tag ${_DOCKER_IMAGE_NAME}-subscriber:${SHORT_SHA} ${_DOCKER_IMAGE_NAME}-subscriber:${TAG_NAME} + docker push ${_DOCKER_IMAGE_NAME}:${TAG_NAME} + docker push ${_DOCKER_IMAGE_NAME}-onprem:${TAG_NAME} + docker push ${_DOCKER_IMAGE_NAME}-resque:${TAG_NAME} + docker push ${_DOCKER_IMAGE_NAME}-subscriber:${TAG_NAME} fi - -substitutions: - _BRANCH_TAG: ${BRANCH_NAME//\//-} timeout: 1800s +substitutions: + _BRANCH_TAG: ${BRANCH_NAME//\//-} + _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder + _REPO: cartodb +options: + machineType: 'E2_HIGHCPU_32' + + diff --git a/script/ci/cloudbuild-build-pr-branch.yaml b/script/ci/cloudbuild-build-pr-branch.yaml index eda28496b0..3cf28833c2 100644 --- a/script/ci/cloudbuild-build-pr-branch.yaml +++ b/script/ci/cloudbuild-build-pr-branch.yaml @@ -50,7 +50,7 @@ steps: # Checkout repo with submodules - name: 'gcr.io/cloud-builders/git' entrypoint: 'sh' - args: ['-c', 'git clone git@github.com:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .'] + args: ['-c', 'git clone git@github.com:CartoDB/${_REPO} -b "${BRANCH_NAME}" --depth 1 --recursive .'] volumes: - name: 'ssh' path: /root/.ssh @@ -81,16 +81,15 @@ steps: args: - -cx - | - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} if [ $? -ne 0 ] then - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder:latest gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:latest fi - - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --build-arg COMPILE_ASSETS=true --build-arg BUNDLE_JOBS=4 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} . - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.resque . - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.subscriber . + set -e + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --build-arg COMPILE_ASSETS=true --build-arg BUNDLE_JOBS=4 -t ${_DOCKER_IMAGE_NAME}:latest -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} --cache-from ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} --cache-from ${_DOCKER_IMAGE_NAME}:latest . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-resque:latest -t ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-resque:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.resque . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-subscriber:latest -t ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-subscriber:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.subscriber . # Checkout onprem licensing gear - name: 'gcr.io/cloud-builders/git' @@ -106,36 +105,26 @@ steps: args: - -cx - | - docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --label=com.carto.onprem.revision=$(git -C private_gears/onpremises_licensing_gear/ rev-parse HEAD) -f private/Dockerfile.onprem --build-arg BUNDLE_JOBS=4 -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG}--${SHORT_SHA} . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --label=com.carto.onprem.revision=$(git -C private_gears/onpremises_licensing_gear/ rev-parse HEAD) -f private/Dockerfile.onprem --build-arg BUNDLE_JOBS=4 -t ${_DOCKER_IMAGE_NAME}-onprem:latest -t ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-onprem:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG}--${SHORT_SHA} . -# Push image -- name: gcr.io/cloud-builders/docker - entrypoint: /bin/bash - args: - - -cx - - | - echo 'Pushing builder image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} - - echo 'Pushing builder-onprem image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-onprem:${_BRANCH_TAG}--${SHORT_SHA} - - echo 'Pushing builder-resque image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${_BRANCH_TAG}--${SHORT_SHA} - - echo 'Pushing builder-subscriber image...' - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA} - docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${_BRANCH_TAG}--${SHORT_SHA} - -substitutions: - _BRANCH_TAG: ${BRANCH_NAME//\//-} -options: - machineType: 'E2_HIGHCPU_8' timeout: 1800s +images: + - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} + - ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG}--${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG} + - ${_DOCKER_IMAGE_NAME}-onprem:${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG}--${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG} + - ${_DOCKER_IMAGE_NAME}-resque:${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG}--${SHORT_SHA} + - ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG} + - ${_DOCKER_IMAGE_NAME}-subscriber:${SHORT_SHA} +substitutions: + _BRANCH_TAG: ${BRANCH_NAME//\//-} + _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder + _REPO: cartodb +options: + machineType: 'E2_HIGHCPU_32' + diff --git a/script/ci/cloudbuild-onprem-dev-tests.yaml b/script/ci/cloudbuild-onprem-dev-tests.yaml new file mode 100644 index 0000000000..ae0146ee37 --- /dev/null +++ b/script/ci/cloudbuild-onprem-dev-tests.yaml @@ -0,0 +1,311 @@ +steps: + +# Print logs url +- name: ubuntu + entrypoint: /bin/bash + args: + - -c + - | + echo "[LOGS] Tests & compose logs will be stored at https://console.cloud.google.com/storage/browser/${_REPO}-ci-tmp-logs/test_logs_${BUILD_ID}" + +# Cancel previous job on the same branch +- name: gcr.io/cloud-builders/gcloud + entrypoint: /bin/bash + args: + - '-c' + - 'gcloud builds list --ongoing --filter="buildTriggerId=3ab7bd24-60d1-4cf3-a30e-86bae7bf69d0 AND substitutions.BRANCH_NAME=${BRANCH_NAME} AND id!=${BUILD_ID}" --format="get(ID)" > jobs_to_cancel' + +- name: gcr.io/cloud-builders/gcloud + entrypoint: /bin/bash + args: + - '-c' + - 'gcloud builds cancel $(cat jobs_to_cancel | xargs) || true' + +# Decrypt github key +- name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=script/ci/.id_github_cartofante.enc + - --plaintext-file=/root/.ssh/id_rsa + - --location=global + - --keyring=keyring-github-ci + - --key=github-key + volumes: + - name: 'ssh' + path: /root/.ssh + +# Set up git with key and domain +- name: 'gcr.io/cloud-builders/git' + entrypoint: 'bash' + args: + - '-c' + - | + chmod 600 /root/.ssh/id_rsa + cat </root/.ssh/config + Hostname github.com + IdentityFile /root/.ssh/id_rsa + EOF + mv script/ci/known_hosts /root/.ssh/known_hosts + volumes: + - name: 'ssh' + path: /root/.ssh + +# Clean working directory +- name: alpine + args: ['sh', '-c', 'rm -rf .* | true && rm -rf *'] + +# Checkout repo with submodules +- name: 'gcr.io/cloud-builders/git' + entrypoint: 'sh' + args: ['-c', 'git clone git@github.com:CartoDB/${_REPO} -b "${BRANCH_NAME}" --depth 1 --recursive cartodb'] + volumes: + - name: 'ssh' + path: /root/.ssh + +# Checkout db-connectors +- name: 'gcr.io/cloud-builders/git' + entrypoint: 'sh' + dir: cartodb + args: ['-c', 'git clone git@github.com:CartoDB/db-connectors --depth 1 private_gears/db-connectors'] + volumes: + - name: 'ssh' + path: /root/.ssh + +# Checkout repo with submodules +- name: 'gcr.io/cloud-builders/git' + entrypoint: 'sh' + args: ['-c', 'git clone git@github.com:CartoDB/docker-dev-env --depth 1'] + volumes: + - name: 'ssh' + path: /root/.ssh + +# Checkout repo with submodules +- name: 'gcr.io/cloud-builders/git' + entrypoint: 'sh' + args: ['-c', 'git clone git@github.com:CartoDB/dockerize-onpremises --depth 1'] + volumes: + - name: 'ssh' + path: /root/.ssh + +# Prepare dev-env files +- name: ubuntu + entrypoint: /bin/bash + dir: /workspace/docker-dev-env + args: + - -cx + - | + cp .env.sample .env + echo 'CARTO_BUILDER_SERVER=rails' >> .env + echo 'CARTO_DOMAIN=localhost.lan' >> .env + sed -i 's@'"$_DOCKER_IMAGE_NAME":latest'@'"$_DOCKER_IMAGE_NAME":"$SHORT_SHA"'@' .env + cat .env | grep 'IMAGE' + +# Pull images in background +- name: 'docker/compose:1.28.0' + entrypoint: /bin/sh + dir: /workspace/docker-dev-env + args: + - -cx + - | + docker-compose pull & + +# Copy build files +- name: ubuntu + entrypoint: /bin/bash + dir: cartodb + args: + - -c + - | + cp private/Dockerfil* . + cp config/unicorn.conf.rb.sample config/unicorn.conf.rb + cp config/app_config.yml.sample config/app_config.yml + cp config/database.yml.sample config/database.yml + cp lib/assets/javascripts/cdb/secrets.example.json lib/assets/javascripts/cdb/secrets.json + +# Build image +- name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + dir: cartodb + args: + - -cx + - | + docker pull ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} + if [ $? -ne 0 ] + then + docker pull ${_DOCKER_IMAGE_NAME}:latest + fi + set -e + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --build-arg COMPILE_ASSETS=true --build-arg BUNDLE_JOBS=4 -t ${_DOCKER_IMAGE_NAME}:latest -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} --cache-from ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} --cache-from ${_DOCKER_IMAGE_NAME}:latest . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-resque:latest -t ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-resque:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-resque:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.resque . + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-subscriber:latest -t ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-subscriber:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-subscriber:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.subscriber . + +# Checkout onprem licensing gear +- name: 'gcr.io/cloud-builders/git' + entrypoint: 'sh' + dir: cartodb + args: ['-c', 'git clone git@github.com:CartoDB/onpremises_licensing_gear --depth 1 private_gears/onpremises_licensing_gear'] + volumes: + - name: 'ssh' + path: /root/.ssh + +# Build onprem image +- name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + dir: cartodb + args: + - -cx + - | + docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} --label=com.carto.onprem.revision=$(git -C private_gears/onpremises_licensing_gear/ rev-parse HEAD) -f private/Dockerfile.onprem --build-arg BUNDLE_JOBS=4 -t ${_DOCKER_IMAGE_NAME}-onprem:latest -t ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-onprem:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem:${_BRANCH_TAG}--${SHORT_SHA} . + +## docker-dev-env tests ## + +# Init dev-env +- name: 'docker/compose:1.28.0' + entrypoint: /bin/sh + dir: /workspace/docker-dev-env + args: ['scripts/init.sh'] + +# Run dev tests +- name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + dir: /workspace/docker-dev-env + args: + - -cx + - | + source .env + docker run --rm --name tavern-tester-dev \ + --env "CARTO_USER=$${USER_USERNAME}" \ + --env "CARTO_DOMAIN=$${CARTO_DOMAIN}" \ + --network host \ + --env-file .env \ + gcr.io/cartodb-on-gcp-main-artifacts/tavern-tester:latest \ + all dev > dev_output 2>&1 + echo $? >> dev_result + cat dev_output + +# Copy dev compose logs +- name: 'docker/compose:1.28.2' + entrypoint: /bin/sh + dir: /workspace/docker-dev-env + args: + - -c + - | + docker-compose logs --no-color > dev_docker_compose_logs + echo "Logs will be available during 14 days at gs://${_REPO}-ci-tmp-logs/${BUILD_ID}/" + +# Upload dev logs to gcs +- name: gcr.io/cloud-builders/gsutil + entrypoint: /bin/sh + dir: /workspace/docker-dev-env + args: + - -c + - | + gsutil -m cp dev_output gs://${_REPO}-ci-tmp-logs/test_logs_${BUILD_ID}/ + gsutil -m cp dev_docker_compose_logs gs://${_REPO}-ci-tmp-logs/test_logs_${BUILD_ID}/ + +# Stop dev services and volumes +- name: 'docker/compose:1.28.0' + dir: /workspace/docker-dev-env + args: ['down', '-v'] + +## Onprem tests ## +# Change nginx host port to 8443: 443 is bussy inside Cloud Build host +- name: ubuntu + entrypoint: /bin/bash + dir: /workspace/dockerize-onpremises + args: + - -cx + - | + sed -i 's@'"$_DOCKER_IMAGE_NAME":latest'@'"$_DOCKER_IMAGE_NAME":"$SHORT_SHA"'@' .env + cat .env | grep 'IMAGE' + sed -i 's/443:443/8443:443/g' docker-compose.yml + +# Init onprem +- name: 'docker/compose:1.28.2' + entrypoint: /bin/sh + dir: /workspace/dockerize-onpremises + args: ['ci/init.sh'] + +# Install license +- name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + dir: /workspace/dockerize-onpremises + args: + - -cex + - | + mkdir -p license && cp -f ci/test_license/dockerize-onpremises-tests-license.lic license/carto.lic + ./script/license.sh + +# Run onprem tests +- name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + dir: /workspace/dockerize-onpremises + args: + - -cx + - | + source .env + source ci/.env.ci + docker run --rm --name tavern-tester-onprem \ + --env "CARTO_USER=$${USER_USERNAME}" \ + --env "CARTO_DOMAIN=$${CARTO_DOMAIN}" \ + --network "$${COMPOSE_PROJECT_NAME}_default" \ + --env-file .env \ + gcr.io/cartodb-on-gcp-main-artifacts/tavern-tester:latest \ + all onprem > onprem_output 2>&1 + echo $? >> onprem_result + cat onprem_output + +# Copy onprem compose logs +- name: 'docker/compose:1.28.2' + entrypoint: /bin/sh + dir: /workspace/dockerize-onpremises + args: + - -c + - | + docker-compose -f docker-compose.yml -f ci/docker-compose.ci.yml logs --no-color > onprem_docker_compose_logs + echo "Logs will be available during 14 days at gs://${_REPO}-ci-tmp-logs/test_logs_${BUILD_ID}/" + +# Upload onprem logs to gcs +- name: gcr.io/cloud-builders/gsutil + entrypoint: /bin/sh + dir: /workspace/dockerize-onpremises + args: + - -c + - | + gsutil -m cp onprem_output gs://${_REPO}-ci-tmp-logs/test_logs_${BUILD_ID}/ + gsutil -m cp onprem_docker_compose_logs gs://${_REPO}-ci-tmp-logs/test_logs_${BUILD_ID}/ + +# Handle exit code +- name: ubuntu + entrypoint: bash + dir: /workspace + args: + - -cx + - | + echo "[LOGS] Tests & compose logs are at https://console.cloud.google.com/storage/browser/cartodb-ci-tmp-logs/test_logs_${BUILD_ID}" + if [ "$(cat /workspace/docker-dev-env/dev_result)" -eq 0 ] && [ "$(cat /workspace/dockerize-onpremises/onprem_result)" -eq 0 ];then + echo '[PASSED] All tests passed' + exit 0 + elif [ "$(cat /workspace/docker-dev-env/dev_result)" -ne 0 ];then + echo '[FAILED] Development environment tests failed' + cat /workspace/docker-dev-env/dev_output + exit 1 + elif [ "$(cat /workspace/dockerize-onpremises/onprem_result)" -ne 0 ];then + echo '[FAILED] Onprem tests failed' + cat /workspace/dockerize-onpremises/onprem_output + exit 1 + else + echo '[FAILED] All tests failed' + cat /workspace/docker-dev-env/dev_output + cat /workspace/dockerize-onpremises/onprem_output + exit 1 + fi + +timeout: 1800s +options: + machineType: 'E2_HIGHCPU_32' +substitutions: + _BRANCH_TAG: ${BRANCH_NAME//\//-} + _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder + _REPO: cartodb diff --git a/script/ci/cloudbuild-rails-tests-pr.yml b/script/ci/cloudbuild-rails-tests-pr.yml index 39e98d9ace..ac808f38c8 100644 --- a/script/ci/cloudbuild-rails-tests-pr.yml +++ b/script/ci/cloudbuild-rails-tests-pr.yml @@ -50,7 +50,7 @@ steps: # Checkout repo with submodules - name: 'gcr.io/cloud-builders/git' entrypoint: 'sh' - args: ['-c', 'git clone git@github.com:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .'] + args: ['-c', 'git clone git@github.com:CartoDB/${_REPO} -b "${BRANCH_NAME}" --depth 1 --recursive .'] volumes: - name: 'ssh' path: /root/.ssh @@ -82,13 +82,13 @@ steps: args: - -cx - | - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} if [ $? -ne 0 ] then - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder:latest gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:latest fi - docker build --build-arg COMPILE_ASSETS=false --build-arg BUNDLE_JOBS=16 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:current -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} . + set -e + docker build --build-arg COMPILE_ASSETS=false --build-arg BUNDLE_JOBS=16 -t ${_DOCKER_IMAGE_NAME}:current -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} --cache-from ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} --cache-from ${_DOCKER_IMAGE_NAME}:latest . # Start necessary services (redis, postgres) in background - name: 'docker/compose:1.28.0' @@ -170,9 +170,11 @@ steps: echo "Detailed logs are at https://console.cloud.google.com/storage/browser/cartodb-ci-tmp-logs/parallel_test_logs_${BUILD_ID}" exit $(cat tests_exit_code) - +timeout: 2400s substitutions: - _BRANCH_TAG: ${BRANCH_NAME//\//-} + _BRANCH_TAG: ${BRANCH_NAME//\//-} + _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder + _REPO: cartodb options: - machineType: N1_HIGHCPU_8 -timeout: 2400s + machineType: 'E2_HIGHCPU_32' + diff --git a/script/ci/cloudbuild-tests-pr-pg12.yaml b/script/ci/cloudbuild-tests-pr-pg12.yaml index a8e8ac317f..b4dbbd67b1 100644 --- a/script/ci/cloudbuild-tests-pr-pg12.yaml +++ b/script/ci/cloudbuild-tests-pr-pg12.yaml @@ -50,7 +50,7 @@ steps: # Checkout repo with submodules - name: 'gcr.io/cloud-builders/git' entrypoint: 'sh' - args: ['-c', 'git clone git@github.com:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .'] + args: ['-c', 'git clone git@github.com:CartoDB/${_REPO} -b "${BRANCH_NAME}" --depth 1 --recursive .'] volumes: - name: 'ssh' path: /root/.ssh @@ -82,13 +82,13 @@ steps: args: - -cx - | - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} if [ $? -ne 0 ] then - docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest - docker tag gcr.io/cartodb-on-gcp-main-artifacts/builder:latest gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} + docker pull ${_DOCKER_IMAGE_NAME}:latest fi - docker build --build-arg COMPILE_ASSETS=false --build-arg BUNDLE_JOBS=16 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:current -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG}--${SHORT_SHA} --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:${_BRANCH_TAG} . + set -e + docker build --build-arg COMPILE_ASSETS=false --build-arg BUNDLE_JOBS=16 -t ${_DOCKER_IMAGE_NAME}:current -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} --cache-from ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} --cache-from ${_DOCKER_IMAGE_NAME}:latest . # Start necessary services (redis, postgres) in background - name: 'docker/compose:1.22.0' @@ -132,9 +132,10 @@ steps: exit 1 fi - +timeout: 2400s substitutions: - _BRANCH_TAG: ${BRANCH_NAME//\//-} + _BRANCH_TAG: ${BRANCH_NAME//\//-} + _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder + _REPO: cartodb options: - machineType: 'N1_HIGHCPU_32' -timeout: 2400s + machineType: 'N1_HIGHCPU_32'