element-android-Github/tools/release/download_buildkite_artifacts.py

#!/usr/bin/env python3
#
# Copyright 2020 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import hashlib
import json
import os
# Run `pip3 install requests` if not installed yet
import requests

# This script downloads artifacts from buildkite.
# Ref: https://buildkite.com/docs/apis/rest-api/artifacts#download-an-artifact

# Those two variables are specific to the Element Android project
ORG_SLUG = "matrix-dot-org"
PIPELINE_SLUG = "element-android"

### Arguments

parser = argparse.ArgumentParser(description='Download artifacts from Buildkite.')
parser.add_argument('-t',
                    '--token',
                    required=True,
                    help='The buildkite token.')
parser.add_argument('-b',
                    '--build',
                    type=int,
                    required=True,
                    help='the buildkite build number.')
parser.add_argument('-f',
                    '--filename',
                    help='the filename, to download only one artifact.')
parser.add_argument('-e',
                    '--expecting',
                    type=int,
                    help='the expected number of artifacts. If omitted, no check will be done.')
parser.add_argument('-i',
                    '--ignoreErrors',
                    help='Ignore errors that can be ignored. Build state and number of artifacts.',
                    action="store_true")
parser.add_argument('-d',
                    '--directory',
                    default="",
                    help='the target directory, where files will be downloaded. If not provided the build number will be used to create a directory.')
parser.add_argument('-v',
                    '--verbose',
                    help="increase output verbosity.",
                    action="store_true")
parser.add_argument('-s',
                    '--simulate',
                    help="simulate action, do not create folder or download any file.",
                    action="store_true")

args = parser.parse_args()

# parser has checked that the build was an int, convert to String for the rest of the script
build_str = str(args.build)

if args.verbose:
    print("Argument:")
    print(args)

headers = {'Authorization': "Bearer %s" % args.token}
base_url = "https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds/%s" % (ORG_SLUG, PIPELINE_SLUG, build_str)

### Fetch build state

buildkite_build_state_url = base_url

buildkite_url = "https://buildkite.com/%s/%s/builds/%s" % (ORG_SLUG, PIPELINE_SLUG, build_str)

print("Getting build state of project %s/%s build %s (%s)" % (ORG_SLUG, PIPELINE_SLUG, build_str, buildkite_url))

if args.verbose:
    print("Url: %s" % buildkite_build_state_url)

r0 = requests.get(buildkite_build_state_url, headers=headers)
data0 = json.loads(r0.content.decode())

if args.verbose:
    print("Json data:")
    print(data0)

print("   git branch         : %s" % data0.get('branch'))
print("   git commit         : \"%s\"" % data0.get('commit'))
print("   git commit message : \"%s\"" % data0.get('message'))
print("   build state        : %s" % data0.get('state'))

error = False

if data0.get('state') != 'passed':
    print("❌ Error, the build is in state '%s', and not 'passed'" % data0.get('state'))
    if args.ignoreErrors:
        error = True
    else:
        exit(1)

### Fetch artifacts list

buildkite_artifacts_url = base_url + "/artifacts"

print("Getting artifacts list of project %s/%s build %s" % (ORG_SLUG, PIPELINE_SLUG, build_str))

if args.verbose:
    print("Url: %s" % buildkite_artifacts_url)

r = requests.get(buildkite_artifacts_url, headers=headers)
data = json.loads(r.content.decode())

print("   %d artifact(s) found." % len(data))

if args.expecting is not None and args.expecting != len(data):
    print("❌ Error, expecting %d artifacts and found %d." % (args.expecting, len(data)))
    if args.ignoreErrors:
        error = True
    else:
        exit(1)

if args.verbose:
    print("Json data:")
    print(data)

if args.verbose:
    print("Create subfolder %s to download artifacts..." % build_str)

if args.directory == "":
    targetDir = build_str
else:
    targetDir = args.directory

if not args.simulate:
    os.makedirs(targetDir, exist_ok=True)

for elt in data:
    if args.verbose:
        print()
        print("Artifact info:")
        for key, value in elt.items():
            print("   %s: %s" % (key, str(value)))
    url = elt.get("download_url")
    filename = elt.get("filename")
    if args.filename is not None and args.filename != filename:
        continue
    target = targetDir + "/" + filename
    print("Downloading %s to '%s'..." % (filename, targetDir))
    if not args.simulate:
        # open file to write in binary mode
        with open(target, "wb") as file:
            # get request
            response = requests.get(url, headers=headers)
            # write to file
            file.write(response.content)
        print("Verifying checksum...")
        # open file to read in binary mode
        with open(target, "rb") as file:
            data = file.read()
            hash = hashlib.sha1(data).hexdigest()
        if elt.get("sha1sum") != hash:
            error = True
            print("❌ Checksum mismatch: expecting %s and get %s" % (elt.get("sha1sum"), hash))

if error:
    print("❌ Error(s) occurred, please check the log")
    exit(1)
else:
    print("Done!")