From 51498e12388d2b29fa6d25465ed0175bf02e7dad Mon Sep 17 00:00:00 2001 From: Clement Tsang <34804052+ClementTsang@users.noreply.github.com> Date: Thu, 27 Oct 2022 06:27:04 -0400 Subject: [PATCH] ci: automatically create and get build artifacts from Cirrus CI (#854) This automatically triggers and grabs the build artifacts for systems that are only supported on Cirrus CI (as of now, FreeBSD and M1 macOS). * ci: add cirrus build trigger script * ci: modify build scripts to include cirrus build * fix some stuff * update docs * more fixes --- .cirrus.yml | 8 +- .github/workflows/build_releases.yml | 32 +++++ .github/workflows/deployment.yml | 3 + .github/workflows/nightly.yml | 3 + .markdownlint.json | 1 + README.md | 3 +- deployment/cirrus/build.py | 180 +++++++++++++++++++++++++++ 7 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 deployment/cirrus/build.py diff --git a/.cirrus.yml b/.cirrus.yml index 6ced047f..e0a9de61 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,3 +1,5 @@ +%YAML 1.1 +--- # Configuration for CirrusCI. This is primarily used for # FreeBSD and macOS M1 tests and builds. @@ -35,7 +37,8 @@ env: CARGO_HUSKY_DONT_INSTALL_HOOKS: true test_task: - only_if: $CIRRUS_CRON == "" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "") + auto_cancellation: false + only_if: $CIRRUS_BUILD_SOURCE != "api" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "") matrix: - name: "FreeBSD 13 Test" freebsd_instance: @@ -60,7 +63,8 @@ test_task: <<: *CLEANUP_TEMPLATE build_task: - only_if: $CIRRUS_RELEASE != "" || $CIRRUS_CRON == "nightly" || $CIRRUS_API_CREATED == true || $CIRRUS_BRANCH == "master" + auto_cancellation: false + only_if: $CIRRUS_BUILD_SOURCE == "api" env: BTM_GENERATE: true COMPLETION_DIR: "target/tmp/bottom/completion/" diff --git a/.github/workflows/build_releases.yml b/.github/workflows/build_releases.yml index 2bd3abbd..de3da73f 100644 --- a/.github/workflows/build_releases.yml +++ b/.github/workflows/build_releases.yml @@ -8,6 +8,12 @@ name: "Build Releases" on: workflow_dispatch: workflow_call: + inputs: + caller: + description: "The calling workflow." + default: "" + required: false + type: string env: CARGO_INCREMENTAL: 0 @@ -249,6 +255,32 @@ jobs: name: release path: release + build-cirrus: + name: "Build using Cirrus CI" + runs-on: "ubuntu-latest" + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Create release directory + run: | + mkdir -p release + + - name: Execute Cirrus CI build script + env: + CIRRUS_KEY: ${{ secrets.CIRRUS_TOKEN }} + run: | + python ./deployment/cirrus/build.py "${{ github.ref_name }}" "release/" "${{ inputs.caller }}" + + - name: Save release as artifact + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + with: + retention-days: 3 + name: release + path: release + build-deb: name: "Build Debian installers" runs-on: "ubuntu-20.04" diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 814c8ec0..53d94ff9 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -50,6 +50,9 @@ jobs: build-release: needs: [initialize-release-job] uses: ./.github/workflows/build_releases.yml + with: + caller: "deployment" + secrets: inherit generate-choco: needs: [build-release] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d4defa60..0384beba 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,6 +36,9 @@ jobs: build-release: needs: [initialize-job] uses: ./.github/workflows/build_releases.yml + with: + caller: "nightly" + secrets: inherit upload-release: name: upload-release diff --git a/.markdownlint.json b/.markdownlint.json index af02c0b7..00c8abaf 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -1,5 +1,6 @@ { "MD013": false, + "MD041": false, "MD033": false, "MD040": false, "MD024": false, diff --git a/README.md b/README.md index bb89fd26..cd318eb8 100644 --- a/README.md +++ b/README.md @@ -284,8 +284,7 @@ You can also try to use the generated release binaries and manually install on y - [Latest stable release](https://github.com/ClementTsang/bottom/releases/latest), generated off of the release branch - [Latest nightly release](https://github.com/ClementTsang/bottom/releases/tag/nightly), generated daily off of the master branch at 00:00 UTC - - FreeBSD builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/freebsd_build/binaries.zip) - - macOS ARM builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/macos_build/binaries.zip) + - Note that for now, FreeBSD and ARM macOS builds are primarily only available on the nightly release. #### Auto-completion diff --git a/deployment/cirrus/build.py b/deployment/cirrus/build.py new file mode 100644 index 00000000..6cb113af --- /dev/null +++ b/deployment/cirrus/build.py @@ -0,0 +1,180 @@ +#!/bin/python3 + +# A simple script to trigger Cirrus CI builds and get the release artifacts through Cirrus CI's GraphQL interface. +# Expects the API key to be set in CIRRUS_KEY. + +import os +import json +import sys +from textwrap import dedent +from time import sleep, time +from pathlib import Path +from typing import Optional + +from urllib.request import Request, urlopen, urlretrieve + +URL = "https://api.cirrus-ci.com/graphql" +TASKS = [ + ("freebsd_build", "bottom_x86_64-unknown-freebsd.tar.gz"), + ("macos_build", "bottom_aarch64-apple-darwin.tar.gz"), +] +DL_URL_TEMPLATE = "https://api.cirrus-ci.com/v1/artifact/build/%s/%s/binaries/%s" + + +def make_query_request(key: str, branch: str, build_type: str): + print("Creating query request.") + mutation_id = "Cirrus CI Build {}-{}-{}".format(build_type, branch, int(time())) + query = """ + mutation CreateCirrusCIBuild ( + $repo: ID!, + $branch: String!, + $mutation_id: String! + ) { + createBuild( + input: { + repositoryId: $repo, + branch: $branch, + clientMutationId: $mutation_id, + } + ) { + build { + id, + status + } + } + } + """ + params = { + "repo": "6646638922956800", + "branch": branch, + "mutation_id": mutation_id, + } + data = {"query": dedent(query), "variables": params} + data = json.dumps(data).encode() + + request = Request(URL, data=data, method="POST") + request.add_header("Authorization", "Bearer {}".format(key)) + + return request + + +def check_build_status(key: str, id: str) -> Optional[str]: + query = """ + query BuildStatus($id: ID!) { + build(id: $id) { + status + } + } + """ + params = { + "id": id, + } + + data = {"query": dedent(query), "variables": params} + data = json.dumps(data).encode() + + request = Request(URL, data=data, method="POST") + request.add_header("Authorization", "Bearer {}".format(key)) + with urlopen(request) as response: + response = json.load(response) + if response.get("errors") is not None: + print("There was an error in the returned response.") + return None + + try: + status = response["data"]["build"]["status"] + return status + except KeyError: + print("There was an issue with creating a build job.") + return None + + +def try_download(build_id: str, dl_path: Path): + for (task, file) in TASKS: + url = DL_URL_TEMPLATE % (build_id, task, file) + out = dl_path / file + print("Downloading {} to {}".format(file, out)) + urlretrieve(url, out) + + +def main(): + args = sys.argv + env = os.environ + + key = env["CIRRUS_KEY"] + branch = args[1] + dl_path = args[2] if len(args) >= 3 else "" + dl_path = Path(dl_path) + build_type = args[3] if len(args) >= 4 else "build" + build_id = args[4] if len(args) >= 5 else None + + # Check if this build has already been completed before. + if build_id is not None: + print("Previous build ID was provided, checking if complete.") + status = check_build_status(key, build_id) + if status.startswith("COMPLETE"): + print("Starting download of previous build ID") + try_download(build_id, dl_path) + else: + # Try up to three times + MAX_ATTEMPTS = 3 + success = False + + for i in range(MAX_ATTEMPTS): + if success: + break + print("Attempt {}:".format(i + 1)) + + with urlopen(make_query_request(key, branch, build_type)) as response: + response = json.load(response) + + if response.get("errors") is not None: + print("There was an error in the returned response.") + continue + + try: + build_id = response["data"]["createBuild"]["build"]["id"] + print("Created build job {}.".format(build_id)) + except KeyError: + print("There was an issue with creating a build job.") + continue + + # First, sleep 4 minutes, as it's unlikely it'll finish before then. + SLEEP_MINUTES = 4 + print("Sleeping for {} minutes.".format(SLEEP_MINUTES)) + sleep(60 * SLEEP_MINUTES) + print("Mandatory nap over. Starting to check for completion.") + + MINUTES = 10 + SLEEP_SEC = 30 + TRIES = int(MINUTES * (60 / SLEEP_SEC)) # Works out to 20 tries. + + for attempt in range(TRIES): + print("Checking...") + status = check_build_status(key, build_id) + if status.startswith("COMPLETE"): + print("Build complete. Downloading artifact files.") + sleep(5) + try_download(build_id, dl_path) + success = True + break + else: + print("Build status: {}".format(status or "unknown")) + if status == "ABORTED": + print("Build aborted, bailing.") + break + elif status.lower().startswith("fail"): + print("Build failed, bailing.") + break + elif attempt + 1 < TRIES: + sleep(SLEEP_SEC) + else: + print("Build failed to complete after {} minutes, bailing.".format(MINUTES)) + continue + + if not success: + exit(2) + + +if __name__ == "__main__": + main()