ci: automatically create and get build artifacts from Cirrus CI (#854)
This automatically triggers and grabs the build artifacts for systems that are only supported on Cirrus CI (as of now, FreeBSD and M1 macOS). * ci: add cirrus build trigger script * ci: modify build scripts to include cirrus build * fix some stuff * update docs * more fixes
This commit is contained in:
Clement Tsang
GitHub
parent
5eba26f9e5
commit
51498e1238
|
|
@ -1,3 +1,5 @@
|
|||
%YAML 1.1
|
||||
---
|
||||
# Configuration for CirrusCI. This is primarily used for
|
||||
# FreeBSD and macOS M1 tests and builds.
|
||||
|
||||
|
|
@ -35,7 +37,8 @@ env:
|
|||
CARGO_HUSKY_DONT_INSTALL_HOOKS: true
|
||||
|
||||
test_task:
|
||||
only_if: $CIRRUS_CRON == "" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "")
|
||||
auto_cancellation: false
|
||||
only_if: $CIRRUS_BUILD_SOURCE != "api" && ($CIRRUS_BRANCH == "master" || $CIRRUS_PR != "")
|
||||
matrix:
|
||||
- name: "FreeBSD 13 Test"
|
||||
freebsd_instance:
|
||||
|
|
@ -60,7 +63,8 @@ test_task:
|
|||
<<: *CLEANUP_TEMPLATE
|
||||
|
||||
build_task:
|
||||
only_if: $CIRRUS_RELEASE != "" || $CIRRUS_CRON == "nightly" || $CIRRUS_API_CREATED == true || $CIRRUS_BRANCH == "master"
|
||||
auto_cancellation: false
|
||||
only_if: $CIRRUS_BUILD_SOURCE == "api"
|
||||
env:
|
||||
BTM_GENERATE: true
|
||||
COMPLETION_DIR: "target/tmp/bottom/completion/"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@ name: "Build Releases"
|
|||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
inputs:
|
||||
caller:
|
||||
description: "The calling workflow."
|
||||
default: ""
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
|
@ -249,6 +255,32 @@ jobs:
|
|||
name: release
|
||||
path: release
|
||||
|
||||
build-cirrus:
|
||||
name: "Build using Cirrus CI"
|
||||
runs-on: "ubuntu-latest"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Create release directory
|
||||
run: |
|
||||
mkdir -p release
|
||||
|
||||
- name: Execute Cirrus CI build script
|
||||
env:
|
||||
CIRRUS_KEY: ${{ secrets.CIRRUS_TOKEN }}
|
||||
run: |
|
||||
python ./deployment/cirrus/build.py "${{ github.ref_name }}" "release/" "${{ inputs.caller }}"
|
||||
|
||||
- name: Save release as artifact
|
||||
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
|
||||
with:
|
||||
retention-days: 3
|
||||
name: release
|
||||
path: release
|
||||
|
||||
build-deb:
|
||||
name: "Build Debian installers"
|
||||
runs-on: "ubuntu-20.04"
|
||||
|
|
|
|||
|
|
@ -50,6 +50,9 @@ jobs:
|
|||
build-release:
|
||||
needs: [initialize-release-job]
|
||||
uses: ./.github/workflows/build_releases.yml
|
||||
with:
|
||||
caller: "deployment"
|
||||
secrets: inherit
|
||||
|
||||
generate-choco:
|
||||
needs: [build-release]
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ jobs:
|
|||
build-release:
|
||||
needs: [initialize-job]
|
||||
uses: ./.github/workflows/build_releases.yml
|
||||
with:
|
||||
caller: "nightly"
|
||||
secrets: inherit
|
||||
|
||||
upload-release:
|
||||
name: upload-release
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
"MD013": false,
|
||||
"MD041": false,
|
||||
"MD033": false,
|
||||
"MD040": false,
|
||||
"MD024": false,
|
||||
|
|
|
|||
|
|
@ -284,8 +284,7 @@ You can also try to use the generated release binaries and manually install on y
|
|||
|
||||
- [Latest stable release](https://github.com/ClementTsang/bottom/releases/latest), generated off of the release branch
|
||||
- [Latest nightly release](https://github.com/ClementTsang/bottom/releases/tag/nightly), generated daily off of the master branch at 00:00 UTC
|
||||
- FreeBSD builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/freebsd_build/binaries.zip)
|
||||
- macOS ARM builds can be found [here](https://api.cirrus-ci.com/v1/artifact/github/ClementTsang/bottom/macos_build/binaries.zip)
|
||||
- Note that for now, FreeBSD and ARM macOS builds are primarily only available on the nightly release.
|
||||
|
||||
#### Auto-completion
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
#!/bin/python3
|
||||
|
||||
# A simple script to trigger Cirrus CI builds and get the release artifacts through Cirrus CI's GraphQL interface.
|
||||
# Expects the API key to be set in CIRRUS_KEY.
|
||||
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
from textwrap import dedent
|
||||
from time import sleep, time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from urllib.request import Request, urlopen, urlretrieve
|
||||
|
||||
URL = "https://api.cirrus-ci.com/graphql"
|
||||
TASKS = [
|
||||
("freebsd_build", "bottom_x86_64-unknown-freebsd.tar.gz"),
|
||||
("macos_build", "bottom_aarch64-apple-darwin.tar.gz"),
|
||||
]
|
||||
DL_URL_TEMPLATE = "https://api.cirrus-ci.com/v1/artifact/build/%s/%s/binaries/%s"
|
||||
|
||||
|
||||
def make_query_request(key: str, branch: str, build_type: str):
|
||||
print("Creating query request.")
|
||||
mutation_id = "Cirrus CI Build {}-{}-{}".format(build_type, branch, int(time()))
|
||||
query = """
|
||||
mutation CreateCirrusCIBuild (
|
||||
$repo: ID!,
|
||||
$branch: String!,
|
||||
$mutation_id: String!
|
||||
) {
|
||||
createBuild(
|
||||
input: {
|
||||
repositoryId: $repo,
|
||||
branch: $branch,
|
||||
clientMutationId: $mutation_id,
|
||||
}
|
||||
) {
|
||||
build {
|
||||
id,
|
||||
status
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
params = {
|
||||
"repo": "6646638922956800",
|
||||
"branch": branch,
|
||||
"mutation_id": mutation_id,
|
||||
}
|
||||
data = {"query": dedent(query), "variables": params}
|
||||
data = json.dumps(data).encode()
|
||||
|
||||
request = Request(URL, data=data, method="POST")
|
||||
request.add_header("Authorization", "Bearer {}".format(key))
|
||||
|
||||
return request
|
||||
|
||||
|
||||
def check_build_status(key: str, id: str) -> Optional[str]:
|
||||
query = """
|
||||
query BuildStatus($id: ID!) {
|
||||
build(id: $id) {
|
||||
status
|
||||
}
|
||||
}
|
||||
"""
|
||||
params = {
|
||||
"id": id,
|
||||
}
|
||||
|
||||
data = {"query": dedent(query), "variables": params}
|
||||
data = json.dumps(data).encode()
|
||||
|
||||
request = Request(URL, data=data, method="POST")
|
||||
request.add_header("Authorization", "Bearer {}".format(key))
|
||||
with urlopen(request) as response:
|
||||
response = json.load(response)
|
||||
if response.get("errors") is not None:
|
||||
print("There was an error in the returned response.")
|
||||
return None
|
||||
|
||||
try:
|
||||
status = response["data"]["build"]["status"]
|
||||
return status
|
||||
except KeyError:
|
||||
print("There was an issue with creating a build job.")
|
||||
return None
|
||||
|
||||
|
||||
def try_download(build_id: str, dl_path: Path):
|
||||
for (task, file) in TASKS:
|
||||
url = DL_URL_TEMPLATE % (build_id, task, file)
|
||||
out = dl_path / file
|
||||
print("Downloading {} to {}".format(file, out))
|
||||
urlretrieve(url, out)
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv
|
||||
env = os.environ
|
||||
|
||||
key = env["CIRRUS_KEY"]
|
||||
branch = args[1]
|
||||
dl_path = args[2] if len(args) >= 3 else ""
|
||||
dl_path = Path(dl_path)
|
||||
build_type = args[3] if len(args) >= 4 else "build"
|
||||
build_id = args[4] if len(args) >= 5 else None
|
||||
|
||||
# Check if this build has already been completed before.
|
||||
if build_id is not None:
|
||||
print("Previous build ID was provided, checking if complete.")
|
||||
status = check_build_status(key, build_id)
|
||||
if status.startswith("COMPLETE"):
|
||||
print("Starting download of previous build ID")
|
||||
try_download(build_id, dl_path)
|
||||
else:
|
||||
# Try up to three times
|
||||
MAX_ATTEMPTS = 3
|
||||
success = False
|
||||
|
||||
for i in range(MAX_ATTEMPTS):
|
||||
if success:
|
||||
break
|
||||
print("Attempt {}:".format(i + 1))
|
||||
|
||||
with urlopen(make_query_request(key, branch, build_type)) as response:
|
||||
response = json.load(response)
|
||||
|
||||
if response.get("errors") is not None:
|
||||
print("There was an error in the returned response.")
|
||||
continue
|
||||
|
||||
try:
|
||||
build_id = response["data"]["createBuild"]["build"]["id"]
|
||||
print("Created build job {}.".format(build_id))
|
||||
except KeyError:
|
||||
print("There was an issue with creating a build job.")
|
||||
continue
|
||||
|
||||
# First, sleep 4 minutes, as it's unlikely it'll finish before then.
|
||||
SLEEP_MINUTES = 4
|
||||
print("Sleeping for {} minutes.".format(SLEEP_MINUTES))
|
||||
sleep(60 * SLEEP_MINUTES)
|
||||
print("Mandatory nap over. Starting to check for completion.")
|
||||
|
||||
MINUTES = 10
|
||||
SLEEP_SEC = 30
|
||||
TRIES = int(MINUTES * (60 / SLEEP_SEC)) # Works out to 20 tries.
|
||||
|
||||
for attempt in range(TRIES):
|
||||
print("Checking...")
|
||||
status = check_build_status(key, build_id)
|
||||
if status.startswith("COMPLETE"):
|
||||
print("Build complete. Downloading artifact files.")
|
||||
sleep(5)
|
||||
try_download(build_id, dl_path)
|
||||
success = True
|
||||
break
|
||||
else:
|
||||
print("Build status: {}".format(status or "unknown"))
|
||||
if status == "ABORTED":
|
||||
print("Build aborted, bailing.")
|
||||
break
|
||||
elif status.lower().startswith("fail"):
|
||||
print("Build failed, bailing.")
|
||||
break
|
||||
elif attempt + 1 < TRIES:
|
||||
sleep(SLEEP_SEC)
|
||||
else:
|
||||
print("Build failed to complete after {} minutes, bailing.".format(MINUTES))
|
||||
continue
|
||||
|
||||
if not success:
|
||||
exit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue