305 lines
9.4 KiB
Python
305 lines
9.4 KiB
Python
#!/bin/python3
|
|
|
|
# A simple script to trigger Cirrus CI builds and download the release artifacts
|
|
# through Cirrus CI's GraphQL interface.
|
|
#
|
|
# Expects the Cirrus CI API key to be set in the CIRRUS_KEY environment variable.
|
|
#
|
|
# TODO: Explain this in docs how the heck this works.
|
|
|
|
import os
|
|
import json
|
|
import sys
|
|
import traceback
|
|
from textwrap import dedent
|
|
from time import sleep, time
|
|
from pathlib import Path
|
|
from typing import List, Optional, Tuple
|
|
|
|
from urllib.request import Request, urlopen, urlretrieve
|
|
|
|
# Form of each task is (TASK_ALIAS, FILE_NAME).
|
|
TASKS: List[Tuple[str, str]] = [
|
|
("freebsd_13_3_build", "bottom_x86_64-unknown-freebsd-13-3.tar.gz"),
|
|
("freebsd_14_0_build", "bottom_x86_64-unknown-freebsd-14-0.tar.gz"),
|
|
("linux_2_17_build", "bottom_x86_64-unknown-linux-gnu-2-17.tar.gz"),
|
|
]
|
|
URL = "https://api.cirrus-ci.com/graphql"
|
|
DL_URL_TEMPLATE = "https://api.cirrus-ci.com/v1/artifact/build/%s/%s/binaries/%s"
|
|
|
|
|
|
def make_query_request(key: str, branch: str, mutation_id: str):
|
|
print("Creating query request.")
|
|
|
|
# Dumb but if it works...
|
|
config_override = (
|
|
Path(".cirrus.yml")
|
|
.read_text()
|
|
.replace("# -PLACEHOLDER FOR CI-", 'BTM_BUILD_RELEASE_CALLER: "ci"')
|
|
)
|
|
|
|
query = """
|
|
mutation CreateCirrusCIBuild (
|
|
$repo: ID!,
|
|
$branch: String!,
|
|
$mutation_id: String!,
|
|
$config_override: String,
|
|
) {
|
|
createBuild (
|
|
input: {
|
|
repositoryId: $repo,
|
|
branch: $branch,
|
|
clientMutationId: $mutation_id,
|
|
configOverride: $config_override
|
|
}
|
|
) {
|
|
build {
|
|
id,
|
|
status
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
params = {
|
|
"repo": "6646638922956800",
|
|
"branch": branch,
|
|
"mutation_id": mutation_id,
|
|
"config_override": dedent(config_override),
|
|
}
|
|
|
|
data = {"query": dedent(query), "variables": params}
|
|
data = json.dumps(data).encode()
|
|
|
|
request = Request(URL, data=data, method="POST")
|
|
request.add_header("Authorization", "Bearer {}".format(key))
|
|
|
|
return request
|
|
|
|
|
|
def check_build_status(key: str, build_id: str) -> Optional[str]:
|
|
query = """
|
|
query BuildStatus($id: ID!) {
|
|
build(id: $id) {
|
|
status
|
|
}
|
|
}
|
|
"""
|
|
|
|
params = {
|
|
"id": build_id,
|
|
}
|
|
|
|
data = {"query": dedent(query), "variables": params}
|
|
data = json.dumps(data).encode()
|
|
|
|
request = Request(URL, data=data, method="POST")
|
|
request.add_header("Authorization", "Bearer {}".format(key))
|
|
with urlopen(request) as response:
|
|
response = json.load(response)
|
|
if response.get("errors") is not None:
|
|
print("There was an error in the returned response.")
|
|
return None
|
|
|
|
try:
|
|
status = response["data"]["build"]["status"]
|
|
return status
|
|
except KeyError:
|
|
print("There was an issue with checking the build status.")
|
|
return None
|
|
|
|
|
|
def check_build_tasks(key: str, build_id: str) -> Optional[List[str]]:
|
|
query = """
|
|
query Build($id:ID!) {
|
|
build(id:$id){
|
|
tasks {
|
|
id
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
params = {
|
|
"id": build_id,
|
|
}
|
|
|
|
data = {"query": dedent(query), "variables": params}
|
|
data = json.dumps(data).encode()
|
|
|
|
request = Request(URL, data=data, method="POST")
|
|
request.add_header("Authorization", "Bearer {}".format(key))
|
|
with urlopen(request) as response:
|
|
response = json.load(response)
|
|
|
|
if response.get("errors") is not None:
|
|
print("There was an error in the returned response.")
|
|
return None
|
|
|
|
try:
|
|
tasks = [task["id"] for task in response["data"]["build"]["tasks"]]
|
|
return tasks
|
|
except KeyError:
|
|
print("There was an issue with getting the list of task ids.")
|
|
return None
|
|
|
|
|
|
def stop_build_tasks(key: str, task_ids: List[str], mutation_id: str) -> bool:
|
|
query = """
|
|
mutation StopCirrusCiTasks (
|
|
$task_ids: [ID!]!,
|
|
$mutation_id: String!,
|
|
) {
|
|
batchAbort (
|
|
input: {
|
|
taskIds: $task_ids,
|
|
clientMutationId: $mutation_id
|
|
}
|
|
) {
|
|
tasks {
|
|
id
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
params = {
|
|
"task_ids": task_ids,
|
|
"mutation_id": mutation_id,
|
|
}
|
|
|
|
data = {"query": dedent(query), "variables": params}
|
|
data = json.dumps(data).encode()
|
|
|
|
request = Request(URL, data=data, method="POST")
|
|
request.add_header("Authorization", "Bearer {}".format(key))
|
|
|
|
with urlopen(request) as response:
|
|
response = json.load(response)
|
|
return len(response["data"]["batchAbort"]["tasks"]) == len(task_ids)
|
|
|
|
|
|
def try_download(build_id: str, dl_path: Path):
|
|
for task, file in TASKS:
|
|
url = DL_URL_TEMPLATE % (build_id, task, file)
|
|
out = os.path.join(dl_path, file)
|
|
print(f"Downloading {file} to {out}")
|
|
urlretrieve(url, out)
|
|
|
|
|
|
def main():
|
|
args = sys.argv
|
|
env = os.environ
|
|
|
|
key = env["CIRRUS_KEY"]
|
|
branch = args[1]
|
|
dl_path = args[2] if len(args) >= 3 else ""
|
|
dl_path = Path(dl_path)
|
|
build_type = args[3] if len(args) >= 4 else "build"
|
|
build_id = args[4] if len(args) >= 5 else None
|
|
|
|
print(f"Running Cirrus script with branch '{branch}'")
|
|
|
|
# Check if this build has already been completed before.
|
|
if build_id is not None:
|
|
print("Previous build ID was provided, checking if complete.")
|
|
status = check_build_status(key, build_id)
|
|
if status.startswith("COMPLETE"):
|
|
print("Starting download of previous build ID")
|
|
try_download(build_id, dl_path)
|
|
else:
|
|
# Try up to three times
|
|
MAX_ATTEMPTS = 5
|
|
success = False
|
|
tasks = []
|
|
mutation_id = None
|
|
|
|
for i in range(MAX_ATTEMPTS):
|
|
if success:
|
|
break
|
|
|
|
print(f"Attempt {i + 1}:")
|
|
|
|
if tasks and mutation_id:
|
|
print("Killing previous tasks first...")
|
|
|
|
if stop_build_tasks(key, tasks, mutation_id):
|
|
print("All previous tasks successfully stopped.")
|
|
else:
|
|
print(
|
|
"Not all previous tasks stopped. This isn't a problem but it is a waste."
|
|
)
|
|
|
|
tasks = []
|
|
mutation_id = "Cirrus CI Build {}-{}-{}".format(
|
|
build_type, branch, int(time())
|
|
)
|
|
|
|
with urlopen(make_query_request(key, branch, mutation_id)) as response:
|
|
response = json.load(response)
|
|
errors = response.get("errors")
|
|
|
|
if errors is not None:
|
|
print(f"There was an error in the returned response: {str(errors)}")
|
|
continue
|
|
|
|
try:
|
|
build_id = response["data"]["createBuild"]["build"]["id"]
|
|
print(f"Created build job {build_id}.")
|
|
except KeyError:
|
|
print("There was an issue with creating a build job.")
|
|
continue
|
|
|
|
# First, sleep X minutes total, as it's unlikely it'll finish before then.
|
|
SLEEP_MINUTES = 4
|
|
print(f"Sleeping for {SLEEP_MINUTES} minutes.")
|
|
|
|
# Sleep and check for tasks out every 10 seconds
|
|
for _ in range(SLEEP_MINUTES * 6):
|
|
sleep(10)
|
|
if not tasks:
|
|
tasks = check_build_tasks(key, build_id)
|
|
|
|
MINUTES = 10
|
|
SLEEP_SEC = 30
|
|
TRIES = int(MINUTES * (60 / SLEEP_SEC)) # Works out to 20 tries.
|
|
|
|
print(f"Mandatory nap over. Checking for completion for {MINUTES} min.")
|
|
|
|
for attempt in range(TRIES):
|
|
print("Checking...")
|
|
try:
|
|
status = check_build_status(key, build_id)
|
|
if status.startswith("COMPLETE"):
|
|
print("Build complete. Downloading artifact files.")
|
|
sleep(5)
|
|
try_download(build_id, dl_path)
|
|
success = True
|
|
break
|
|
else:
|
|
print(f"Build status: {(status or 'unknown')}")
|
|
|
|
if status == "ABORTED":
|
|
print("Build aborted, bailing.")
|
|
break
|
|
elif status.lower().startswith("fail"):
|
|
print("Build failed, bailing.")
|
|
break
|
|
elif attempt + 1 < TRIES:
|
|
sleep(SLEEP_SEC)
|
|
except Exception as ex:
|
|
print("Unexpected error:")
|
|
print(ex)
|
|
print(traceback.format_exc())
|
|
# Sleep for a minute if something went wrong, just in case.
|
|
sleep(60)
|
|
else:
|
|
print(f"Build failed to complete after {MINUTES} minutes, bailing.")
|
|
|
|
if not success:
|
|
exit(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|