From b1c831c54ae6808a9df2dac7890f5d7dbd9f8b9d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Apr 2018 16:01:52 -0700 Subject: [PATCH] Inital pass on comprehensive automated release script Signed-off-by: Joffrey F --- script/release/release.md.tmpl | 34 +++++ script/release/release.py | 197 +++++++++++++++++++++++++++ script/release/release/__init__.py | 0 script/release/release/bintray.py | 40 ++++++ script/release/release/const.py | 9 ++ script/release/release/downloader.py | 72 ++++++++++ script/release/release/repository.py | 161 ++++++++++++++++++++++ script/release/release/utils.py | 63 +++++++++ 8 files changed, 576 insertions(+) create mode 100644 script/release/release.md.tmpl create mode 100755 script/release/release.py create mode 100644 script/release/release/__init__.py create mode 100644 script/release/release/bintray.py create mode 100644 script/release/release/const.py create mode 100644 script/release/release/downloader.py create mode 100644 script/release/release/repository.py create mode 100644 script/release/release/utils.py diff --git a/script/release/release.md.tmpl b/script/release/release.md.tmpl new file mode 100644 index 000000000..ee97ef104 --- /dev/null +++ b/script/release/release.md.tmpl @@ -0,0 +1,34 @@ +If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. + +Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. + +Alternatively, you can use the usual commands to install or upgrade Compose: + +``` +curl -L https://github.com/docker/compose/releases/download/{{version}}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose +``` + +See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. + +## Compose file format compatibility matrix + +| Compose file format | Docker Engine | +| --- | --- | +{% for engine, formats in compat_matrix.items() -%} +| {% for format in formats %}{{format}}{% if not loop.last %}, {% endif %}{% endfor %} | {{engine}}+ | +{% endfor -%} + +## Changes + +{{changelog}} + +Thanks to {% for name in contributors %}@{{name}}{% if not loop.last %}, {% endif %}{% endfor %} for contributing to this release! + +## Integrity check + +Binary name | SHA-256 sum +| --- | --- | +{% for filename, sha in integrity.items() -%} +| `{{filename}}` | `{{sha[1]}}` | +{% endfor -%} diff --git a/script/release/release.py b/script/release/release.py new file mode 100755 index 000000000..f23146288 --- /dev/null +++ b/script/release/release.py @@ -0,0 +1,197 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import os +import sys +import time + +from jinja2 import Template +from release.bintray import BintrayAPI +from release.const import BINTRAY_ORG +from release.const import NAME +from release.const import REPO_ROOT +from release.downloader import BinaryDownloader +from release.repository import get_contributors +from release.repository import Repository +from release.repository import upload_assets +from release.utils import branch_name +from release.utils import compatibility_matrix +from release.utils import read_release_notes_from_changelog +from release.utils import ScriptError +from release.utils import update_init_py_version +from release.utils import update_run_sh_version + + +def create_initial_branch(repository, release, base, bintray_user): + release_branch = repository.create_release_branch(release, base) + print('Updating version info in __init__.py and run.sh') + update_run_sh_version(release) + update_init_py_version(release) + + input('Please add the release notes to the CHANGELOG.md file, then press Enter to continue.') + proceed = '' + while proceed.lower() != 'y': + print(repository.diff()) + proceed = input('Are these changes ok? y/N ') + + repository.create_bump_commit(release_branch, release) + repository.push_branch_to_remote(release_branch) + + bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user) + print('Creating data repository {} on bintray'.format(release_branch.name)) + bintray_api.create_repository(BINTRAY_ORG, release_branch.name, 'generic') + + +def monitor_pr_status(pr_data): + print('Waiting for CI to complete...') + last_commit = pr_data.get_commits().reversed[0] + while True: + status = last_commit.get_combined_status() + if status.state == 'pending': + summary = { + 'pending': 0, + 'success': 0, + 'failure': 0, + } + for detail in status.statuses: + summary[detail.state] += 1 + print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) + if status.total_count == 0: + # Mostly for testing purposes against repos with no CI setup + return True + time.sleep(30) + elif status.state == 'success': + print('{} successes: all clear!'.format(status.total_count)) + return True + else: + raise ScriptError('CI failure detected') + + +def create_release_draft(repository, version, pr_data, files): + print('Creating Github release draft') + with open(os.path.join(os.path.dirname(__file__), 'release.md.tmpl'), 'r') as f: + template = Template(f.read()) + print('Rendering release notes based on template') + release_notes = template.render( + version=version, + compat_matrix=compatibility_matrix(), + integrity=files, + contributors=get_contributors(pr_data), + changelog=read_release_notes_from_changelog(), + ) + gh_release = repository.create_release( + version, release_notes, draft=True, prerelease='-rc' in version, + target_commitish='release' + ) + print('Release draft initialized') + return gh_release + + +def resume(args): + raise NotImplementedError() + try: + repository = Repository(REPO_ROOT, args.repo or NAME) + br_name = branch_name(args.release) + if not repository.branch_exists(br_name): + raise ScriptError('No local branch exists for this release.') + # release_branch = repository.checkout_branch(br_name) + except ScriptError as e: + print(e) + return 1 + return 0 + + +def cancel(args): + try: + repository = Repository(REPO_ROOT, args.repo or NAME) + repository.close_release_pr(args.release) + repository.remove_release(args.release) + repository.remove_bump_branch(args.release) + # TODO: uncomment after testing is complete + # bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], args.bintray_user) + # print('Removing Bintray data repository for {}'.format(args.release)) + # bintray_api.delete_repository(BINTRAY_ORG, branch_name(args.release)) + except ScriptError as e: + print(e) + return 1 + print('Release cancellation complete.') + return 0 + + +def start(args): + try: + repository = Repository(REPO_ROOT, args.repo or NAME) + create_initial_branch(repository, args.release, args.base, args.bintray_user) + pr_data = repository.create_release_pull_request(args.release) + monitor_pr_status(pr_data) + downloader = BinaryDownloader(args.destination) + files = downloader.download_all(args.release) + gh_release = create_release_draft(repository, args.release, pr_data, files) + upload_assets(gh_release, files) + except ScriptError as e: + print(e) + return 1 + + return 0 + + +def main(): + if 'GITHUB_TOKEN' not in os.environ: + print('GITHUB_TOKEN environment variable must be set') + return 1 + + if 'BINTRAY_TOKEN' not in os.environ: + print('BINTRAY_TOKEN environment variable must be set') + return 1 + + parser = argparse.ArgumentParser( + description='Orchestrate a new release of docker/compose. This tool assumes that you have' + 'obtained a Github API token and Bintray API key and set the GITHUB_TOKEN and' + 'BINTRAY_TOKEN environment variables accordingly.', + epilog='''Example uses: + * Start a new feature release (includes all changes currently in master) + release.py -b user start 1.23.0 + * Start a new patch release + release.py -b user --patch 1.21.0 start 1.21.1 + * Cancel / rollback an existing release draft + release.py -b user cancel 1.23.0 + * Restart a previously aborted patch release + release.py -b user -p 1.21.0 resume 1.21.1 + ''', formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + 'action', choices=['start', 'resume', 'cancel'], + help='The action to be performed for this release' + ) + parser.add_argument('release', help='Release number, e.g. 1.9.0-rc1, 2.1.1') + parser.add_argument( + '--patch', '-p', dest='base', + help='Which version is being patched by this release' + ) + parser.add_argument( + '--repo', '-r', dest='repo', + help='Start a release for the given repo (default: {})'.format(NAME) + ) + parser.add_argument( + '-b', dest='bintray_user', required=True, metavar='USER', + help='Username associated with the Bintray API key' + ) + parser.add_argument( + '--destination', '-o', metavar='DIR', default='binaries', + help='Directory where release binaries will be downloaded relative to the project root' + ) + args = parser.parse_args() + + if args.action == 'start': + return start(args) + elif args.action == 'resume': + return resume(args) + elif args.action == 'cancel': + return cancel(args) + print('Unexpected action "{}"'.format(args.action), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/script/release/release/__init__.py b/script/release/release/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/script/release/release/bintray.py b/script/release/release/bintray.py new file mode 100644 index 000000000..d99d372c6 --- /dev/null +++ b/script/release/release/bintray.py @@ -0,0 +1,40 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import json + +import requests + +from .const import NAME + + +class BintrayAPI(requests.Session): + def __init__(self, api_key, user, *args, **kwargs): + super(BintrayAPI, self).__init__(*args, **kwargs) + self.auth = (user, api_key) + self.base_url = 'https://api.bintray.com/' + + def create_repository(self, subject, repo_name, repo_type='generic'): + url = '{base}/repos/{subject}/{repo_name}'.format( + base=self.base_url, subject=subject, repo_name=repo_name, + ) + data = { + 'name': repo_name, + 'type': repo_type, + 'private': False, + 'desc': 'Automated release for {}: {}'.format(NAME, repo_name), + 'labels': ['docker-compose', 'docker', 'release-bot'], + } + return self.post_json(url, data) + + def delete_repository(self, subject, repo_name): + url = '{base}/repos/{subject}/{repo_name}'.format( + base=self.base_url, subject=subject, repo_name=repo_name, + ) + return self.delete(url) + + def post_json(self, url, data, **kwargs): + if 'headers' not in kwargs: + kwargs['headers'] = {} + kwargs['headers']['Content-Type'] = 'application/json' + return self.post(url, data=json.dumps(data), **kwargs) diff --git a/script/release/release/const.py b/script/release/release/const.py new file mode 100644 index 000000000..34f338a89 --- /dev/null +++ b/script/release/release/const.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os + + +REPO_ROOT = os.path.join(os.path.dirname(__file__), '..', '..', '..') +NAME = 'shin-/compose' +BINTRAY_ORG = 'shin-compose' diff --git a/script/release/release/downloader.py b/script/release/release/downloader.py new file mode 100644 index 000000000..cd43bc993 --- /dev/null +++ b/script/release/release/downloader.py @@ -0,0 +1,72 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import hashlib +import os + +import requests + +from .const import BINTRAY_ORG +from .const import NAME +from .const import REPO_ROOT +from .utils import branch_name + + +class BinaryDownloader(requests.Session): + base_bintray_url = 'https://dl.bintray.com/{}'.format(BINTRAY_ORG) + base_appveyor_url = 'https://ci.appveyor.com/api/projects/{}/artifacts/'.format(NAME) + + def __init__(self, destination, *args, **kwargs): + super(BinaryDownloader, self).__init__(*args, **kwargs) + self.destination = destination + os.makedirs(self.destination, exist_ok=True) + + def download_from_bintray(self, repo_name, filename): + print('Downloading {} from bintray'.format(filename)) + url = '{base}/{repo_name}/{filename}'.format( + base=self.base_bintray_url, repo_name=repo_name, filename=filename + ) + full_dest = os.path.join(REPO_ROOT, self.destination, filename) + return self._download(url, full_dest) + + def download_from_appveyor(self, branch_name, filename): + print('Downloading {} from appveyor'.format(filename)) + url = '{base}/dist%2F{filename}?branch={branch_name}'.format( + base=self.base_appveyor_url, filename=filename, branch_name=branch_name + ) + full_dest = os.path.join(REPO_ROOT, self.destination, filename) + return self.download(url, full_dest) + + def _download(self, url, full_dest): + m = hashlib.sha256() + with open(full_dest, 'wb') as f: + r = self.get(url, stream=True) + for chunk in r.iter_content(chunk_size=1024 * 600, decode_unicode=False): + print('.', end='', flush=True) + m.update(chunk) + f.write(chunk) + + print(' download complete') + hex_digest = m.hexdigest() + with open(full_dest + '.sha256', 'w') as f: + f.write('{} {}\n'.format(hex_digest, os.path.basename(full_dest))) + return full_dest, hex_digest + + def download_all(self, version): + files = { + 'docker-compose-Darwin-x86_64': None, + 'docker-compose-Linux-x86_64': None, + # 'docker-compose-Windows-x86_64.exe': None, + } + + for filename in files.keys(): + if 'Windows' in filename: + files[filename] = self.download_from_appveyor( + branch_name(version), filename + ) + else: + files[filename] = self.download_from_bintray( + branch_name(version), filename + ) + return files diff --git a/script/release/release/repository.py b/script/release/release/repository.py new file mode 100644 index 000000000..77c697a99 --- /dev/null +++ b/script/release/release/repository.py @@ -0,0 +1,161 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os + +from git import GitCommandError +from git import Repo +from github import Github + +from .const import NAME +from .const import REPO_ROOT +from .utils import branch_name +from .utils import read_release_notes_from_changelog +from .utils import ScriptError + + +class Repository(object): + def __init__(self, root=None, gh_name=None): + if root is None: + root = REPO_ROOT + if gh_name is None: + gh_name = NAME + self.git_repo = Repo(root) + self.gh_client = Github(os.environ['GITHUB_TOKEN']) + self.gh_repo = self.gh_client.get_repo(gh_name) + + def create_release_branch(self, version, base=None): + print('Creating release branch {} based on {}...'.format(version, base or 'master')) + remote = self.find_remote(self.gh_repo.full_name) + remote.fetch() + if self.branch_exists(branch_name(version)): + raise ScriptError( + "Branch {} already exists locally. " + "Please remove it before running the release script.".format(branch_name(version)) + ) + if base is not None: + base = self.git_repo.tag('refs/tags/{}'.format(base)) + else: + base = 'refs/remotes/{}/master'.format(remote.name) + release_branch = self.git_repo.create_head(branch_name(version), commit=base) + release_branch.checkout() + self.git_repo.git.merge('--strategy=ours', '--no-edit', '{}/release'.format(remote.name)) + with release_branch.config_writer() as cfg: + cfg.set_value('release', version) + return release_branch + + def find_remote(self, remote_name=None): + if not remote_name: + remote_name = self.gh_repo.full_name + for remote in self.git_repo.remotes: + for url in remote.urls: + if remote_name in url: + return remote + return None + + def create_bump_commit(self, bump_branch, version): + print('Creating bump commit...') + bump_branch.checkout() + self.git_repo.git.commit('-a', '-s', '-m "Bump {}"'.format(version), '--no-verify') + + def diff(self): + return self.git_repo.git.diff() + + def checkout_branch(self, name): + return self.git_repo.branches[name].checkout() + + def push_branch_to_remote(self, branch, remote_name=None): + print('Pushing branch {} to remote...'.format(branch.name)) + remote = self.find_remote(remote_name) + remote.push(refspec=branch) + + def branch_exists(self, name): + return name in [h.name for h in self.git_repo.heads] + + def create_release_pull_request(self, version): + return self.gh_repo.create_pull( + title='Bump {}'.format(version), + body='Automated release for docker-compose {}\n\n{}'.format( + version, read_release_notes_from_changelog() + ), + base='release', + head=branch_name(version), + ) + + def create_release(self, version, release_notes, **kwargs): + return self.gh_repo.create_git_release( + tag=version, name=version, message=release_notes, **kwargs + ) + + def remove_release(self, version): + print('Removing release draft for {}'.format(version)) + releases = self.gh_repo.get_releases() + for release in releases: + if release.tag_name == version and release.title == version: + if not release.draft: + print( + 'The release at {} is no longer a draft. If you TRULY intend ' + 'to remove it, please do so manually.' + ) + continue + release.delete_release() + + def remove_bump_branch(self, version, remote_name=None): + name = branch_name(version) + if not self.branch_exists(name): + return False + print('Removing local branch "{}"'.format(name)) + if self.git_repo.active_branch.name == name: + print('Active branch is about to be deleted. Checking out to master...') + try: + self.checkout_branch('master') + except GitCommandError: + raise ScriptError( + 'Unable to checkout master. Try stashing local changes before proceeding.' + ) + self.git_repo.branches[name].delete(self.git_repo, name, force=True) + print('Removing remote branch "{}"'.format(name)) + remote = self.find_remote(remote_name) + try: + remote.push(name, delete=True) + except GitCommandError as e: + if 'remote ref does not exist' in str(e): + return False + raise ScriptError( + 'Error trying to remove remote branch: {}'.format(e) + ) + return True + + def close_release_pr(self, version): + print('Retrieving and closing release PR for {}'.format(version)) + name = branch_name(version) + open_prs = self.gh_repo.get_pulls(state='open') + count = 0 + for pr in open_prs: + if pr.head.ref == name: + print('Found matching PR #{}'.format(pr.number)) + pr.edit(state='closed') + count += 1 + if count == 0: + print('No open PR for this release branch.') + return count + + +def get_contributors(pr_data): + commits = pr_data.get_commits() + authors = {} + for commit in commits: + author = commit.author.login + authors[author] = authors.get(author, 0) + 1 + return [x[0] for x in sorted(list(authors.items()), key=lambda x: x[1])] + + +def upload_assets(gh_release, files): + print('Uploading binaries and hash sums') + for filename, filedata in files.items(): + print('Uploading {}...'.format(filename)) + gh_release.upload_asset(filedata[0], content_type='application/octet-stream') + gh_release.upload_asset('{}.sha256'.format(filedata[0]), content_type='text/plain') + gh_release.upload_asset( + os.path.join(REPO_ROOT, 'script', 'run', 'run.sh'), content_type='text/plain' + ) diff --git a/script/release/release/utils.py b/script/release/release/utils.py new file mode 100644 index 000000000..b0e1f6a84 --- /dev/null +++ b/script/release/release/utils.py @@ -0,0 +1,63 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import re + +from .const import REPO_ROOT +from compose import const as compose_const + +section_header_re = re.compile(r'^[0-9]+\.[0-9]+\.[0-9]+ \([0-9]{4}-[01][0-9]-[0-3][0-9]\)$') + + +class ScriptError(Exception): + pass + + +def branch_name(version): + return 'bump-{}'.format(version) + + +def read_release_notes_from_changelog(): + with open(os.path.join(REPO_ROOT, 'CHANGELOG.md'), 'r') as f: + lines = f.readlines() + i = 0 + while i < len(lines): + if section_header_re.match(lines[i]): + break + i += 1 + + j = i + 1 + while j < len(lines): + if section_header_re.match(lines[j]): + break + j += 1 + + return ''.join(lines[i + 2:j - 1]) + + +def update_init_py_version(version): + path = os.path.join(REPO_ROOT, 'compose', '__init__.py') + with open(path, 'r') as f: + contents = f.read() + contents = re.sub(r"__version__ = '[0-9a-z.-]+'", "__version__ = '{}'".format(version), contents) + with open(path, 'w') as f: + f.write(contents) + + +def update_run_sh_version(version): + path = os.path.join(REPO_ROOT, 'script', 'run', 'run.sh') + with open(path, 'r') as f: + contents = f.read() + contents = re.sub(r'VERSION="[0-9a-z.-]+"', 'VERSION="{}"'.format(version), contents) + with open(path, 'w') as f: + f.write(contents) + + +def compatibility_matrix(): + result = {} + for engine_version in compose_const.API_VERSION_TO_ENGINE_VERSION.values(): + result[engine_version] = [] + for fmt, api_version in compose_const.API_VERSIONS.items(): + result[compose_const.API_VERSION_TO_ENGINE_VERSION[api_version]].append(fmt.vstring) + return result