Refactor release and build scripts

- Make use of the same Dockerfile when producing
an image for testing and for deploying to
DockerHub

Signed-off-by: Ulysses Souza <ulysses.souza@docker.com>
This commit is contained in:
Ulysses Souza 2019-04-12 18:46:06 +02:00
parent c217bab7f6
commit 2b24eb693c
13 changed files with 126 additions and 68 deletions

8
Jenkinsfile vendored
View File

@ -10,7 +10,13 @@ def buildImage = { String baseImage ->
try { try {
image.pull() image.pull()
} catch (Exception exc) { } catch (Exception exc) {
sh "docker build -t ${imageName} --target build --build-arg BUILD_PLATFORM=${baseImage} ." sh """GIT_COMMIT=\$(script/build/write-git-sha) && \\
docker build -t ${imageName} \\
--target build \\
--build-arg BUILD_PLATFORM="${baseImage}" \\
--build-arg GIT_COMMIT="${GIT_COMMIT}" \\
.\\
"""
sh "docker push ${imageName}" sh "docker push ${imageName}"
echo "${imageName}" echo "${imageName}"
return imageName return imageName

View File

@ -7,11 +7,14 @@ if [ -z "$1" ]; then
exit 1 exit 1
fi fi
TAG=$1 TAG="$1"
VERSION="$(python setup.py --version)" VERSION="$(python setup.py --version)"
./script/build/write-git-sha DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
python setup.py sdist bdist_wheel python setup.py sdist bdist_wheel
./script/build/linux
docker build -t docker/compose:$TAG -f Dockerfile.run . docker build \
--build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" \
-t "${TAG}" .

View File

@ -4,15 +4,14 @@ set -ex
./script/clean ./script/clean
TMP_CONTAINER="tmpcontainer" DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
TAG="docker/compose:tmp-glibc-linux-binary" TAG="docker/compose:tmp-glibc-linux-binary-${DOCKER_COMPOSE_GITSHA}"
DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha)
docker build -t "${TAG}" . \ docker build -t "${TAG}" . \
--build-arg BUILD_PLATFORM=debian \ --build-arg BUILD_PLATFORM=debian \
--build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}"
docker create --name ${TMP_CONTAINER} ${TAG} TMP_CONTAINER=$(docker create "${TAG}")
mkdir -p dist mkdir -p dist
docker cp ${TMP_CONTAINER}:/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64
docker container rm -f ${TMP_CONTAINER} docker container rm -f "${TMP_CONTAINER}"
docker image rm -f ${TAG} docker image rm -f "${TAG}"

View File

@ -3,16 +3,19 @@
set -ex set -ex
CODE_PATH=/code CODE_PATH=/code
VENV=${CODE_PATH}/.tox/py37 VENV="${CODE_PATH}"/.tox/py37
cd ${CODE_PATH} cd "${CODE_PATH}"
mkdir -p dist mkdir -p dist
chmod 777 dist chmod 777 dist
${VENV}/bin/pip3 install -q -r requirements-build.txt "${VENV}"/bin/pip3 install -q -r requirements-build.txt
# TODO(ulyssessouza) To check if really needed # TODO(ulyssessouza) To check if really needed
./script/build/write-git-sha if [ -z "${DOCKER_COMPOSE_GITSHA}" ]; then
DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
fi
echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
export PATH="${CODE_PATH}/pyinstaller:${PATH}" export PATH="${CODE_PATH}/pyinstaller:${PATH}"
@ -21,15 +24,15 @@ if [ ! -z "${BUILD_BOOTLOADER}" ]; then
git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller
cd /tmp/pyinstaller/bootloader cd /tmp/pyinstaller/bootloader
git checkout v3.4 git checkout v3.4
${VENV}/bin/python3 ./waf configure --no-lsb all "${VENV}"/bin/python3 ./waf configure --no-lsb all
${VENV}/bin/pip3 install .. "${VENV}"/bin/pip3 install ..
cd ${CODE_PATH} cd "${CODE_PATH}"
rm -Rf /tmp/pyinstaller rm -Rf /tmp/pyinstaller
else else
echo "NOT compiling bootloader!!!" echo "NOT compiling bootloader!!!"
fi fi
${VENV}/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec "${VENV}"/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec
ls -la dist/ ls -la dist/
ldd dist/docker-compose ldd dist/docker-compose
mv dist/docker-compose /usr/local/bin mv dist/docker-compose /usr/local/bin

View File

@ -5,11 +5,12 @@ TOOLCHAIN_PATH="$(realpath $(dirname $0)/../../build/toolchain)"
rm -rf venv rm -rf venv
virtualenv -p ${TOOLCHAIN_PATH}/bin/python3 venv virtualenv -p "${TOOLCHAIN_PATH}"/bin/python3 venv
venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements.txt
venv/bin/pip install -r requirements-build.txt venv/bin/pip install -r requirements-build.txt
venv/bin/pip install --no-deps . venv/bin/pip install --no-deps .
./script/build/write-git-sha DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
venv/bin/pyinstaller docker-compose.spec venv/bin/pyinstaller docker-compose.spec
mv dist/docker-compose dist/docker-compose-Darwin-x86_64 mv dist/docker-compose dist/docker-compose-Darwin-x86_64
dist/docker-compose-Darwin-x86_64 version dist/docker-compose-Darwin-x86_64 version

View File

@ -10,9 +10,9 @@ fi
TAG="$1" TAG="$1"
IMAGE="docker/compose-tests" IMAGE="docker/compose-tests"
DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
docker build -t "${IMAGE}:${TAG}" . \ docker build -t "${IMAGE}:${TAG}" . \
--target build \ --target build \
--build-arg BUILD_PLATFORM=debian \ --build-arg BUILD_PLATFORM="debian" \
--build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}"
docker tag ${IMAGE}:${TAG} ${IMAGE}:latest docker tag "${IMAGE}":"${TAG}" "${IMAGE}":latest

View File

@ -9,4 +9,4 @@ if [[ "${?}" != "0" ]]; then
echo "Couldn't get revision of the git repository. Setting to 'unknown' instead" echo "Couldn't get revision of the git repository. Setting to 'unknown' instead"
DOCKER_COMPOSE_GITSHA="unknown" DOCKER_COMPOSE_GITSHA="unknown"
fi fi
echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA echo "${DOCKER_COMPOSE_GITSHA}"

View File

@ -204,7 +204,7 @@ def resume(args):
delete_assets(gh_release) delete_assets(gh_release)
upload_assets(gh_release, files) upload_assets(gh_release, files)
img_manager = ImageManager(args.release) img_manager = ImageManager(args.release)
img_manager.build_images(repository, files) img_manager.build_images(repository)
except ScriptError as e: except ScriptError as e:
print(e) print(e)
return 1 return 1
@ -244,7 +244,7 @@ def start(args):
gh_release = create_release_draft(repository, args.release, pr_data, files) gh_release = create_release_draft(repository, args.release, pr_data, files)
upload_assets(gh_release, files) upload_assets(gh_release, files)
img_manager = ImageManager(args.release) img_manager = ImageManager(args.release)
img_manager.build_images(repository, files) img_manager.build_images(repository)
except ScriptError as e: except ScriptError as e:
print(e) print(e)
return 1 return 1
@ -258,7 +258,8 @@ def finalize(args):
try: try:
check_pypirc() check_pypirc()
repository = Repository(REPO_ROOT, args.repo) repository = Repository(REPO_ROOT, args.repo)
img_manager = ImageManager(args.release) tag_as_latest = _check_if_tag_latest(args.release)
img_manager = ImageManager(args.release, tag_as_latest)
pr_data = repository.find_release_pr(args.release) pr_data = repository.find_release_pr(args.release)
if not pr_data: if not pr_data:
raise ScriptError('No PR found for {}'.format(args.release)) raise ScriptError('No PR found for {}'.format(args.release))
@ -314,6 +315,12 @@ EPILOG = '''Example uses:
''' '''
# Checks if this version respects the GA version format ('x.y.z') and not an RC
def _check_if_tag_latest(version):
ga_version = all(n.isdigit() for n in version.split('.')) and version.count('.') == 2
return ga_version and yesno('Should this release be tagged as \"latest\"? Y/n', default=True)
def main(): def main():
if 'GITHUB_TOKEN' not in os.environ: if 'GITHUB_TOKEN' not in os.environ:
print('GITHUB_TOKEN environment variable must be set') print('GITHUB_TOKEN environment variable must be set')

View File

@ -5,18 +5,29 @@ from __future__ import unicode_literals
import base64 import base64
import json import json
import os import os
import shutil
import docker import docker
from enum import Enum
from .const import NAME
from .const import REPO_ROOT from .const import REPO_ROOT
from .utils import ScriptError from .utils import ScriptError
class Platform(Enum):
ALPINE = 'alpine'
DEBIAN = 'debian'
def __str__(self):
return self.value
class ImageManager(object): class ImageManager(object):
def __init__(self, version): def __init__(self, version, latest=False):
self.built_tags = []
self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env())
self.version = version self.version = version
self.latest = latest
if 'HUB_CREDENTIALS' in os.environ: if 'HUB_CREDENTIALS' in os.environ:
print('HUB_CREDENTIALS found in environment, issuing login') print('HUB_CREDENTIALS found in environment, issuing login')
credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS'])) credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS']))
@ -24,16 +35,31 @@ class ImageManager(object):
username=credentials['Username'], password=credentials['Password'] username=credentials['Username'], password=credentials['Password']
) )
def build_images(self, repository, files): def _tag(self, image, existing_tag, new_tag):
print("Building release images...") existing_repo_tag = '{image}:{tag}'.format(image=image, tag=existing_tag)
repository.write_git_sha() new_repo_tag = '{image}:{tag}'.format(image=image, tag=new_tag)
distdir = os.path.join(REPO_ROOT, 'dist') self.docker_client.tag(existing_repo_tag, new_repo_tag)
os.makedirs(distdir, exist_ok=True) self.built_tags.append(new_repo_tag)
shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir)
os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755) def build_runtime_image(self, repository, platform):
print('Building docker/compose image') git_sha = repository.write_git_sha()
compose_image_base_name = NAME
print('Building {image} image ({platform} based)'.format(
image=compose_image_base_name,
platform=platform
))
full_version = '{version}-{platform}'.format(version=self.version, platform=platform)
build_tag = '{image_base_image}:{full_version}'.format(
image_base_image=compose_image_base_name,
full_version=full_version
)
logstream = self.docker_client.build( logstream = self.docker_client.build(
REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', REPO_ROOT,
tag=build_tag,
buildargs={
'BUILD_PLATFORM': platform.value,
'GIT_COMMIT': git_sha,
},
decode=True decode=True
) )
for chunk in logstream: for chunk in logstream:
@ -42,9 +68,32 @@ class ImageManager(object):
if 'stream' in chunk: if 'stream' in chunk:
print(chunk['stream'], end='') print(chunk['stream'], end='')
print('Building test image (for UCP e2e)') self.built_tags.append(build_tag)
if platform == Platform.ALPINE:
self._tag(compose_image_base_name, full_version, self.version)
if self.latest:
self._tag(compose_image_base_name, full_version, platform)
if platform == Platform.ALPINE:
self._tag(compose_image_base_name, full_version, 'latest')
# Used for producing a test image for UCP
def build_ucp_test_image(self, repository):
print('Building test image (debian based for UCP e2e)')
git_sha = repository.write_git_sha()
compose_tests_image_base_name = NAME + '-tests'
ucp_test_image_tag = '{image}:{tag}'.format(
image=compose_tests_image_base_name,
tag=self.version
)
logstream = self.docker_client.build( logstream = self.docker_client.build(
REPO_ROOT, tag='docker-compose-tests:tmp', decode=True REPO_ROOT,
tag=ucp_test_image_tag,
target='build',
buildargs={
'BUILD_PLATFORM': Platform.DEBIAN.value,
'GIT_COMMIT': git_sha,
},
decode=True
) )
for chunk in logstream: for chunk in logstream:
if 'error' in chunk: if 'error' in chunk:
@ -52,26 +101,16 @@ class ImageManager(object):
if 'stream' in chunk: if 'stream' in chunk:
print(chunk['stream'], end='') print(chunk['stream'], end='')
container = self.docker_client.create_container( self.built_tags.append(ucp_test_image_tag)
'docker-compose-tests:tmp', entrypoint='tox' self._tag(compose_tests_image_base_name, self.version, 'latest')
)
self.docker_client.commit(container, 'docker/compose-tests', 'latest')
self.docker_client.tag(
'docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version)
)
self.docker_client.remove_container(container, force=True)
self.docker_client.remove_image('docker-compose-tests:tmp', force=True)
@property def build_images(self, repository):
def image_names(self): self.build_runtime_image(repository, Platform.ALPINE)
return [ self.build_runtime_image(repository, Platform.DEBIAN)
'docker/compose-tests:latest', self.build_ucp_test_image(repository)
'docker/compose-tests:{}'.format(self.version),
'docker/compose:{}'.format(self.version)
]
def check_images(self): def check_images(self):
for name in self.image_names: for name in self.built_tags:
try: try:
self.docker_client.inspect_image(name) self.docker_client.inspect_image(name)
except docker.errors.ImageNotFound: except docker.errors.ImageNotFound:
@ -80,7 +119,7 @@ class ImageManager(object):
return True return True
def push_images(self): def push_images(self):
for name in self.image_names: for name in self.built_tags:
print('Pushing {} to Docker Hub'.format(name)) print('Pushing {} to Docker Hub'.format(name))
logstream = self.docker_client.push(name, stream=True, decode=True) logstream = self.docker_client.push(name, stream=True, decode=True)
for chunk in logstream: for chunk in logstream:

View File

@ -175,6 +175,7 @@ class Repository(object):
def write_git_sha(self): def write_git_sha(self):
with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f: with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f:
f.write(self.git_repo.head.commit.hexsha[:7]) f.write(self.git_repo.head.commit.hexsha[:7])
return self.git_repo.head.commit.hexsha[:7]
def cherry_pick_prs(self, release_branch, ids): def cherry_pick_prs(self, release_branch, ids):
if not ids: if not ids:

View File

@ -48,7 +48,7 @@ fi
# Only allocate tty if we detect one # Only allocate tty if we detect one
if [ -t 0 -a -t 1 ]; then if [ -t 0 -a -t 1 ]; then
DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t"
fi fi
# Always set -i to support piped and terminal input in run/exec # Always set -i to support piped and terminal input in run/exec

View File

@ -8,8 +8,7 @@ set -e
docker run --rm \ docker run --rm \
--tty \ --tty \
${GIT_VOLUME} \ ${GIT_VOLUME} \
--entrypoint="tox" \ "$TAG" tox -e pre-commit
"$TAG" -e pre-commit
get_versions="docker run --rm get_versions="docker run --rm
--entrypoint=/code/.tox/py27/bin/python --entrypoint=/code/.tox/py27/bin/python

View File

@ -5,16 +5,16 @@ set -ex
TAG="docker-compose:alpine-$(git rev-parse --short HEAD)" TAG="docker-compose:alpine-$(git rev-parse --short HEAD)"
# By default use the Dockerfile.alpine, but can be overridden to use an alternative file # By default use the Dockerfile, but can be overridden to use an alternative file
# e.g DOCKERFILE=Dockerfile.armhf script/test/default # e.g DOCKERFILE=Dockerfile.armhf script/test/default
DOCKERFILE="${DOCKERFILE:-Dockerfile.alpine}" DOCKERFILE="${DOCKERFILE:-Dockerfile}"
DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}" DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}"
rm -rf coverage-html rm -rf coverage-html
# Create the host directory so it's owned by $USER # Create the host directory so it's owned by $USER
mkdir -p coverage-html mkdir -p coverage-html
docker build -f ${DOCKERFILE} -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" . docker build -f "${DOCKERFILE}" -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" .
GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" GIT_VOLUME="--volume=$(pwd)/.git:/code/.git"
. script/test/all . script/test/all