Merge pull request #6080 from chris-crone/macos-rework-build

Rework build on macOS

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   test:
     macos:
-      xcode: "8.3.3"
+      xcode: "9.4.1"
     steps:
     - checkout
     - run:
@@ -17,7 +17,7 @@ jobs:
 
   build-osx-binary:
     macos:
-      xcode: "8.3.3"
+      xcode: "9.4.1"
     steps:
       - checkout
       - run:
@@ -25,18 +25,17 @@ jobs:
           command: sudo pip install --upgrade pip virtualenv
       - run:
          name: setup script
-         command: ./script/setup/osx
+         command: DEPLOYMENT_TARGET=10.11 ./script/setup/osx
       - run:
          name: build script
          command: ./script/build/osx
       - store_artifacts:
           path: dist/docker-compose-Darwin-x86_64
           destination: docker-compose-Darwin-x86_64
-      # - deploy:
+      - deploy:
-      #     name: Deploy binary to bintray
+          name: Deploy binary to bintray
-      #     command: |
+          command: |
-      #       OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh
+            OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh
-
 
   build-linux-binary:
     machine:
@@ -54,28 +53,6 @@ jobs:
           command: |
             OS_NAME=Linux PKG_NAME=linux ./script/circle/bintray-deploy.sh
 
-  trigger-osx-binary-deploy:
-    # We use a separate repo to build OSX binaries meant for distribution
-    # with support for OSSX 10.11 (xcode 7). This job triggers a build on
-    # that repo.
-    docker:
-      - image: alpine:3.6
-
-    steps:
-      - run:
-          name: install curl
-          command: apk update && apk add curl
-
-      - run:
-          name: API trigger
-          command: |
-            curl -X POST -H "Content-Type: application/json" -d "{\
-              \"build_parameters\": {\
-                \"COMPOSE_BRANCH\": \"${CIRCLE_BRANCH}\"\
-              }\
-            }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release?circle-token=${OSX_RELEASE_TOKEN} \
-            > /dev/null
-
 
 workflows:
   version: 2
@@ -84,9 +61,3 @@ workflows:
       - test
       - build-linux-binary
       - build-osx-binary
-      - trigger-osx-binary-deploy:
-          filters:
-            branches:
-              only:
-                - master
-                - /bump-.*/

script/build/osx

@@ -1,11 +1,11 @@
 #!/bin/bash
 set -ex
 
-PATH="/usr/local/bin:$PATH"
+TOOLCHAIN_PATH="$(realpath $(dirname $0)/../../build/toolchain)"
 
 rm -rf venv
 
-virtualenv -p /usr/local/bin/python3 venv
+virtualenv -p ${TOOLCHAIN_PATH}/bin/python3 venv
 venv/bin/pip install -r requirements.txt
 venv/bin/pip install -r requirements-build.txt
 venv/bin/pip install --no-deps .

script/setup/osx

@@ -1,43 +1,104 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -ex
 
-python_version() {
+. $(dirname $0)/osx_helpers.sh
-  python -V 2>&1
-}
 
-python3_version() {
+DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET:-"$(macos_version)"}
-  python3 -V 2>&1
+SDK_FETCH=
-}
+if ! [ ${DEPLOYMENT_TARGET} == "$(macos_version)" ]; then
+  SDK_FETCH=1
+  # SDK URL from https://github.com/docker/golang-cross/blob/master/osx-cross.sh
+  SDK_URL=https://s3.dockerproject.org/darwin/v2/MacOSX${DEPLOYMENT_TARGET}.sdk.tar.xz
+  SDK_SHA1=dd228a335194e3392f1904ce49aff1b1da26ca62
+fi
 
-openssl_version() {
+OPENSSL_VERSION=1.1.0h
-  python -c "import ssl; print ssl.OPENSSL_VERSION"
+OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-}
+OPENSSL_SHA1=0fc39f6aa91b6e7f4d05018f7c5e991e1d2491fd
 
-desired_python3_version="3.6.4"
+PYTHON_VERSION=3.6.6
-desired_python3_brew_version="3.6.4_2"
+PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
-python3_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e69a9a592232fa5a82741f6acecffc2f1d198d/Formula/python3.rb"
+PYTHON_SHA1=ae1fc9ddd29ad8c1d5f7b0d799ff0787efeb9652
 
-PATH="/usr/local/bin:$PATH"
+#
-
+# Install prerequisites.
-if !(which brew); then
+#
+if ! [ -x "$(command -v brew)" ]; then
   ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
 fi
-
+if ! [ -x "$(command -v grealpath)" ]; then
-brew update > /dev/null
+  brew update > /dev/null
-
+  brew install coreutils
-if !(python3_version | grep "$desired_python3_version"); then
-  if brew list | grep python3; then
-    brew unlink python3
-  fi
-
-  brew install "$python3_formula"
-  brew switch python3 "$desired_python3_brew_version"
 fi
-
+if ! [ -x "$(command -v python3)" ]; then
-echo "*** Using $(python3_version) ; $(python_version)"
+  brew update > /dev/null
-echo "*** Using $(openssl_version)"
+  brew install python3
-
+fi
-if !(which virtualenv); then
+if ! [ -x "$(command -v virtualenv)" ]; then
   pip install virtualenv
 fi
+
+#
+# Create toolchain directory.
+#
+BUILD_PATH="$(grealpath $(dirname $0)/../../build)"
+mkdir -p ${BUILD_PATH}
+TOOLCHAIN_PATH="${BUILD_PATH}/toolchain"
+mkdir -p ${TOOLCHAIN_PATH}
+
+#
+# Set macOS SDK.
+#
+if [ ${SDK_FETCH} ]; then
+  SDK_PATH=${TOOLCHAIN_PATH}/MacOSX${DEPLOYMENT_TARGET}.sdk
+  fetch_tarball ${SDK_URL} ${SDK_PATH} ${SDK_SHA1}
+else
+  SDK_PATH="$(xcode-select --print-path)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX${DEPLOYMENT_TARGET}.sdk"
+fi
+
+#
+# Build OpenSSL.
+#
+OPENSSL_SRC_PATH=${TOOLCHAIN_PATH}/openssl-${OPENSSL_VERSION}
+if ! [ -f ${TOOLCHAIN_PATH}/bin/openssl ]; then
+  rm -rf ${OPENSSL_SRC_PATH}
+  fetch_tarball ${OPENSSL_URL} ${OPENSSL_SRC_PATH} ${OPENSSL_SHA1}
+  (
+    cd ${OPENSSL_SRC_PATH}
+    export MACOSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET}
+    export SDKROOT=${SDK_PATH}
+    ./Configure darwin64-x86_64-cc --prefix=${TOOLCHAIN_PATH}
+    make install_sw install_dev
+  )
+fi
+
+#
+# Build Python.
+#
+PYTHON_SRC_PATH=${TOOLCHAIN_PATH}/Python-${PYTHON_VERSION}
+if ! [ -f ${TOOLCHAIN_PATH}/bin/python3 ]; then
+  rm -rf ${PYTHON_SRC_PATH}
+  fetch_tarball ${PYTHON_URL} ${PYTHON_SRC_PATH} ${PYTHON_SHA1}
+  (
+    cd ${PYTHON_SRC_PATH}
+    ./configure --prefix=${TOOLCHAIN_PATH} \
+      --enable-ipv6 --without-ensurepip --with-dtrace --without-gcc \
+      --datarootdir=${TOOLCHAIN_PATH}/share \
+      --datadir=${TOOLCHAIN_PATH}/share \
+      --enable-framework=${TOOLCHAIN_PATH}/Frameworks \
+      MACOSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET} \
+      CFLAGS="-isysroot ${SDK_PATH} -I${TOOLCHAIN_PATH}/include" \
+      CPPFLAGS="-I${SDK_PATH}/usr/include -I${TOOLCHAIN_PATH}include" \
+      LDFLAGS="-isysroot ${SDK_PATH} -L ${TOOLCHAIN_PATH}/lib"
+    make -j 4
+    make install PYTHONAPPSDIR=${TOOLCHAIN_PATH}
+    make frameworkinstallextras PYTHONAPPSDIR=${TOOLCHAIN_PATH}/share
+  )
+fi
+
+echo ""
+echo "*** Targeting macOS: ${DEPLOYMENT_TARGET}"
+echo "*** Using SDK ${SDK_PATH}"
+echo "*** Using $(python3_version ${TOOLCHAIN_PATH})"
+echo "*** Using $(openssl_version ${TOOLCHAIN_PATH})"

script/setup/osx_helpers.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Check file's ($1) SHA1 ($2).
+check_sha1() {
+  echo -n "$2 *$1" | shasum -c -
+}
+
+# Download URL ($1) to path ($2).
+download() {
+  curl -L $1 -o $2
+}
+
+# Extract tarball ($1) in folder ($2).
+extract() {
+  tar xf $1 -C $2
+}
+
+# Download URL ($1), check SHA1 ($3), and extract utility ($2).
+fetch_tarball() {
+  url=$1
+  tarball=$2.tarball
+  sha1=$3
+  download $url $tarball
+  check_sha1 $tarball $sha1
+  extract $tarball $(dirname $tarball)
+}
+
+# Version of Python at toolchain path ($1).
+python3_version() {
+  $1/bin/python3 -V 2>&1
+}
+
+# Version of OpenSSL used by toolchain ($1) Python.
+openssl_version() {
+  $1/bin/python3 -c "import ssl; print(ssl.OPENSSL_VERSION)"
+}
+
+# System macOS version.
+macos_version() {
+  sw_vers -productVersion | cut -f1,2 -d'.'
+}