Merge pull request #5588 from docker/python3-binaries

Compile binaries on Python 3.6

.circleci/config.yml

@@ -5,15 +5,15 @@ jobs:
       xcode: "8.3.3"
     steps:
     - checkout
-#    - run:
-#        name: install python3
-#        command: brew install python3
+    - run:
+        name: install python3
+        command: brew update > /dev/null && brew install python3
     - run:
         name: install tox
         command: sudo pip install --upgrade tox==2.1.1
     - run:
         name: unit tests
-        command: tox -e py27 -- tests/unit
+        command: tox -e py27,py36 -- tests/unit
 
   build-osx-binary:
     macos:

Dockerfile

@@ -39,19 +39,19 @@ RUN set -ex; \
     rm -rf /Python-2.7.13; \
     rm Python-2.7.13.tgz
 
-# Build python 3.4 from source
+# Build python 3.6 from source
 RUN set -ex; \
-    curl -LO https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz && \
-    SHA256=fe59daced99549d1d452727c050ae486169e9716a890cffb0d468b376d916b48; \
-    echo "${SHA256}  Python-3.4.6.tgz" | sha256sum -c - && \
-    tar -xzf Python-3.4.6.tgz; \
-    cd Python-3.4.6; \
+    curl -LO https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz && \
+    SHA256=9de6494314ea199e3633211696735f65; \
+    echo "${SHA256}  Python-3.6.4.tgz" | md5sum -c - && \
+    tar -xzf Python-3.6.4.tgz; \
+    cd Python-3.6.4; \
     ./configure --enable-shared; \
     make; \
     make install; \
     cd ..; \
-    rm -rf /Python-3.4.6; \
-    rm Python-3.4.6.tgz
+    rm -rf /Python-3.6.4; \
+    rm Python-3.6.4.tgz
 
 # Make libpython findable
 ENV LD_LIBRARY_PATH /usr/local/lib

Dockerfile.armhf

@@ -33,15 +33,15 @@ RUN set -ex; \
     cd ..; \
     rm -rf /Python-2.7.13
 
-# Build python 3.4 from source
+# Build python 3.6 from source
 RUN set -ex; \
-    curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \
-    cd Python-3.4.6; \
+    curl -L https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz | tar -xz; \
+    cd Python-3.6.4; \
     ./configure --enable-shared; \
     make; \
     make install; \
     cd ..; \
-    rm -rf /Python-3.4.6
+    rm -rf /Python-3.6.4
 
 # Make libpython findable
 ENV LD_LIBRARY_PATH /usr/local/lib

Jenkinsfile

@@ -60,5 +60,5 @@ buildImage()
 parallel(
   failFast: true,
   all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"),
-  all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"),
+  all_py36: runTests(pythonVersions: "py36", dockerVersions: "all"),
 )

appveyor.yml

@@ -2,15 +2,15 @@
 version: '{branch}-{build}'
 
 install:
-  - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%"
+  - "SET PATH=C:\\Python36-x64;C:\\Python36-x64\\Scripts;%PATH%"
   - "python --version"
-  - "pip install tox==2.1.1 virtualenv==13.1.2"
+  - "pip install tox==2.9.1 virtualenv==15.1.0"
 
 # Build the binary after tests
 build: false
 
 test_script:
-  - "tox -e py27,py34 -- tests/unit"
+  - "tox -e py27,py36 -- tests/unit"
   - ps: ".\\script\\build\\windows.ps1"
 
 artifacts:

requirements-build.txt

@@ -1 +1 @@
-pyinstaller==3.2.1
+pyinstaller==3.3.1

requirements-dev.txt

@@ -1,5 +1,5 @@
-coverage==3.7.1
+coverage==4.4.2
 flake8==3.5.0
 mock>=1.0.1
-pytest==2.7.2
-pytest-cov==2.1.0
+pytest==2.9.2
+pytest-cov==2.5.1

requirements.txt

@@ -12,7 +12,8 @@ git+git://github.com/tartley/colorama.git@bd378c725b45eba0b8e5cc091c3ca76a954c92
 idna==2.5
 ipaddress==1.0.18
 jsonschema==2.6.0
-pypiwin32==219; sys_platform == 'win32'
+pypiwin32==219; sys_platform == 'win32' and python_version < '3.6'
+pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6'
 PySocks==1.6.7
 PyYAML==3.12
 requests==2.18.4

script/build/linux-entrypoint

@@ -3,7 +3,7 @@
 set -ex
 
 TARGET=dist/docker-compose-$(uname -s)-$(uname -m)
-VENV=/code/.tox/py27
+VENV=/code/.tox/py36
 
 mkdir -p `pwd`/dist
 chmod 777 `pwd`/dist

script/build/osx

@@ -5,7 +5,7 @@ PATH="/usr/local/bin:$PATH"
 
 rm -rf venv
 
-virtualenv -p /usr/local/bin/python venv
+virtualenv -p /usr/local/bin/python3 venv
 venv/bin/pip install -r requirements.txt
 venv/bin/pip install -r requirements-build.txt
 venv/bin/pip install --no-deps .

script/build/windows.ps1

@@ -6,17 +6,17 @@
 #
 #        http://git-scm.com/download/win
 #
-# 2. Install Python 2.7.10:
+# 2. Install Python 3.6.4:
 #
 #        https://www.python.org/downloads/
 #
-# 3. Append ";C:\Python27;C:\Python27\Scripts" to the "Path" environment variable:
+# 3. Append ";C:\Python36;C:\Python36\Scripts" to the "Path" environment variable:
 #
 #        https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx?mfr=true
 #
 # 4. In Powershell, run the following commands:
 #
-#        $ pip install virtualenv
+#        $ pip install 'virtualenv>=15.1.0'
 #        $ Set-ExecutionPolicy -Scope CurrentUser RemoteSigned
 #
 # 5. Clone the repository:
@@ -45,7 +45,12 @@ virtualenv .\venv
 $ErrorActionPreference = "Continue"
 
 # Install dependencies
-.\venv\Scripts\pip install pypiwin32==219
+# Fix for https://github.com/pypa/pip/issues/3964
+# Remove-Item -Recurse -Force .\venv\Lib\site-packages\pip
+# .\venv\Scripts\easy_install pip==9.0.1
+# .\venv\Scripts\pip install --upgrade pip setuptools
+# End fix
+.\venv\Scripts\pip install pypiwin32==220
 .\venv\Scripts\pip install -r requirements.txt
 .\venv\Scripts\pip install --no-deps .
 .\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt

script/circle/bintray-deploy.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -x
+
 curl -f -u$BINTRAY_USERNAME:$BINTRAY_API_KEY -X GET \
   https://api.bintray.com/repos/docker-compose/${CIRCLE_BRANCH}
 

script/clean

@@ -2,6 +2,7 @@
 set -e
 
 find . -type f -name '*.pyc' -delete
+rm -rf .coverage-binfiles
 find . -name .coverage.* -delete
 find . -name __pycache__ -delete
 rm -rf docs/_site build dist docker-compose.egg-info

script/setup/osx

@@ -6,11 +6,36 @@ python_version() {
   python -V 2>&1
 }
 
+python3_version() {
+  python3 -V 2>&1
+}
+
 openssl_version() {
   python -c "import ssl; print ssl.OPENSSL_VERSION"
 }
 
-echo "*** Using $(python_version)"
+desired_python3_version="3.6.4"
+desired_python3_brew_version="3.6.4_2"
+python3_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e69a9a592232fa5a82741f6acecffc2f1d198d/Formula/python3.rb"
+
+PATH="/usr/local/bin:$PATH"
+
+if !(which brew); then
+  ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+fi
+
+brew update > /dev/null
+
+if !(python3_version | grep "$desired_python3_version"); then
+  if brew list | grep python3; then
+    brew unlink python3
+  fi
+
+  brew install "$python3_formula"
+  brew switch python3 "$desired_python3_brew_version"
+fi
+
+echo "*** Using $(python3_version) ; $(python_version)"
 echo "*** Using $(openssl_version)"
 
 if !(which virtualenv); then

script/test/all

@@ -24,7 +24,7 @@ fi
 
 
 BUILD_NUMBER=${BUILD_NUMBER-$USER}
-PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py34}
+PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py36}
 
 for version in $DOCKER_VERSIONS; do
   >&2 echo "Running tests against Docker $version"

setup.py

@@ -99,5 +99,6 @@ setup(
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.6',
     ],
 )

tests/unit/cli/docker_client_test.py

@@ -22,7 +22,10 @@ class DockerClientTestCase(unittest.TestCase):
 
     def test_docker_client_no_home(self):
         with mock.patch.dict(os.environ):
-            del os.environ['HOME']
+            try:
+                del os.environ['HOME']
+            except KeyError:
+                pass
             docker_client(os.environ)
 
     @mock.patch.dict(os.environ)

tox.ini

@@ -1,8 +1,9 @@
 [tox]
-envlist = py27,py34,pre-commit
+envlist = py27,py36,pre-commit
 
 [testenv]
 usedevelop=True
+whitelist_externals=mkdir
 passenv =
     LD_LIBRARY_PATH
     DOCKER_HOST
@@ -17,6 +18,7 @@ deps =
     -rrequirements.txt
     -rrequirements-dev.txt
 commands =
+    mkdir -p .coverage-binfiles
     py.test -v \
         --cov=compose \
         --cov-report html \
@@ -35,6 +37,7 @@ commands =
 # Coverage configuration
 [run]
 branch = True
+data_file = .coverage-binfiles/.coverage
 
 [report]
 show_missing = true