Merge pull request #2635 from aanand/use-networking

Use networking for version 2 Compose files

compose/cli/command.py

@@ -13,6 +13,7 @@ from requests.exceptions import SSLError
 from . import errors
 from . import verbose_proxy
 from .. import config
+from ..const import API_VERSIONS
 from ..project import Project
 from .docker_client import docker_client
 from .utils import call_silently
@@ -49,8 +50,6 @@ def project_from_options(base_dir, options):
         get_config_path_from_options(options),
         project_name=options.get('--project-name'),
         verbose=options.get('--verbose'),
-        use_networking=options.get('--x-networking'),
-        network_driver=options.get('--x-network-driver'),
     )
 
 
@@ -75,18 +74,17 @@ def get_client(verbose=False, version=None):
     return client
 
 
-def get_project(base_dir, config_path=None, project_name=None, verbose=False,
+def get_project(base_dir, config_path=None, project_name=None, verbose=False):
-                use_networking=False, network_driver=None):
     config_details = config.find(base_dir, config_path)
+    project_name = get_project_name(config_details.working_dir, project_name)
+    config_data = config.load(config_details)
 
-    api_version = '1.21' if use_networking else None
+    api_version = os.environ.get(
-    return Project.from_config(
+        'COMPOSE_API_VERSION',
-        get_project_name(config_details.working_dir, project_name),
+        API_VERSIONS[config_data.version])
-        config.load(config_details),
+    client = get_client(verbose=verbose, version=api_version)
-        get_client(verbose=verbose, version=api_version),
+
-        use_networking=use_networking,
+    return Project.from_config(project_name, config_data, client)
-        network_driver=network_driver
-    )
 
 
 def get_project_name(working_dir, project_name=None):

compose/cli/docker_client.py

@@ -11,8 +11,6 @@ from ..const import HTTP_TIMEOUT
 
 log = logging.getLogger(__name__)
 
-DEFAULT_API_VERSION = '1.21'
-
 
 def docker_client(version=None):
     """
@@ -23,8 +21,7 @@ def docker_client(version=None):
         log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.')
 
     kwargs = kwargs_from_env(assert_hostname=False)
-    kwargs['version'] = version or os.environ.get(
+    if version:
-        'COMPOSE_API_VERSION',
+        kwargs['version'] = version
-        DEFAULT_API_VERSION)
     kwargs['timeout'] = HTTP_TIMEOUT
     return Client(**kwargs)

compose/cli/main.py

@@ -122,10 +122,6 @@ class TopLevelCommand(DocoptCommand):
     Options:
       -f, --file FILE           Specify an alternate compose file (default: docker-compose.yml)
       -p, --project-name NAME   Specify an alternate project name (default: directory name)
-      --x-networking            (EXPERIMENTAL) Use new Docker networking functionality.
-                                Requires Docker 1.9 or later.
-      --x-network-driver DRIVER (EXPERIMENTAL) Specify a network driver (default: "bridge").
-                                Requires Docker 1.9 or later.
       --verbose                 Show more output
       -v, --version             Print version and exit
 

compose/config/service_schema_v2.json

@@ -66,12 +66,10 @@
         },
 
         "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
-        "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
         "hostname": {"type": "string"},
         "image": {"type": "string"},
         "ipc": {"type": "string"},
         "labels": {"$ref": "#/definitions/list_or_dict"},
-        "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
 
         "logging": {
             "type": "object",

compose/const.py

@@ -15,3 +15,10 @@ LABEL_SERVICE = 'com.docker.compose.service'
 LABEL_VERSION = 'com.docker.compose.version'
 LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'
 COMPOSEFILE_VERSIONS = (1, 2)
+
+API_VERSIONS = {
+    1: '1.21',
+
+    # TODO: update to 1.22 when there's a Docker 1.10 build to test against
+    2: '1.21',
+}

compose/project.py

@@ -48,11 +48,12 @@ class Project(object):
         ]
 
     @classmethod
-    def from_config(cls, name, config_data, client, use_networking=False, network_driver=None):
+    def from_config(cls, name, config_data, client):
         """
         Construct a Project from a config.Config object.
         """
-        project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver)
+        use_networking = (config_data.version and config_data.version >= 2)
+        project = cls(name, [], client, use_networking=use_networking)
 
         if use_networking:
             remove_links(config_data.services)
@@ -185,7 +186,7 @@ class Project(object):
         net = service_dict.pop('net', None)
         if not net:
             if self.use_networking:
-                return Net(self.name)
+                return Net(self.default_network_name)
             return Net(None)
 
         net_name = get_service_name_from_net(net)
@@ -383,7 +384,7 @@ class Project(object):
 
     def get_network(self):
         try:
-            return self.client.inspect_network(self.name)
+            return self.client.inspect_network(self.default_network_name)
         except NotFound:
             return None
 
@@ -396,9 +397,9 @@ class Project(object):
 
             log.info(
                 'Creating network "{}" with {}'
-                .format(self.name, driver_name)
+                .format(self.default_network_name, driver_name)
             )
-            self.client.create_network(self.name, driver=self.network_driver)
+            self.client.create_network(self.default_network_name, driver=self.network_driver)
 
     def remove_network(self):
         network = self.get_network()
@@ -406,7 +407,11 @@ class Project(object):
             self.client.remove_network(network['Id'])
 
     def uses_default_network(self):
-        return any(service.net.mode == self.name for service in self.services)
+        return any(service.net.mode == self.default_network_name for service in self.services)
+
+    @property
+    def default_network_name(self):
+        return '{}_default'.format(self.name)
 
     def _inject_deps(self, acc, service):
         dep_names = service.get_dependency_names()

contrib/completion/bash/docker-compose

@@ -116,15 +116,11 @@ _docker_compose_docker_compose() {
 		--project-name|-p)
 			return
 			;;
-		--x-network-driver)
-			COMPREPLY=( $( compgen -W "bridge host none overlay" -- "$cur" ) )
-			return
-			;;
 	esac
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v --x-networking --x-network-driver" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v" -- "$cur" ) )
 			;;
 		*)
 			COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) )
@@ -416,9 +412,6 @@ _docker_compose() {
 				(( counter++ ))
 				compose_project="${words[$counter]}"
 				;;
-			--x-network-driver)
-				(( counter++ ))
-				;;
 			-*)
 				;;
 			*)

contrib/completion/zsh/_docker-compose

@@ -332,8 +332,6 @@ _docker-compose() {
         '(- :)'{-v,--version}'[Print version and exit]' \
         '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \
         '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \
-        '--x-networking[(EXPERIMENTAL) Use new Docker networking functionality. Requires Docker 1.9 or later.]' \
-        '--x-network-driver[(EXPERIMENTAL) Specify a network driver (default: "bridge"). Requires Docker 1.9 or later.]:Network Driver:(bridge host none overlay)' \
         '(-): :->command' \
         '(-)*:: :->option-or-argument' && ret=0
 

tests/acceptance/cli_test.py

@@ -16,7 +16,6 @@ from docker import errors
 
 from .. import mock
 from compose.cli.command import get_project
-from compose.cli.docker_client import docker_client
 from compose.container import Container
 from tests.integration.testcases import DockerClientTestCase
 from tests.integration.testcases import get_links
@@ -336,13 +335,10 @@ class CLITestCase(DockerClientTestCase):
         assert 'another_1 | another' in result.stdout
 
     def test_up_without_networking(self):
-        self.require_api_version('1.21')
-
         self.base_dir = 'tests/fixtures/links-composefile'
         self.dispatch(['up', '-d'], None)
-        client = docker_client(version='1.21')
 
-        networks = client.networks(names=[self.project.name])
+        networks = self.client.networks(names=[self.project.default_network_name])
         self.assertEqual(len(networks), 0)
 
         for service in self.project.get_services():
@@ -354,21 +350,18 @@ class CLITestCase(DockerClientTestCase):
         self.assertTrue(web_container.get('HostConfig.Links'))
 
     def test_up_with_networking(self):
-        self.require_api_version('1.21')
+        self.base_dir = 'tests/fixtures/v2-simple'
-
+        self.dispatch(['up', '-d'], None)
-        self.base_dir = 'tests/fixtures/links-composefile'
-        self.dispatch(['--x-networking', 'up', '-d'], None)
-        client = docker_client(version='1.21')
 
         services = self.project.get_services()
 
-        networks = client.networks(names=[self.project.name])
+        networks = self.client.networks(names=[self.project.default_network_name])
         for n in networks:
-            self.addCleanup(client.remove_network, n['Id'])
+            self.addCleanup(self.client.remove_network, n['Id'])
         self.assertEqual(len(networks), 1)
         self.assertEqual(networks[0]['Driver'], 'bridge')
 
-        network = client.inspect_network(networks[0]['Id'])
+        network = self.client.inspect_network(networks[0]['Id'])
         self.assertEqual(len(network['Containers']), len(services))
 
         for service in services:
@@ -376,10 +369,21 @@ class CLITestCase(DockerClientTestCase):
             self.assertEqual(len(containers), 1)
             self.assertIn(containers[0].id, network['Containers'])
 
-        web_container = self.project.get_service('web').containers()[0]
+        web_container = self.project.get_service('simple').containers()[0]
         self.assertFalse(web_container.get('HostConfig.Links'))
 
-    def test_up_with_links(self):
+    def test_up_with_links_is_invalid(self):
+        self.base_dir = 'tests/fixtures/v2-simple'
+
+        result = self.dispatch(
+            ['-f', 'links-invalid.yml', 'up', '-d'],
+            returncode=1)
+
+        # TODO: fix validation error messages for v2 files
+        # assert "Unsupported config option for service 'simple': 'links'" in result.stderr
+        assert "Unsupported config option" in result.stderr
+
+    def test_up_with_links_v1(self):
         self.base_dir = 'tests/fixtures/links-composefile'
         self.dispatch(['up', '-d', 'web'], None)
         web = self.project.get_service('web')
@@ -652,15 +656,13 @@ class CLITestCase(DockerClientTestCase):
         self.assertEqual(container.name, name)
 
     def test_run_with_networking(self):
-        self.require_api_version('1.21')
+        self.base_dir = 'tests/fixtures/v2-simple'
-        client = docker_client(version='1.21')
+        self.dispatch(['run', 'simple', 'true'], None)
-        self.base_dir = 'tests/fixtures/simple-dockerfile'
-        self.dispatch(['--x-networking', 'run', 'simple', 'true'], None)
         service = self.project.get_service('simple')
         container, = service.containers(stopped=True, one_off=True)
-        networks = client.networks(names=[self.project.name])
+        networks = self.client.networks(names=[self.project.default_network_name])
         for n in networks:
-            self.addCleanup(client.remove_network, n['Id'])
+            self.addCleanup(self.client.remove_network, n['Id'])
         self.assertEqual(len(networks), 1)
         self.assertEqual(container.human_readable_command, u'true')
 

tests/fixtures/v2-simple/docker-compose.yml

@@ -0,0 +1,8 @@
+version: 2
+services:
+  simple:
+    image: busybox:latest
+    command: top
+  another:
+    image: busybox:latest
+    command: top

tests/fixtures/v2-simple/links-invalid.yml

@@ -0,0 +1,10 @@
+version: 2
+services:
+  simple:
+    image: busybox:latest
+    command: top
+    links:
+      - another
+  another:
+    image: busybox:latest
+    command: top

tests/integration/project_test.py

@@ -6,7 +6,6 @@ import random
 import py
 
 from .testcases import DockerClientTestCase
-from compose.cli.docker_client import docker_client
 from compose.config import config
 from compose.config.types import VolumeFromSpec
 from compose.config.types import VolumeSpec
@@ -105,20 +104,18 @@ class ProjectTest(DockerClientTestCase):
         self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw'])
 
     def test_get_network_does_not_exist(self):
-        self.require_api_version('1.21')
+        project = Project('composetest', [], self.client)
-        client = docker_client(version='1.21')
-
-        project = Project('composetest', [], client)
         assert project.get_network() is None
 
     def test_get_network(self):
-        self.require_api_version('1.21')
+        project_name = 'network_does_exist'
-        client = docker_client(version='1.21')
+        network_name = '{}_default'.format(project_name)
 
-        network_name = 'network_does_exist'
+        project = Project(project_name, [], self.client)
-        project = Project(network_name, [], client)
+        self.client.create_network(network_name)
-        client.create_network(network_name)
+        self.addCleanup(self.client.remove_network, network_name)
-        self.addCleanup(client.remove_network, network_name)
+
+        assert isinstance(project.get_network(), dict)
         assert project.get_network()['Name'] == network_name
 
     def test_net_from_service(self):
@@ -476,15 +473,13 @@ class ProjectTest(DockerClientTestCase):
         self.assertEqual(len(project.get_service('console').containers()), 0)
 
     def test_project_up_with_custom_network(self):
-        self.require_api_version('1.21')
-        client = docker_client(version='1.21')
         network_name = 'composetest-custom'
 
-        client.create_network(network_name)
+        self.client.create_network(network_name)
-        self.addCleanup(client.remove_network, network_name)
+        self.addCleanup(self.client.remove_network, network_name)
 
         web = self.create_service('web', net=Net(network_name))
-        project = Project('composetest', [web], client, use_networking=True)
+        project = Project('composetest', [web], self.client, use_networking=True)
         project.up()
 
         assert project.get_network() is None

tests/integration/service_test.py

@@ -718,8 +718,6 @@ class ServiceTest(DockerClientTestCase):
         """Test that calling scale on a service that has a custom container name
         results in warning output.
         """
-        # Disable this test against earlier versions because it is flaky
-        self.require_api_version('1.21')
         service = self.create_service('app', container_name='custom-container')
         self.assertEqual(service.custom_container_name(), 'custom-container')
 

tests/integration/testcases.py

@@ -36,14 +36,21 @@ class DockerClientTestCase(unittest.TestCase):
                 all=True,
                 filters={'label': '%s=composetest' % LABEL_PROJECT}):
             self.client.remove_container(c['Id'], force=True)
+
         for i in self.client.images(
                 filters={'label': 'com.docker.compose.test_image'}):
             self.client.remove_image(i)
+
         volumes = self.client.volumes().get('Volumes') or []
         for v in volumes:
             if 'composetest_' in v['Name']:
                 self.client.remove_volume(v['Name'])
 
+        networks = self.client.networks()
+        for n in networks:
+            if 'composetest_' in n['Name']:
+                self.client.remove_network(n['Name'])
+
     def create_service(self, name, **kwargs):
         if 'image' not in kwargs and 'build' not in kwargs:
             kwargs['image'] = 'busybox:latest'

tests/unit/config/config_test.py

@@ -268,8 +268,8 @@ class ConfigTest(unittest.TestCase):
             {
                 'name': 'web',
                 'build': os.path.abspath('/'),
-                'links': ['db'],
                 'volumes': [VolumeSpec.parse('/home/user/project:/code')],
+                'links': ['db'],
             },
             {
                 'name': 'db',
@@ -405,7 +405,6 @@ class ConfigTest(unittest.TestCase):
                 'services': {
                     'web': {
                         'image': 'example/web',
-                        'links': ['db'],
                     },
                     'db': {
                         'image': 'example/db',
@@ -431,7 +430,6 @@ class ConfigTest(unittest.TestCase):
                 'name': 'web',
                 'build': os.path.abspath('/'),
                 'image': 'example/web',
-                'links': ['db'],
                 'volumes': [VolumeSpec.parse('/home/user/project:/code')],
             },
             {

tests/unit/project_test.py

@@ -39,7 +39,7 @@ class ProjectTest(unittest.TestCase):
         self.assertEqual(project.get_service('db').options['image'], 'busybox:latest')
 
     def test_from_config(self):
-        dicts = Config(None, [
+        config = Config(None, [
             {
                 'name': 'web',
                 'image': 'busybox:latest',
@@ -49,12 +49,28 @@ class ProjectTest(unittest.TestCase):
                 'image': 'busybox:latest',
             },
         ], None)
-        project = Project.from_config('composetest', dicts, None)
+        project = Project.from_config('composetest', config, None)
         self.assertEqual(len(project.services), 2)
         self.assertEqual(project.get_service('web').name, 'web')
         self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
         self.assertEqual(project.get_service('db').name, 'db')
         self.assertEqual(project.get_service('db').options['image'], 'busybox:latest')
+        self.assertFalse(project.use_networking)
+
+    def test_from_config_v2(self):
+        config = Config(2, [
+            {
+                'name': 'web',
+                'image': 'busybox:latest',
+            },
+            {
+                'name': 'db',
+                'image': 'busybox:latest',
+            },
+        ], None)
+        project = Project.from_config('composetest', config, None)
+        self.assertEqual(len(project.services), 2)
+        self.assertTrue(project.use_networking)
 
     def test_get_service(self):
         web = Service(
@@ -346,7 +362,7 @@ class ProjectTest(unittest.TestCase):
         self.assertEqual(service.net.mode, 'container:' + container_name)
 
     def test_uses_default_network_true(self):
-        web = Service('web', project='test', image="alpine", net=Net('test'))
+        web = Service('web', project='test', image="alpine", net=Net('test_default'))
         db = Service('web', project='test', image="alpine", net=Net('other'))
         project = Project('test', [web, db], None)
         assert project.uses_default_network()