Merge pull request #2564 from aanand/2478-networks-in-compose-file

Declare networks in Compose file

compose/cli/main.py

@@ -696,8 +696,7 @@ def run_one_off_container(container_options, project, service, options):
                 start_deps=True,
                 strategy=ConvergenceStrategy.never)
 
-    if project.use_networking:
+    project.initialize_networks()
-        project.ensure_network_exists()
 
     container = service.create_container(
         quiet=True,
@@ -705,7 +704,7 @@ def run_one_off_container(container_options, project, service, options):
         **container_options)
 
     if options['-d']:
-        container.start()
+        service.start_container(container)
         print(container.name)
         return
 
@@ -717,6 +716,7 @@ def run_one_off_container(container_options, project, service, options):
     try:
         try:
             dockerpty.start(project.client, container.id, interactive=not options['-T'])
+            service.connect_container_to_networks(container)
             exit_code = container.wait()
         except signals.ShutdownException:
             project.client.stop(container.id)

compose/config/config.py

@@ -139,14 +139,16 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
         return {} if self.version == 1 else self.config.get('volumes', {})
 
 
-class Config(namedtuple('_Config', 'version services volumes')):
+class Config(namedtuple('_Config', 'version services volumes networks')):
     """
     :param version: configuration version
     :type  version: int
     :param services: List of service description dictionaries
     :type  services: :class:`list`
-    :param volumes: List of volume description dictionaries
+    :param volumes: Dictionary mapping volume names to description dictionaries
-    :type  volumes: :class:`list`
+    :type  volumes: :class:`dict`
+    :param networks: Dictionary mapping network names to description dictionaries
+    :type  networks: :class:`dict`
     """
 
 
@@ -256,39 +258,44 @@ def load(config_details):
     config_details = config_details._replace(config_files=processed_files)
 
     main_file = config_details.config_files[0]
-    volumes = load_volumes(config_details.config_files)
+    volumes = load_mapping(config_details.config_files, 'volumes', 'Volume')
+    networks = load_mapping(config_details.config_files, 'networks', 'Network')
     service_dicts = load_services(
         config_details.working_dir,
         main_file.filename,
         [file.get_service_dicts() for file in config_details.config_files],
         main_file.version)
-    return Config(main_file.version, service_dicts, volumes)
+    return Config(main_file.version, service_dicts, volumes, networks)
 
 
-def load_volumes(config_files):
+def load_mapping(config_files, key, entity_type):
-    volumes = {}
+    mapping = {}
+
     for config_file in config_files:
-        for name, volume_config in config_file.get_volumes().items():
+        for name, config in config_file.config.get(key, {}).items():
-            volumes[name] = volume_config or {}
+            mapping[name] = config or {}
-            if not volume_config:
+            if not config:
                 continue
 
-            external = volume_config.get('external')
+            external = config.get('external')
             if external:
-                if len(volume_config.keys()) > 1:
+                if len(config.keys()) > 1:
                     raise ConfigurationError(
-                        'Volume {0} declared as external but specifies'
+                        '{} {} declared as external but specifies'
-                        ' additional attributes ({1}). '.format(
+                        ' additional attributes ({}). '.format(
+                            entity_type,
                             name,
-                            ', '.join([k for k in volume_config.keys() if k != 'external'])
+                            ', '.join([k for k in config.keys() if k != 'external'])
                         )
                     )
                 if isinstance(external, dict):
-                    volume_config['external_name'] = external.get('name')
+                    config['external_name'] = external.get('name')
                 else:
-                    volume_config['external_name'] = name
+                    config['external_name'] = name
 
-    return volumes
+            mapping[name] = config
+
+    return mapping
 
 
 def load_services(working_dir, filename, service_configs, version):

compose/config/fields_schema_v2.json

@@ -17,6 +17,15 @@
       },
       "additionalProperties": false
     },
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
     "volumes": {
       "id": "#/properties/volumes",
       "type": "object",
@@ -30,6 +39,10 @@
   },
 
   "definitions": {
+    "network": {
+      "id": "#/definitions/network",
+      "type": "object"
+    },
     "volume": {
       "id": "#/definitions/volume",
       "type": ["object", "null"],

compose/config/service_schema_v2.json

@@ -89,7 +89,13 @@
         "mac_address": {"type": "string"},
         "mem_limit": {"type": ["number", "string"]},
         "memswap_limit": {"type": ["number", "string"]},
-        "net": {"type": "string"},
+
+        "networks": {
+          "type": "array",
+          "items": {"type": "string"},
+          "uniqueItems": true
+        },
+
         "pid": {"type": ["string", "null"]},
 
         "ports": {

compose/network.py

@@ -0,0 +1,79 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import logging
+
+from docker.errors import NotFound
+
+from .config import ConfigurationError
+
+
+log = logging.getLogger(__name__)
+
+
+class Network(object):
+    def __init__(self, client, project, name, driver=None, driver_opts=None,
+                 external_name=None):
+        self.client = client
+        self.project = project
+        self.name = name
+        self.driver = driver
+        self.driver_opts = driver_opts
+        self.external_name = external_name
+
+    def ensure(self):
+        if self.external_name:
+            try:
+                self.inspect()
+                log.debug(
+                    'Network {0} declared as external. No new '
+                    'network will be created.'.format(self.name)
+                )
+            except NotFound:
+                raise ConfigurationError(
+                    'Network {name} declared as external, but could'
+                    ' not be found. Please create the network manually'
+                    ' using `{command} {name}` and try again.'.format(
+                        name=self.external_name,
+                        command='docker network create'
+                    )
+                )
+            return
+
+        try:
+            data = self.inspect()
+            if self.driver and data['Driver'] != self.driver:
+                raise ConfigurationError(
+                    'Network {} needs to be recreated - driver has changed'
+                    .format(self.full_name))
+            if data['Options'] != (self.driver_opts or {}):
+                raise ConfigurationError(
+                    'Network {} needs to be recreated - options have changed'
+                    .format(self.full_name))
+        except NotFound:
+            driver_name = 'the default driver'
+            if self.driver:
+                driver_name = 'driver "{}"'.format(self.driver)
+
+            log.info(
+                'Creating network "{}" with {}'
+                .format(self.full_name, driver_name)
+            )
+
+            self.client.create_network(
+                self.full_name, self.driver, self.driver_opts
+            )
+
+    def remove(self):
+        # TODO: don't remove external networks
+        log.info("Removing network {}".format(self.full_name))
+        self.client.remove_network(self.full_name)
+
+    def inspect(self):
+        return self.client.inspect_network(self.full_name)
+
+    @property
+    def full_name(self):
+        if self.external_name:
+            return self.external_name
+        return '{0}_{1}'.format(self.project, self.name)

compose/project.py

@@ -17,6 +17,7 @@ from .const import LABEL_ONE_OFF
 from .const import LABEL_PROJECT
 from .const import LABEL_SERVICE
 from .container import Container
+from .network import Network
 from .service import ContainerNet
 from .service import ConvergenceStrategy
 from .service import Net
@@ -33,12 +34,14 @@ class Project(object):
     """
     A collection of services.
     """
-    def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None):
+    def __init__(self, name, services, client, networks=None, volumes=None,
+                 use_networking=False, network_driver=None):
         self.name = name
         self.services = services
         self.client = client
         self.use_networking = use_networking
         self.network_driver = network_driver
+        self.networks = networks or []
         self.volumes = volumes or []
 
     def labels(self, one_off=False):
@@ -55,23 +58,47 @@ class Project(object):
         use_networking = (config_data.version and config_data.version >= 2)
         project = cls(name, [], client, use_networking=use_networking)
 
-        if use_networking:
+        custom_networks = []
-            remove_links(config_data.services)
+        if config_data.networks:
+            for network_name, data in config_data.networks.items():
+                custom_networks.append(
+                    Network(
+                        client=client, project=name, name=network_name,
+                        driver=data.get('driver'),
+                        driver_opts=data.get('driver_opts'),
+                        external_name=data.get('external_name'),
+                    )
+                )
 
         for service_dict in config_data.services:
-            links = project.get_links(service_dict)
+            if use_networking:
-            volumes_from = get_volumes_from(project, service_dict)
+                networks = get_networks(
+                    service_dict,
+                    custom_networks + [project.default_network])
+                net = Net(networks[0]) if networks else Net("none")
+                links = []
+            else:
+                networks = []
                 net = project.get_net(service_dict)
+                links = project.get_links(service_dict)
+
+            volumes_from = get_volumes_from(project, service_dict)
 
             project.services.append(
                 Service(
                     client=client,
                     project=name,
                     use_networking=use_networking,
+                    networks=networks,
                     links=links,
                     net=net,
                     volumes_from=volumes_from,
                     **service_dict))
+
+        project.networks += custom_networks
+        if project.uses_default_network():
+            project.networks.append(project.default_network)
+
         if config_data.volumes:
             for vol_name, data in config_data.volumes.items():
                 project.volumes.append(
@@ -82,6 +109,7 @@ class Project(object):
                         external_name=data.get('external_name')
                     )
                 )
+
         return project
 
     @property
@@ -124,11 +152,8 @@ class Project(object):
         Raises NoSuchService if any of the named services do not exist.
         """
         if service_names is None or len(service_names) == 0:
-            return self.get_services(
+            service_names = self.service_names
-                service_names=self.service_names,
+
-                include_deps=include_deps
-            )
-        else:
         unsorted = [self.get_service(name) for name in service_names]
         services = [s for s in self.services if s in unsorted]
 
@@ -137,6 +162,7 @@ class Project(object):
 
         uniques = []
         [uniques.append(s) for s in services if s not in uniques]
+
         return uniques
 
     def get_services_without_duplicate(self, service_names=None, include_deps=False):
@@ -165,8 +191,6 @@ class Project(object):
     def get_net(self, service_dict):
         net = service_dict.pop('net', None)
         if not net:
-            if self.use_networking:
-                return Net(self.default_network_name)
             return Net(None)
 
         net_name = get_service_name_from_net(net)
@@ -251,7 +275,7 @@ class Project(object):
     def down(self, remove_image_type, include_volumes):
         self.stop()
         self.remove_stopped(v=include_volumes)
-        self.remove_network()
+        self.remove_default_network()
 
         if include_volumes:
             self.remove_volumes()
@@ -262,10 +286,33 @@ class Project(object):
         for service in self.get_services():
             service.remove_image(remove_image_type)
 
+    def remove_default_network(self):
+        if not self.use_networking:
+            return
+        if self.uses_default_network():
+            self.default_network.remove()
+
     def remove_volumes(self):
         for volume in self.volumes:
             volume.remove()
 
+    def initialize_networks(self):
+        if not self.use_networking:
+            return
+
+        for network in self.networks:
+            network.ensure()
+
+    def uses_default_network(self):
+        return any(
+            self.default_network.full_name in service.networks
+            for service in self.services
+        )
+
+    @property
+    def default_network(self):
+        return Network(client=self.client, project=self.name, name='default')
+
     def restart(self, service_names=None, **options):
         containers = self.containers(service_names, stopped=True)
         parallel.parallel_restart(containers, options)
@@ -335,9 +382,7 @@ class Project(object):
 
         plans = self._get_convergence_plans(services, strategy)
 
-        if self.use_networking and self.uses_default_network():
+        self.initialize_networks()
-            self.ensure_network_exists()
-
         self.initialize_volumes()
 
         return [
@@ -395,40 +440,6 @@ class Project(object):
 
         return [c for c in containers if matches_service_names(c)]
 
-    def get_network(self):
-        try:
-            return self.client.inspect_network(self.default_network_name)
-        except NotFound:
-            return None
-
-    def ensure_network_exists(self):
-        # TODO: recreate network if driver has changed?
-        if self.get_network() is None:
-            driver_name = 'the default driver'
-            if self.network_driver:
-                driver_name = 'driver "{}"'.format(self.network_driver)
-
-            log.info(
-                'Creating network "{}" with {}'
-                .format(self.default_network_name, driver_name)
-            )
-            self.client.create_network(self.default_network_name, driver=self.network_driver)
-
-    def remove_network(self):
-        if not self.use_networking:
-            return
-        network = self.get_network()
-        if network:
-            log.info("Removing network %s", self.default_network_name)
-            self.client.remove_network(network['Id'])
-
-    def uses_default_network(self):
-        return any(service.net.mode == self.default_network_name for service in self.services)
-
-    @property
-    def default_network_name(self):
-        return '{}_default'.format(self.name)
-
     def _inject_deps(self, acc, service):
         dep_names = service.get_dependency_names()
 
@@ -444,24 +455,20 @@ class Project(object):
         return acc + dep_services
 
 
-def remove_links(service_dicts):
+def get_networks(service_dict, network_definitions):
-    services_with_links = [s for s in service_dicts if 'links' in s]
+    networks = []
-    if not services_with_links:
+    for name in service_dict.pop('networks', ['default']):
-        return
+        if name in ['bridge', 'host']:
-
+            networks.append(name)
-    if len(services_with_links) == 1:
-        prefix = '"{}" defines'.format(services_with_links[0]['name'])
         else:
-        prefix = 'Some services ({}) define'.format(
+            matches = [n for n in network_definitions if n.name == name]
-            ", ".join('"{}"'.format(s['name']) for s in services_with_links))
+            if matches:
-
+                networks.append(matches[0].full_name)
-    log.warn(
+            else:
-        '\n{} links, which are not compatible with Docker networking and will be ignored.\n'
+                raise ConfigurationError(
-        'Future versions of Docker will not support links - you should remove them for '
+                    'Service "{}" uses an undefined network "{}"'
-        'forwards-compatibility.\n'.format(prefix))
+                    .format(service_dict['name'], name))
-
+    return networks
-    for s in services_with_links:
-        del s['links']
 
 
 def get_volumes_from(project, service_dict):

compose/service.py

@@ -116,6 +116,7 @@ class Service(object):
         links=None,
         volumes_from=None,
         net=None,
+        networks=None,
         **options
     ):
         self.name = name
@@ -125,6 +126,7 @@ class Service(object):
         self.links = links or []
         self.volumes_from = volumes_from or []
         self.net = net or Net(None)
+        self.networks = networks or []
         self.options = options
 
     def containers(self, stopped=False, one_off=False, filters={}):
@@ -175,7 +177,7 @@ class Service(object):
 
         def create_and_start(service, number):
             container = service.create_container(number=number, quiet=True)
-            container.start()
+            service.start_container(container)
             return container
 
         running_containers = self.containers(stopped=False)
@@ -348,7 +350,7 @@ class Service(object):
                 container.attach_log_stream()
 
             if start:
-                container.start()
+                self.start_container(container)
 
             return [container]
 
@@ -406,7 +408,7 @@ class Service(object):
         if attach_logs:
             new_container.attach_log_stream()
         if start_new_container:
-            new_container.start()
+            self.start_container(new_container)
         container.remove()
         return new_container
 
@@ -415,9 +417,18 @@ class Service(object):
             log.info("Starting %s" % container.name)
             if attach_logs:
                 container.attach_log_stream()
+            return self.start_container(container)
+
+    def start_container(self, container):
         container.start()
+        self.connect_container_to_networks(container)
         return container
 
+    def connect_container_to_networks(self, container):
+        for network in self.networks:
+            log.debug('Connecting "{}" to "{}"'.format(container.name, network))
+            self.client.connect_container_to_network(container.id, network)
+
     def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT):
         for c in self.duplicate_containers():
             log.info('Removing %s' % c.name)

tests/acceptance/cli_test.py

@@ -103,8 +103,15 @@ class CLITestCase(DockerClientTestCase):
         if self.base_dir:
             self.project.kill()
             self.project.remove_stopped()
+
             for container in self.project.containers(stopped=True, one_off=True):
                 container.remove(force=True)
+
+            networks = self.client.networks()
+            for n in networks:
+                if n['Name'].startswith('{}_'.format(self.project.name)):
+                    self.client.remove_network(n['Name'])
+
         super(CLITestCase, self).tearDown()
 
     @property
@@ -119,6 +126,20 @@ class CLITestCase(DockerClientTestCase):
         proc = start_process(self.base_dir, project_options + options)
         return wait_on_process(proc, returncode=returncode)
 
+    def execute(self, container, cmd):
+        # Remove once Hijack and CloseNotifier sign a peace treaty
+        self.client.close()
+        exc = self.client.exec_create(container.id, cmd)
+        self.client.exec_start(exc)
+        return self.client.exec_inspect(exc)['ExitCode']
+
+    def lookup(self, container, service_name):
+        exit_code = self.execute(container, [
+            "nslookup",
+            "{}_{}_1".format(self.project.name, service_name)
+        ])
+        return exit_code == 0
+
     def test_help(self):
         self.base_dir = 'tests/fixtures/no-composefile'
         result = self.dispatch(['help', 'up'], returncode=1)
@@ -350,43 +371,127 @@ class CLITestCase(DockerClientTestCase):
         assert 'simple_1  | simple' in result.stdout
         assert 'another_1 | another' in result.stdout
 
-    def test_up_without_networking(self):
+    def test_up(self):
-        self.base_dir = 'tests/fixtures/links-composefile'
-        self.dispatch(['up', '-d'], None)
-
-        networks = self.client.networks(names=[self.project.default_network_name])
-        self.assertEqual(len(networks), 0)
-
-        for service in self.project.get_services():
-            containers = service.containers()
-            self.assertEqual(len(containers), 1)
-            self.assertNotEqual(containers[0].get('Config.Hostname'), service.name)
-
-        web_container = self.project.get_service('web').containers()[0]
-        self.assertTrue(web_container.get('HostConfig.Links'))
-
-    def test_up_with_networking(self):
         self.base_dir = 'tests/fixtures/v2-simple'
         self.dispatch(['up', '-d'], None)
 
         services = self.project.get_services()
 
-        networks = self.client.networks(names=[self.project.default_network_name])
+        networks = self.client.networks(names=[self.project.default_network.full_name])
-        for n in networks:
-            self.addCleanup(self.client.remove_network, n['Id'])
         self.assertEqual(len(networks), 1)
         self.assertEqual(networks[0]['Driver'], 'bridge')
 
         network = self.client.inspect_network(networks[0]['Id'])
-        self.assertEqual(len(network['Containers']), len(services))
 
         for service in services:
             containers = service.containers()
             self.assertEqual(len(containers), 1)
-            self.assertIn(containers[0].id, network['Containers'])
 
-        web_container = self.project.get_service('simple').containers()[0]
+            container = containers[0]
-        self.assertFalse(web_container.get('HostConfig.Links'))
+            self.assertIn(container.id, network['Containers'])
+
+            networks = list(container.get('NetworkSettings.Networks'))
+            self.assertEqual(networks, [network['Name']])
+
+    def test_up_with_networks(self):
+        self.base_dir = 'tests/fixtures/networks'
+        self.dispatch(['up', '-d'], None)
+
+        back_name = '{}_back'.format(self.project.name)
+        front_name = '{}_front'.format(self.project.name)
+
+        networks = [
+            n for n in self.client.networks()
+            if n['Name'].startswith('{}_'.format(self.project.name))
+        ]
+
+        # Two networks were created: back and front
+        assert sorted(n['Name'] for n in networks) == [back_name, front_name]
+
+        back_network = [n for n in networks if n['Name'] == back_name][0]
+        front_network = [n for n in networks if n['Name'] == front_name][0]
+
+        web_container = self.project.get_service('web').containers()[0]
+        app_container = self.project.get_service('app').containers()[0]
+        db_container = self.project.get_service('db').containers()[0]
+
+        # db and app joined the back network
+        assert sorted(back_network['Containers']) == sorted([db_container.id, app_container.id])
+
+        # web and app joined the front network
+        assert sorted(front_network['Containers']) == sorted([web_container.id, app_container.id])
+
+        # web can see app but not db
+        assert self.lookup(web_container, "app")
+        assert not self.lookup(web_container, "db")
+
+        # app can see db
+        assert self.lookup(app_container, "db")
+
+    def test_up_missing_network(self):
+        self.base_dir = 'tests/fixtures/networks'
+
+        result = self.dispatch(
+            ['-f', 'missing-network.yml', 'up', '-d'],
+            returncode=1)
+
+        assert 'Service "web" uses an undefined network "foo"' in result.stderr
+
+    def test_up_predefined_networks(self):
+        filename = 'predefined-networks.yml'
+
+        self.base_dir = 'tests/fixtures/networks'
+        self._project = get_project(self.base_dir, [filename])
+
+        self.dispatch(['-f', filename, 'up', '-d'], None)
+
+        networks = [
+            n for n in self.client.networks()
+            if n['Name'].startswith('{}_'.format(self.project.name))
+        ]
+        assert not networks
+
+        for name in ['bridge', 'host', 'none']:
+            container = self.project.get_service(name).containers()[0]
+            assert list(container.get('NetworkSettings.Networks')) == [name]
+            assert container.get('HostConfig.NetworkMode') == name
+
+    def test_up_external_networks(self):
+        filename = 'external-networks.yml'
+
+        self.base_dir = 'tests/fixtures/networks'
+        self._project = get_project(self.base_dir, [filename])
+
+        result = self.dispatch(['-f', filename, 'up', '-d'], returncode=1)
+        assert 'declared as external, but could not be found' in result.stderr
+
+        networks = [
+            n['Name'] for n in self.client.networks()
+            if n['Name'].startswith('{}_'.format(self.project.name))
+        ]
+        assert not networks
+
+        network_names = ['{}_{}'.format(self.project.name, n) for n in ['foo', 'bar']]
+        for name in network_names:
+            self.client.create_network(name)
+
+        self.dispatch(['-f', filename, 'up', '-d'])
+        container = self.project.containers()[0]
+        assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted(network_names)
+
+    def test_up_no_services(self):
+        self.base_dir = 'tests/fixtures/no-services'
+        self.dispatch(['up', '-d'], None)
+
+        network_names = [
+            n['Name'] for n in self.client.networks()
+            if n['Name'].startswith('{}_'.format(self.project.name))
+        ]
+
+        assert sorted(network_names) == [
+            '{}_{}'.format(self.project.name, name)
+            for name in ['bar', 'foo']
+        ]
 
     def test_up_with_links_is_invalid(self):
         self.base_dir = 'tests/fixtures/v2-simple'
@@ -402,13 +507,48 @@ class CLITestCase(DockerClientTestCase):
     def test_up_with_links_v1(self):
         self.base_dir = 'tests/fixtures/links-composefile'
         self.dispatch(['up', '-d', 'web'], None)
+
+        # No network was created
+        networks = self.client.networks(names=[self.project.default_network.full_name])
+        assert networks == []
+
         web = self.project.get_service('web')
         db = self.project.get_service('db')
         console = self.project.get_service('console')
+
+        # console was not started
         self.assertEqual(len(web.containers()), 1)
         self.assertEqual(len(db.containers()), 1)
         self.assertEqual(len(console.containers()), 0)
 
+        # web has links
+        web_container = web.containers()[0]
+        self.assertTrue(web_container.get('HostConfig.Links'))
+
+    def test_up_with_net_is_invalid(self):
+        self.base_dir = 'tests/fixtures/net-container'
+
+        result = self.dispatch(
+            ['-f', 'v2-invalid.yml', 'up', '-d'],
+            returncode=1)
+
+        # TODO: fix validation error messages for v2 files
+        # assert "Unsupported config option for service 'web': 'net'" in exc.exconly()
+        assert "Unsupported config option" in result.stderr
+
+    def test_up_with_net_v1(self):
+        self.base_dir = 'tests/fixtures/net-container'
+        self.dispatch(['up', '-d'], None)
+
+        bar = self.project.get_service('bar')
+        bar_container = bar.containers()[0]
+
+        foo = self.project.get_service('foo')
+        foo_container = foo.containers()[0]
+
+        assert foo_container.get('HostConfig.NetworkMode') == \
+            'container:{}'.format(bar_container.id)
+
     def test_up_with_no_deps(self):
         self.base_dir = 'tests/fixtures/links-composefile'
         self.dispatch(['up', '-d', '--no-deps', 'web'], None)
@@ -698,9 +838,7 @@ class CLITestCase(DockerClientTestCase):
         self.dispatch(['run', 'simple', 'true'], None)
         service = self.project.get_service('simple')
         container, = service.containers(stopped=True, one_off=True)
-        networks = self.client.networks(names=[self.project.default_network_name])
+        networks = self.client.networks(names=[self.project.default_network.full_name])
-        for n in networks:
-            self.addCleanup(self.client.remove_network, n['Id'])
         self.assertEqual(len(networks), 1)
         self.assertEqual(container.human_readable_command, u'true')
 
@@ -858,7 +996,7 @@ class CLITestCase(DockerClientTestCase):
     def test_restart(self):
         service = self.project.get_service('simple')
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         started_at = container.dictionary['State']['StartedAt']
         self.dispatch(['restart', '-t', '1'], None)
         container.inspect()

tests/fixtures/net-container/docker-compose.yml

@@ -0,0 +1,7 @@
+foo:
+  image: busybox
+  command: top
+  net: "container:bar"
+bar:
+  image: busybox
+  command: top

tests/fixtures/net-container/v2-invalid.yml

@@ -0,0 +1,10 @@
+version: 2
+
+services:
+  foo:
+    image: busybox
+    command: top
+  bar:
+    image: busybox
+    command: top
+    net: "container:foo"

tests/fixtures/networks/docker-compose.yml

@@ -0,0 +1,19 @@
+version: 2
+
+services:
+  web:
+    image: busybox
+    command: top
+    networks: ["front"]
+  app:
+    image: busybox
+    command: top
+    networks: ["front", "back"]
+  db:
+    image: busybox
+    command: top
+    networks: ["back"]
+
+networks:
+  front: {}
+  back: {}

tests/fixtures/networks/external-networks.yml

@@ -0,0 +1,16 @@
+version: 2
+
+services:
+  web:
+    image: busybox
+    command: top
+    networks:
+      - networks_foo
+      - bar
+
+networks:
+  networks_foo:
+    external: true
+  bar:
+    external:
+      name: networks_bar

tests/fixtures/networks/missing-network.yml

@@ -0,0 +1,10 @@
+version: 2
+
+services:
+  web:
+    image: busybox
+    command: top
+    networks: ["foo"]
+
+networks:
+  bar: {}

tests/fixtures/networks/predefined-networks.yml

@@ -0,0 +1,17 @@
+version: 2
+
+services:
+  bridge:
+    image: busybox
+    command: top
+    networks: ["bridge"]
+
+  host:
+    image: busybox
+    command: top
+    networks: ["host"]
+
+  none:
+    image: busybox
+    command: top
+    networks: []

tests/fixtures/no-links-composefile/docker-compose.yml

@@ -0,0 +1,9 @@
+db:
+  image: busybox:latest
+  command: top
+web:
+  image: busybox:latest
+  command: top
+console:
+  image: busybox:latest
+  command: top

tests/fixtures/no-services/docker-compose.yml

@@ -0,0 +1,5 @@
+version: 2
+
+networks:
+  foo: {}
+  bar: {}

tests/integration/project_test.py

@@ -14,7 +14,6 @@ from compose.const import LABEL_PROJECT
 from compose.container import Container
 from compose.project import Project
 from compose.service import ConvergenceStrategy
-from compose.service import Net
 
 
 def build_service_dicts(service_config):
@@ -104,21 +103,6 @@ class ProjectTest(DockerClientTestCase):
         db = project.get_service('db')
         self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw'])
 
-    def test_get_network_does_not_exist(self):
-        project = Project('composetest', [], self.client)
-        assert project.get_network() is None
-
-    def test_get_network(self):
-        project_name = 'network_does_exist'
-        network_name = '{}_default'.format(project_name)
-
-        project = Project(project_name, [], self.client)
-        self.client.create_network(network_name)
-        self.addCleanup(self.client.remove_network, network_name)
-
-        assert isinstance(project.get_network(), dict)
-        assert project.get_network()['Name'] == network_name
-
     def test_net_from_service(self):
         project = Project.from_config(
             name='composetest',
@@ -473,18 +457,6 @@ class ProjectTest(DockerClientTestCase):
         self.assertEqual(len(project.get_service('data').containers(stopped=True)), 1)
         self.assertEqual(len(project.get_service('console').containers()), 0)
 
-    def test_project_up_with_custom_network(self):
-        network_name = 'composetest-custom'
-
-        self.client.create_network(network_name)
-        self.addCleanup(self.client.remove_network, network_name)
-
-        web = self.create_service('web', net=Net(network_name))
-        project = Project('composetest', [web], self.client, use_networking=True)
-        project.up()
-
-        assert project.get_network() is None
-
     def test_unscale_after_restart(self):
         web = self.create_service('web')
         project = Project('composetest', [web], self.client)
@@ -510,15 +482,50 @@ class ProjectTest(DockerClientTestCase):
         service = project.get_service('web')
         self.assertEqual(len(service.containers()), 1)
 
+    def test_project_up_networks(self):
+        config_data = config.Config(
+            version=2,
+            services=[{
+                'name': 'web',
+                'image': 'busybox:latest',
+                'command': 'top',
+            }],
+            volumes={},
+            networks={
+                'foo': {'driver': 'bridge'},
+                'bar': {'driver': None},
+                'baz': {},
+            },
+        )
+
+        project = Project.from_config(
+            client=self.client,
+            name='composetest',
+            config_data=config_data,
+        )
+        project.up()
+        self.assertEqual(len(project.containers()), 1)
+
+        for net_name in ['foo', 'bar', 'baz']:
+            full_net_name = 'composetest_{}'.format(net_name)
+            network_data = self.client.inspect_network(full_net_name)
+            self.assertEqual(network_data['Name'], full_net_name)
+
+        foo_data = self.client.inspect_network('composetest_foo')
+        self.assertEqual(foo_data['Driver'], 'bridge')
+
     def test_project_up_volumes(self):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={vol_name: {'driver': 'local'}}
+            }],
+            volumes={vol_name: {'driver': 'local'}},
+            networks={},
         )
 
         project = Project.from_config(
@@ -587,11 +594,14 @@ class ProjectTest(DockerClientTestCase):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={vol_name: {}}
+            }],
+            volumes={vol_name: {}},
+            networks={},
         )
 
         project = Project.from_config(
@@ -608,11 +618,14 @@ class ProjectTest(DockerClientTestCase):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={vol_name: {}}
+            }],
+            volumes={vol_name: {}},
+            networks={},
         )
 
         project = Project.from_config(
@@ -629,11 +642,14 @@ class ProjectTest(DockerClientTestCase):
         vol_name = '{0:x}'.format(random.getrandbits(32))
 
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={vol_name: {'driver': 'foobar'}}
+            }],
+            volumes={vol_name: {'driver': 'foobar'}},
+            networks={},
         )
 
         project = Project.from_config(
@@ -648,11 +664,14 @@ class ProjectTest(DockerClientTestCase):
         full_vol_name = 'composetest_{0}'.format(vol_name)
 
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={vol_name: {'driver': 'local'}}
+            }],
+            volumes={vol_name: {'driver': 'local'}},
+            networks={},
         )
         project = Project.from_config(
             name='composetest',
@@ -683,13 +702,16 @@ class ProjectTest(DockerClientTestCase):
         full_vol_name = 'composetest_{0}'.format(vol_name)
         self.client.create_volume(vol_name)
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={
+            }],
+            volumes={
                 vol_name: {'external': True, 'external_name': vol_name}
-            }
+            },
+            networks=None,
         )
         project = Project.from_config(
             name='composetest',
@@ -704,13 +726,16 @@ class ProjectTest(DockerClientTestCase):
         vol_name = '{0:x}'.format(random.getrandbits(32))
 
         config_data = config.Config(
-            version=2, services=[{
+            version=2,
+            services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'command': 'top'
-            }], volumes={
+            }],
+            volumes={
                 vol_name: {'external': True, 'external_name': vol_name}
-            }
+            },
+            networks=None,
         )
         project = Project.from_config(
             name='composetest',

tests/integration/resilience_test.py

@@ -17,7 +17,7 @@ class ResilienceTest(DockerClientTestCase):
         self.project = Project('composetest', [self.db], self.client)
 
         container = self.db.create_container()
-        container.start()
+        self.db.start_container(container)
         self.host_path = container.get_mount('/var/db')['Source']
 
     def test_successful_recreate(self):
@@ -35,7 +35,7 @@ class ResilienceTest(DockerClientTestCase):
         self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path)
 
     def test_start_failure(self):
-        with mock.patch('compose.container.Container.start', crash):
+        with mock.patch('compose.service.Service.start_container', crash):
             with self.assertRaises(Crash):
                 self.project.up(strategy=ConvergenceStrategy.always)
 

tests/integration/service_test.py

@@ -32,14 +32,7 @@ from compose.service import Service
 
 def create_and_start_container(service, **override_options):
     container = service.create_container(**override_options)
-    container.start()
+    return service.start_container(container)
-    return container
-
-
-def remove_stopped(service):
-    containers = [c for c in service.containers(stopped=True) if not c.is_running]
-    for container in containers:
-        container.remove()
 
 
 class ServiceTest(DockerClientTestCase):
@@ -88,19 +81,19 @@ class ServiceTest(DockerClientTestCase):
     def test_create_container_with_unspecified_volume(self):
         service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')])
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         assert container.get_mount('/var/db')
 
     def test_create_container_with_volume_driver(self):
         service = self.create_service('db', volume_driver='foodriver')
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver'))
 
     def test_create_container_with_cpu_shares(self):
         service = self.create_service('db', cpu_shares=73)
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(container.get('HostConfig.CpuShares'), 73)
 
     def test_create_container_with_cpu_quota(self):
@@ -113,7 +106,7 @@ class ServiceTest(DockerClientTestCase):
         extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229']
         service = self.create_service('db', extra_hosts=extra_hosts)
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts))
 
     def test_create_container_with_extra_hosts_dicts(self):
@@ -121,33 +114,33 @@ class ServiceTest(DockerClientTestCase):
         extra_hosts_list = ['somehost:162.242.195.82', 'otherhost:50.31.209.229']
         service = self.create_service('db', extra_hosts=extra_hosts)
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts_list))
 
     def test_create_container_with_cpu_set(self):
         service = self.create_service('db', cpuset='0')
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(container.get('HostConfig.CpusetCpus'), '0')
 
     def test_create_container_with_read_only_root_fs(self):
         read_only = True
         service = self.create_service('db', read_only=read_only)
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(container.get('HostConfig.ReadonlyRootfs'), read_only, container.get('HostConfig'))
 
     def test_create_container_with_security_opt(self):
         security_opt = ['label:disable']
         service = self.create_service('db', security_opt=security_opt)
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt))
 
     def test_create_container_with_mac_address(self):
         service = self.create_service('db', mac_address='02:42:ac:11:65:43')
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         self.assertEqual(container.inspect()['Config']['MacAddress'], '02:42:ac:11:65:43')
 
     def test_create_container_with_specified_volume(self):
@@ -158,7 +151,7 @@ class ServiceTest(DockerClientTestCase):
             'db',
             volumes=[VolumeSpec(host_path, container_path, 'rw')])
         container = service.create_container()
-        container.start()
+        service.start_container(container)
         assert container.get_mount(container_path)
 
         # Match the last component ("host-path"), because boot2docker symlinks /tmp
@@ -229,7 +222,7 @@ class ServiceTest(DockerClientTestCase):
             ]
         )
         host_container = host_service.create_container()
-        host_container.start()
+        host_service.start_container(host_container)
         self.assertIn(volume_container_1.id + ':rw',
                       host_container.get('HostConfig.VolumesFrom'))
         self.assertIn(volume_container_2.id + ':rw',
@@ -248,7 +241,7 @@ class ServiceTest(DockerClientTestCase):
         self.assertEqual(old_container.get('Config.Cmd'), ['-d', '1'])
         self.assertIn('FOO=1', old_container.get('Config.Env'))
         self.assertEqual(old_container.name, 'composetest_db_1')
-        old_container.start()
+        service.start_container(old_container)
         old_container.inspect()  # reload volume data
         volume_path = old_container.get_mount('/etc')['Source']
 

tests/unit/project_test.py

@@ -12,8 +12,6 @@ from compose.config.types import VolumeFromSpec
 from compose.const import LABEL_SERVICE
 from compose.container import Container
 from compose.project import Project
-from compose.service import ContainerNet
-from compose.service import Net
 from compose.service import Service
 
 
@@ -21,25 +19,10 @@ class ProjectTest(unittest.TestCase):
     def setUp(self):
         self.mock_client = mock.create_autospec(docker.Client)
 
-    def test_from_dict(self):
-        project = Project.from_config('composetest', Config(None, [
-            {
-                'name': 'web',
-                'image': 'busybox:latest'
-            },
-            {
-                'name': 'db',
-                'image': 'busybox:latest'
-            },
-        ], None), None)
-        self.assertEqual(len(project.services), 2)
-        self.assertEqual(project.get_service('web').name, 'web')
-        self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
-        self.assertEqual(project.get_service('db').name, 'db')
-        self.assertEqual(project.get_service('db').options['image'], 'busybox:latest')
-
     def test_from_config(self):
-        config = Config(None, [
+        config = Config(
+            version=None,
+            services=[
                 {
                     'name': 'web',
                     'image': 'busybox:latest',
@@ -48,8 +31,15 @@ class ProjectTest(unittest.TestCase):
                     'name': 'db',
                     'image': 'busybox:latest',
                 },
-        ], None)
+            ],
-        project = Project.from_config('composetest', config, None)
+            networks=None,
+            volumes=None,
+        )
+        project = Project.from_config(
+            name='composetest',
+            config_data=config,
+            client=None,
+        )
         self.assertEqual(len(project.services), 2)
         self.assertEqual(project.get_service('web').name, 'web')
         self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
@@ -58,7 +48,9 @@ class ProjectTest(unittest.TestCase):
         self.assertFalse(project.use_networking)
 
     def test_from_config_v2(self):
-        config = Config(2, [
+        config = Config(
+            version=2,
+            services=[
                 {
                     'name': 'web',
                     'image': 'busybox:latest',
@@ -67,7 +59,10 @@ class ProjectTest(unittest.TestCase):
                     'name': 'db',
                     'image': 'busybox:latest',
                 },
-        ], None)
+            ],
+            networks=None,
+            volumes=None,
+        )
         project = Project.from_config('composetest', config, None)
         self.assertEqual(len(project.services), 2)
         self.assertTrue(project.use_networking)
@@ -161,13 +156,20 @@ class ProjectTest(unittest.TestCase):
         container_id = 'aabbccddee'
         container_dict = dict(Name='aaa', Id=container_id)
         self.mock_client.inspect_container.return_value = container_dict
-        project = Project.from_config('test', Config(None, [
+        project = Project.from_config(
-            {
+            name='test',
+            client=self.mock_client,
+            config_data=Config(
+                version=None,
+                services=[{
                     'name': 'test',
                     'image': 'busybox:latest',
                     'volumes_from': [VolumeFromSpec('aaa', 'rw', 'container')]
-            }
+                }],
-        ], None), self.mock_client)
+                networks=None,
+                volumes=None,
+            ),
+        )
         assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"]
 
     def test_use_volumes_from_service_no_container(self):
@@ -180,7 +182,12 @@ class ProjectTest(unittest.TestCase):
                 "Image": 'busybox:latest'
             }
         ]
-        project = Project.from_config('test', Config(None, [
+        project = Project.from_config(
+            name='test',
+            client=self.mock_client,
+            config_data=Config(
+                version=None,
+                services=[
                     {
                         'name': 'vol',
                         'image': 'busybox:latest'
@@ -190,13 +197,22 @@ class ProjectTest(unittest.TestCase):
                         'image': 'busybox:latest',
                         'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')]
                     }
-        ], None), self.mock_client)
+                ],
+                networks=None,
+                volumes=None,
+            ),
+        )
         assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"]
 
     def test_use_volumes_from_service_container(self):
         container_ids = ['aabbccddee', '12345']
 
-        project = Project.from_config('test', Config(None, [
+        project = Project.from_config(
+            name='test',
+            client=None,
+            config_data=Config(
+                version=None,
+                services=[
                     {
                         'name': 'vol',
                         'image': 'busybox:latest'
@@ -206,7 +222,11 @@ class ProjectTest(unittest.TestCase):
                         'image': 'busybox:latest',
                         'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')]
                     }
-        ], None), None)
+                ],
+                networks=None,
+                volumes=None,
+            ),
+        )
         with mock.patch.object(Service, 'containers') as mock_return:
             mock_return.return_value = [
                 mock.Mock(id=container_id, spec=Container)
@@ -313,12 +333,21 @@ class ProjectTest(unittest.TestCase):
         ]
 
     def test_net_unset(self):
-        project = Project.from_config('test', Config(None, [
+        project = Project.from_config(
+            name='test',
+            client=self.mock_client,
+            config_data=Config(
+                version=None,
+                services=[
                     {
                         'name': 'test',
                         'image': 'busybox:latest',
                     }
-        ], None), self.mock_client)
+                ],
+                networks=None,
+                volumes=None,
+            ),
+        )
         service = project.get_service('test')
         self.assertEqual(service.net.id, None)
         self.assertNotIn('NetworkMode', service._get_container_host_config({}))
@@ -327,13 +356,22 @@ class ProjectTest(unittest.TestCase):
         container_id = 'aabbccddee'
         container_dict = dict(Name='aaa', Id=container_id)
         self.mock_client.inspect_container.return_value = container_dict
-        project = Project.from_config('test', Config(None, [
+        project = Project.from_config(
+            name='test',
+            client=self.mock_client,
+            config_data=Config(
+                version=None,
+                services=[
                     {
                         'name': 'test',
                         'image': 'busybox:latest',
                         'net': 'container:aaa'
-            }
+                    },
-        ], None), self.mock_client)
+                ],
+                networks=None,
+                volumes=None,
+            ),
+        )
         service = project.get_service('test')
         self.assertEqual(service.net.mode, 'container:' + container_id)
 
@@ -347,7 +385,12 @@ class ProjectTest(unittest.TestCase):
                 "Image": 'busybox:latest'
             }
         ]
-        project = Project.from_config('test', Config(None, [
+        project = Project.from_config(
+            name='test',
+            client=self.mock_client,
+            config_data=Config(
+                version=None,
+                services=[
                     {
                         'name': 'aaa',
                         'image': 'busybox:latest'
@@ -356,36 +399,53 @@ class ProjectTest(unittest.TestCase):
                         'name': 'test',
                         'image': 'busybox:latest',
                         'net': 'container:aaa'
-            }
+                    },
-        ], None), self.mock_client)
+                ],
+                networks=None,
+                volumes=None,
+            ),
+        )
 
         service = project.get_service('test')
         self.assertEqual(service.net.mode, 'container:' + container_name)
 
     def test_uses_default_network_true(self):
-        web = Service('web', project='test', image="alpine", net=Net('test_default'))
+        project = Project.from_config(
-        db = Service('web', project='test', image="alpine", net=Net('other'))
+            name='test',
-        project = Project('test', [web, db], None)
+            client=self.mock_client,
+            config_data=Config(
+                version=2,
+                services=[
+                    {
+                        'name': 'foo',
+                        'image': 'busybox:latest'
+                    },
+                ],
+                networks=None,
+                volumes=None,
+            ),
+        )
+
         assert project.uses_default_network()
 
-    def test_uses_default_network_custom_name(self):
+    def test_uses_default_network_false(self):
-        web = Service('web', project='test', image="alpine", net=Net('other'))
+        project = Project.from_config(
-        project = Project('test', [web], None)
+            name='test',
-        assert not project.uses_default_network()
+            client=self.mock_client,
+            config_data=Config(
+                version=2,
+                services=[
+                    {
+                        'name': 'foo',
+                        'image': 'busybox:latest',
+                        'networks': ['custom']
+                    },
+                ],
+                networks={'custom': {}},
+                volumes=None,
+            ),
+        )
 
-    def test_uses_default_network_host(self):
-        web = Service('web', project='test', image="alpine", net=Net('host'))
-        project = Project('test', [web], None)
-        assert not project.uses_default_network()
-
-    def test_uses_default_network_container(self):
-        container = mock.Mock(id='test')
-        web = Service(
-            'web',
-            project='test',
-            image="alpine",
-            net=ContainerNet(container))
-        project = Project('test', [web], None)
         assert not project.uses_default_network()
 
     def test_container_without_name(self):
@@ -403,11 +463,16 @@ class ProjectTest(unittest.TestCase):
             },
         }
         project = Project.from_config(
-            'test',
+            name='test',
-            Config(None, [{
+            client=self.mock_client,
+            config_data=Config(
+                version=None,
+                services=[{
                     'name': 'web',
                     'image': 'busybox:latest',
-            }], None),
+                }],
-            self.mock_client,
+                networks=None,
+                volumes=None,
+            ),
         )
         self.assertEqual([c.id for c in project.containers()], ['1'])