Merge pull request #5684 from docker/compat_mode

Compatibility mode

compose/cli/command.py

@@ -38,6 +38,7 @@ def project_from_options(project_dir, options):
         tls_config=tls_config_from_options(options, environment),
         environment=environment,
         override_dir=options.get('--project-directory'),
+        compatibility=options.get('--compatibility'),
     )
 
 
@@ -63,7 +64,8 @@ def get_config_from_options(base_dir, options):
         base_dir, options, environment
     )
     return config.load(
-        config.find(base_dir, config_path, environment)
+        config.find(base_dir, config_path, environment),
+        options.get('--compatibility')
     )
 
 
@@ -100,14 +102,15 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N
 
 
 def get_project(project_dir, config_path=None, project_name=None, verbose=False,
-                host=None, tls_config=None, environment=None, override_dir=None):
+                host=None, tls_config=None, environment=None, override_dir=None,
+                compatibility=False):
     if not environment:
         environment = Environment.from_env_file(project_dir)
     config_details = config.find(project_dir, config_path, environment, override_dir)
     project_name = get_project_name(
         config_details.working_dir, project_name, environment
     )
-    config_data = config.load(config_details)
+    config_data = config.load(config_details, compatibility)
 
     api_version = environment.get(
         'COMPOSE_API_VERSION',

compose/cli/main.py

@@ -186,8 +186,10 @@ class TopLevelCommand(object):
       docker-compose -h|--help
 
     Options:
-      -f, --file FILE             Specify an alternate compose file (default: docker-compose.yml)
+      -f, --file FILE             Specify an alternate compose file
-      -p, --project-name NAME     Specify an alternate project name (default: directory name)
+                                  (default: docker-compose.yml)
+      -p, --project-name NAME     Specify an alternate project name
+                                  (default: directory name)
       --verbose                   Show more output
       --log-level LEVEL           Set log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
       --no-ansi                   Do not print ANSI control characters
@@ -199,11 +201,12 @@ class TopLevelCommand(object):
       --tlscert CLIENT_CERT_PATH  Path to TLS certificate file
       --tlskey TLS_KEY_PATH       Path to TLS key file
       --tlsverify                 Use TLS and verify the remote
-      --skip-hostname-check       Don't check the daemon's hostname against the name specified
+      --skip-hostname-check       Don't check the daemon's hostname against the
-                                  in the client certificate (for example if your docker host
+                                  name specified in the client certificate
-                                  is an IP address)
       --project-directory PATH    Specify an alternate working directory
                                   (default: the path of the Compose file)
+      --compatibility             If set, Compose will attempt to convert deploy
+                                  keys in v3 files to their non-Swarm equivalent
 
     Commands:
       build              Build or rebuild services

compose/config/config.py

@@ -16,6 +16,7 @@ from . import types
 from .. import const
 from ..const import COMPOSEFILE_V1 as V1
 from ..const import COMPOSEFILE_V2_1 as V2_1
+from ..const import COMPOSEFILE_V2_3 as V2_3
 from ..const import COMPOSEFILE_V3_0 as V3_0
 from ..const import COMPOSEFILE_V3_4 as V3_4
 from ..utils import build_string_dict
@@ -341,7 +342,7 @@ def find_candidates_in_parent_dirs(filenames, path):
     return (candidates, path)
 
 
-def check_swarm_only_config(service_dicts):
+def check_swarm_only_config(service_dicts, compatibility=False):
     warning_template = (
         "Some services ({services}) use the '{key}' key, which will be ignored. "
         "Compose does not support '{key}' configuration - use "
@@ -357,13 +358,13 @@ def check_swarm_only_config(service_dicts):
                     key=key
                 )
             )
-
+    if not compatibility:
-    check_swarm_only_key(service_dicts, 'deploy')
+        check_swarm_only_key(service_dicts, 'deploy')
     check_swarm_only_key(service_dicts, 'credential_spec')
     check_swarm_only_key(service_dicts, 'configs')
 
 
-def load(config_details):
+def load(config_details, compatibility=False):
     """Load the configuration from a working directory and a list of
     configuration files.  Files are loaded in order, and merged on top
     of each other to create the final configuration.
@@ -391,15 +392,17 @@ def load(config_details):
     configs = load_mapping(
         config_details.config_files, 'get_configs', 'Config', config_details.working_dir
     )
-    service_dicts = load_services(config_details, main_file)
+    service_dicts = load_services(config_details, main_file, compatibility)
 
     if main_file.version != V1:
         for service_dict in service_dicts:
             match_named_volumes(service_dict, volumes)
 
-    check_swarm_only_config(service_dicts)
+    check_swarm_only_config(service_dicts, compatibility)
 
-    return Config(main_file.version, service_dicts, volumes, networks, secrets, configs)
+    version = V2_3 if compatibility and main_file.version >= V3_0 else main_file.version
+
+    return Config(version, service_dicts, volumes, networks, secrets, configs)
 
 
 def load_mapping(config_files, get_func, entity_type, working_dir=None):
@@ -441,7 +444,7 @@ def validate_external(entity_type, name, config, version):
                 entity_type, name, ', '.join(k for k in config if k != 'external')))
 
 
-def load_services(config_details, config_file):
+def load_services(config_details, config_file, compatibility=False):
     def build_service(service_name, service_dict, service_names):
         service_config = ServiceConfig.with_abs_paths(
             config_details.working_dir,
@@ -459,7 +462,9 @@ def load_services(config_details, config_file):
             service_config,
             service_names,
             config_file.version,
-            config_details.environment)
+            config_details.environment,
+            compatibility
+        )
         return service_dict
 
     def build_services(service_config):
@@ -827,7 +832,7 @@ def finalize_service_volumes(service_dict, environment):
     return service_dict
 
 
-def finalize_service(service_config, service_names, version, environment):
+def finalize_service(service_config, service_names, version, environment, compatibility):
     service_dict = dict(service_config.config)
 
     if 'environment' in service_dict or 'env_file' in service_dict:
@@ -868,10 +873,80 @@ def finalize_service(service_config, service_names, version, environment):
 
     normalize_build(service_dict, service_config.working_dir, environment)
 
+    if compatibility:
+        service_dict, ignored_keys = translate_deploy_keys_to_container_config(
+            service_dict
+        )
+        if ignored_keys:
+            log.warn(
+                'The following deploy sub-keys are not supported in compatibility mode and have'
+                ' been ignored: {}'.format(', '.join(ignored_keys))
+            )
+
     service_dict['name'] = service_config.name
     return normalize_v1_service_format(service_dict)
 
 
+def translate_resource_keys_to_container_config(resources_dict, service_dict):
+    if 'limits' in resources_dict:
+        service_dict['mem_limit'] = resources_dict['limits'].get('memory')
+        if 'cpus' in resources_dict['limits']:
+            service_dict['cpus'] = float(resources_dict['limits']['cpus'])
+    if 'reservations' in resources_dict:
+        service_dict['mem_reservation'] = resources_dict['reservations'].get('memory')
+        if 'cpus' in resources_dict['reservations']:
+            return ['resources.reservations.cpus']
+    return []
+
+
+def convert_restart_policy(name):
+    try:
+        return {
+            'any': 'always',
+            'none': 'no',
+            'on-failure': 'on-failure'
+        }[name]
+    except KeyError:
+        raise ConfigurationError('Invalid restart policy "{}"'.format(name))
+
+
+def translate_deploy_keys_to_container_config(service_dict):
+    if 'deploy' not in service_dict:
+        return service_dict, []
+
+    deploy_dict = service_dict['deploy']
+    ignored_keys = [
+        k for k in ['endpoint_mode', 'labels', 'update_config', 'placement']
+        if k in deploy_dict
+    ]
+
+    if 'replicas' in deploy_dict and deploy_dict.get('mode', 'replicated') == 'replicated':
+        service_dict['scale'] = deploy_dict['replicas']
+
+    if 'restart_policy' in deploy_dict:
+        service_dict['restart'] = {
+            'Name': convert_restart_policy(deploy_dict['restart_policy'].get('condition', 'any')),
+            'MaximumRetryCount': deploy_dict['restart_policy'].get('max_attempts', 0)
+        }
+        for k in deploy_dict['restart_policy'].keys():
+            if k != 'condition' and k != 'max_attempts':
+                ignored_keys.append('restart_policy.{}'.format(k))
+
+    ignored_keys.extend(
+        translate_resource_keys_to_container_config(
+            deploy_dict.get('resources', {}), service_dict
+        )
+    )
+
+    del service_dict['deploy']
+    if 'credential_spec' in service_dict:
+        del service_dict['credential_spec']
+    if 'configs' in service_dict:
+        del service_dict['configs']
+
+    return service_dict, ignored_keys
+
+
 def normalize_v1_service_format(service_dict):
     if 'log_driver' in service_dict or 'log_opt' in service_dict:
         if 'logging' not in service_dict:

compose/config/config_schema_v3.6.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "config_schema_v3.5.json",
+  "id": "config_schema_v3.6.json",
   "type": "object",
   "required": ["version"],
 

tests/acceptance/cli_test.py

@@ -395,7 +395,7 @@ class CLITestCase(DockerClientTestCase):
         result = self.dispatch(['config'])
 
         assert yaml.load(result.stdout) == {
-            'version': '3.2',
+            'version': '3.5',
             'volumes': {
                 'foobar': {
                     'labels': {
@@ -419,22 +419,25 @@ class CLITestCase(DockerClientTestCase):
                         },
                         'resources': {
                             'limits': {
-                                'cpus': '0.001',
+                                'cpus': '0.05',
                                 'memory': '50M',
                             },
                             'reservations': {
-                                'cpus': '0.0001',
+                                'cpus': '0.01',
                                 'memory': '20M',
                             },
                         },
                         'restart_policy': {
-                            'condition': 'on_failure',
+                            'condition': 'on-failure',
                             'delay': '5s',
                             'max_attempts': 3,
                             'window': '120s',
                         },
                         'placement': {
-                            'constraints': ['node=foo'],
+                            'constraints': [
+                                'node.hostname==foo', 'node.role != manager'
+                            ],
+                            'preferences': [{'spread': 'node.labels.datacenter'}]
                         },
                     },
 
@@ -464,6 +467,27 @@ class CLITestCase(DockerClientTestCase):
             },
         }
 
+    def test_config_compatibility_mode(self):
+        self.base_dir = 'tests/fixtures/compatibility-mode'
+        result = self.dispatch(['--compatibility', 'config'])
+
+        assert yaml.load(result.stdout) == {
+            'version': '2.3',
+            'volumes': {'foo': {'driver': 'default'}},
+            'services': {
+                'foo': {
+                    'command': '/bin/true',
+                    'image': 'alpine:3.7',
+                    'scale': 3,
+                    'restart': 'always:7',
+                    'mem_limit': '300M',
+                    'mem_reservation': '100M',
+                    'cpus': 0.7,
+                    'volumes': ['foo:/bar:rw']
+                }
+            }
+        }
+
     def test_ps(self):
         self.project.get_service('simple').create_container()
         result = self.dispatch(['ps'])

tests/fixtures/compatibility-mode/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3.5'
+services:
+  foo:
+    image: alpine:3.7
+    command: /bin/true
+    deploy:
+      replicas: 3
+      restart_policy:
+        condition: any
+        max_attempts: 7
+      resources:
+        limits:
+          memory: 300M
+          cpus: '0.7'
+        reservations:
+          memory: 100M
+    volumes:
+      - foo:/bar
+
+volumes:
+  foo:
+    driver: default

tests/fixtures/v3-full/docker-compose.yml

@@ -1,8 +1,7 @@
-version: "3.2"
+version: "3.5"
 services:
   web:
     image: busybox
-
     deploy:
       mode: replicated
       replicas: 6
@@ -15,18 +14,22 @@ services:
         max_failure_ratio: 0.3
       resources:
         limits:
-          cpus: '0.001'
+          cpus: '0.05'
           memory: 50M
         reservations:
-          cpus: '0.0001'
+          cpus: '0.01'
           memory: 20M
       restart_policy:
-        condition: on_failure
+        condition: on-failure
         delay: 5s
         max_attempts: 3
         window: 120s
       placement:
-        constraints: [node=foo]
+        constraints:
+          - node.hostname==foo
+          - node.role != manager
+        preferences:
+          - spread: node.labels.datacenter
 
     healthcheck:
       test: cat /etc/passwd

tests/unit/config/config_test.py

@@ -3303,6 +3303,82 @@ class InterpolationTest(unittest.TestCase):
             assert 'BAR' in warnings[0]
             assert 'FOO' in warnings[1]
 
+    def test_compatibility_mode_warnings(self):
+        config_details = build_config_details({
+            'version': '3.5',
+            'services': {
+                'web': {
+                    'deploy': {
+                        'labels': ['abc=def'],
+                        'endpoint_mode': 'dnsrr',
+                        'update_config': {'max_failure_ratio': 0.4},
+                        'placement': {'constraints': ['node.id==deadbeef']},
+                        'resources': {
+                            'reservations': {'cpus': '0.2'}
+                        },
+                        'restart_policy': {
+                            'delay': '2s',
+                            'window': '12s'
+                        }
+                    },
+                    'image': 'busybox'
+                }
+            }
+        })
+
+        with mock.patch('compose.config.config.log') as log:
+            config.load(config_details, compatibility=True)
+
+        assert log.warn.call_count == 1
+        warn_message = log.warn.call_args[0][0]
+        assert warn_message.startswith(
+            'The following deploy sub-keys are not supported in compatibility mode'
+        )
+        assert 'labels' in warn_message
+        assert 'endpoint_mode' in warn_message
+        assert 'update_config' in warn_message
+        assert 'placement' in warn_message
+        assert 'resources.reservations.cpus' in warn_message
+        assert 'restart_policy.delay' in warn_message
+        assert 'restart_policy.window' in warn_message
+
+    def test_compatibility_mode_load(self):
+        config_details = build_config_details({
+            'version': '3.5',
+            'services': {
+                'foo': {
+                    'image': 'alpine:3.7',
+                    'deploy': {
+                        'replicas': 3,
+                        'restart_policy': {
+                            'condition': 'any',
+                            'max_attempts': 7,
+                        },
+                        'resources': {
+                            'limits': {'memory': '300M', 'cpus': '0.7'},
+                            'reservations': {'memory': '100M'},
+                        },
+                    },
+                },
+            },
+        })
+
+        with mock.patch('compose.config.config.log') as log:
+            cfg = config.load(config_details, compatibility=True)
+
+        assert log.warn.call_count == 0
+
+        service_dict = cfg.services[0]
+        assert service_dict == {
+            'image': 'alpine:3.7',
+            'scale': 3,
+            'restart': {'MaximumRetryCount': 7, 'Name': 'always'},
+            'mem_limit': '300M',
+            'mem_reservation': '100M',
+            'cpus': 0.7,
+            'name': 'foo'
+        }
+
     @mock.patch.dict(os.environ)
     def test_invalid_interpolation(self):
         with pytest.raises(config.ConfigurationError) as cm: