From f17e7268b0a7fd287bd8be3af61f9363222d2e07 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 3 Oct 2019 15:46:49 +0200 Subject: [PATCH 1/7] Properly escape values coming from env_files, fixes #6871 Signed-off-by: Florian Apolloner --- compose/config/config.py | 16 +++++++++------- compose/config/environment.py | 3 ++- tests/fixtures/env/three.env | 2 ++ tests/unit/config/config_test.py | 6 +++++- 4 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/env/three.env diff --git a/compose/config/config.py b/compose/config/config.py index 84933e9c9..3bd49a0f7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -408,7 +408,7 @@ def load(config_details, compatibility=False, interpolate=True): configs = load_mapping( config_details.config_files, 'get_configs', 'Config', config_details.working_dir ) - service_dicts = load_services(config_details, main_file, compatibility) + service_dicts = load_services(config_details, main_file, compatibility, interpolate=interpolate) if main_file.version != V1: for service_dict in service_dicts: @@ -460,7 +460,7 @@ def validate_external(entity_type, name, config, version): entity_type, name, ', '.join(k for k in config if k != 'external'))) -def load_services(config_details, config_file, compatibility=False): +def load_services(config_details, config_file, compatibility=False, interpolate=True): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( config_details.working_dir, @@ -479,7 +479,8 @@ def load_services(config_details, config_file, compatibility=False): service_names, config_file.version, config_details.environment, - compatibility + compatibility, + interpolate ) return service_dict @@ -679,13 +680,13 @@ class ServiceExtendsResolver(object): return filename -def resolve_environment(service_dict, environment=None): +def resolve_environment(service_dict, environment=None, interpolate=True): """Unpack any environment variables from an env_file, if set. Interpolate environment values if set. """ env = {} for env_file in service_dict.get('env_file', []): - env.update(env_vars_from_file(env_file)) + env.update(env_vars_from_file(env_file, interpolate)) env.update(parse_environment(service_dict.get('environment'))) return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(env)) @@ -881,11 +882,12 @@ def finalize_service_volumes(service_dict, environment): return service_dict -def finalize_service(service_config, service_names, version, environment, compatibility): +def finalize_service(service_config, service_names, version, environment, compatibility, + interpolate=True): service_dict = dict(service_config.config) if 'environment' in service_dict or 'env_file' in service_dict: - service_dict['environment'] = resolve_environment(service_dict, environment) + service_dict['environment'] = resolve_environment(service_dict, environment, interpolate) service_dict.pop('env_file', None) if 'volumes_from' in service_dict: diff --git a/compose/config/environment.py b/compose/config/environment.py index 6afbfc972..292029eb5 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -30,7 +30,7 @@ def split_env(env): return key, value -def env_vars_from_file(filename): +def env_vars_from_file(filename, interpolate=True): """ Read in a line delimited file of environment variables. """ @@ -39,6 +39,7 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise EnvFileNotFound("{} is not a file.".format(filename)) + # TODO: now we should do something with interpolate here, but what? return dotenv.dotenv_values(dotenv_path=filename, encoding='utf-8-sig') diff --git a/tests/fixtures/env/three.env b/tests/fixtures/env/three.env new file mode 100644 index 000000000..c2da74f19 --- /dev/null +++ b/tests/fixtures/env/three.env @@ -0,0 +1,2 @@ +FOO=NO $ENV VAR +DOO=NO ${ENV} VAR diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index dc346df95..933f659f8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5420,15 +5420,19 @@ class SerializeTest(unittest.TestCase): 'environment': { 'CURRENCY': '$' }, + 'env_file': ['tests/fixtures/env/three.env'], 'entrypoint': ['$SHELL', '-c'], } } } - config_dict = config.load(build_config_details(cfg), interpolate=False) + config_dict = config.load(build_config_details(cfg, working_dir='.'), interpolate=False) serialized_config = yaml.safe_load(serialize_config(config_dict, escape_dollar=False)) serialized_service = serialized_config['services']['web'] assert serialized_service['environment']['CURRENCY'] == '$' + # Values coming from env_files are not allowed to have variables + assert serialized_service['environment']['FOO'] == 'NO $$ENV VAR' + assert serialized_service['environment']['DOO'] == 'NO $${ENV} VAR' assert serialized_service['command'] == 'echo $FOO' assert serialized_service['entrypoint'][0] == '$SHELL' From edf27e486a1f556a74c915f9c1915c0b4466970e Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 29 Jan 2020 14:29:32 +0100 Subject: [PATCH 2/7] Pass the interpolation value to python-dotenv Signed-off-by: Ulysses Souza --- compose/config/environment.py | 6 ++++-- requirements.txt | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 292029eb5..d8810ebb6 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -39,8 +39,10 @@ def env_vars_from_file(filename, interpolate=True): elif not os.path.isfile(filename): raise EnvFileNotFound("{} is not a file.".format(filename)) - # TODO: now we should do something with interpolate here, but what? - return dotenv.dotenv_values(dotenv_path=filename, encoding='utf-8-sig') + env = dotenv.dotenv_values(dotenv_path=filename, encoding='utf-8-sig', interpolate=interpolate) + for k, v in env.items(): + env[k] = v if interpolate else v.replace('$', '$$') + return env class Environment(dict): diff --git a/requirements.txt b/requirements.txt index ee57c26b5..8b1eca573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' +git+git://github.com/ulyssessouza/python-dotenv.git@no-interpolate#egg=python-dotenv idna==2.8 ipaddress==1.0.23 jsonschema==3.2.0 @@ -17,7 +18,7 @@ paramiko==2.7.1 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.7.1 -python-dotenv==0.10.5 +#python-dotenv==0.10.5 PyYAML==5.3 requests==2.22.0 six==1.12.0 From 5fe0858450303f701175dbc879d68fc558adda2d Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 6 Feb 2020 09:44:37 +0100 Subject: [PATCH 3/7] Updated requirements.txt back to the released python-dotenv 0.11.0. Signed-off-by: Florian Apolloner --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8b1eca573..92f5d104d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' -git+git://github.com/ulyssessouza/python-dotenv.git@no-interpolate#egg=python-dotenv idna==2.8 ipaddress==1.0.23 jsonschema==3.2.0 @@ -18,7 +17,7 @@ paramiko==2.7.1 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.7.1 -#python-dotenv==0.10.5 +python-dotenv==0.11.0 PyYAML==5.3 requests==2.22.0 six==1.12.0 From cfefeaa6f750037b33a4fa4d18706b0dfad9d378 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 27 Feb 2020 10:48:37 +0100 Subject: [PATCH 4/7] Resolve a compatibility issue Signed-off-by: Ulysses Souza --- requirements-dev.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d723b3705..7f17d7bba 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ Click==7.0 coverage==5.0.3 ddt==1.2.2 flake8==3.7.9 -gitpython==2.1.14 +gitpython==2.1.15 mock==3.0.5 pytest==5.3.4; python_version >= '3.5' pytest==4.6.5; python_version < '3.5' diff --git a/requirements.txt b/requirements.txt index 990482351..b1bc69442 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' -idna==2.9 +idna==2.8 ipaddress==1.0.23 jsonschema==3.2.0 paramiko==2.7.1 From 2769c33a7eae5941b840e6f16c2919c9dcfc4342 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Thu, 27 Feb 2020 17:16:38 +0100 Subject: [PATCH 5/7] Update Jenkins build status badge Signed-off-by: Stefan Scherer --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1ae7e1ed..27c8dcb34 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Installation and documentation Contributing ------------ -[![Build Status](https://jenkins.dockerproject.org/buildStatus/icon?job=docker/compose/master)](https://jenkins.dockerproject.org/job/docker/job/compose/job/master/) +[![Build Status](https://ci-next.docker.com/public/buildStatus/icon?job=compose/master)](https://ci-next.docker.com/public/job/compose/job/master/) Want to help build Compose? Check out our [contributing documentation](https://github.com/docker/compose/blob/master/CONTRIBUTING.md). From 98abe0764637c48518b9af3c3bfedfd08559db67 Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Thu, 5 Mar 2020 15:46:45 +0100 Subject: [PATCH 6/7] Fix v3.8 schema support for binaries Signed-off-by: Anca Iordache --- docker-compose.spec | 5 +++++ docker-compose_darwin.spec | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/docker-compose.spec b/docker-compose.spec index 5ca1e4c24..4c8db3e51 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -87,6 +87,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.7.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.8.json', + 'compose/config/config_schema_v3.8.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/docker-compose_darwin.spec b/docker-compose_darwin.spec index 344c070d5..df7fcdd6f 100644 --- a/docker-compose_darwin.spec +++ b/docker-compose_darwin.spec @@ -96,6 +96,11 @@ coll = COLLECT(exe, 'compose/config/config_schema_v3.7.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.8.json', + 'compose/config/config_schema_v3.8.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', From 0ace76114b520360fe9bb31a1098be4a1805d07a Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 25 Feb 2020 17:09:14 +0100 Subject: [PATCH 7/7] Add conformity tests hack - That can be used with: ./script/test/acceptance Signed-off-by: Ulysses Souza --- script/test/acceptance | 3 + tests/acceptance/cli_test.py | 4 +- tests/conftest.py | 243 +++++++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 1 deletion(-) create mode 100755 script/test/acceptance create mode 100644 tests/conftest.py diff --git a/script/test/acceptance b/script/test/acceptance new file mode 100755 index 000000000..92710a76a --- /dev/null +++ b/script/test/acceptance @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +pytest --conformity --binary ${1:-docker-compose} tests/acceptance/ diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43d5a3f5c..9f5853143 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -38,6 +38,8 @@ from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only +DOCKER_COMPOSE_EXECUTABLE = 'docker-compose' + ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -65,7 +67,7 @@ COMPOSE_COMPATIBILITY_DICT = { def start_process(base_dir, options): proc = subprocess.Popen( - ['docker-compose'] + options, + [DOCKER_COMPOSE_EXECUTABLE] + options, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..013368689 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,243 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import pytest + +import tests.acceptance.cli_test + +# FIXME Skipping all the acceptance tests when in `--conformity` +non_conformity_tests = [ + "test_build_failed", + "test_build_failed_forcerm", + "test_build_log_level", + "test_build_memory_build_option", + "test_build_no_cache", + "test_build_no_cache_pull", + "test_build_override_dir", + "test_build_override_dir_invalid_path", + "test_build_parallel", + "test_build_plain", + "test_build_pull", + "test_build_rm", + "test_build_shm_size_build_option", + "test_build_with_buildarg_cli_override", + "test_build_with_buildarg_from_compose_file", + "test_build_with_buildarg_old_api_version", + "test_config_compatibility_mode", + "test_config_compatibility_mode_from_env", + "test_config_compatibility_mode_from_env_and_option_precedence", + "test_config_default", + "test_config_external_network", + "test_config_external_network_v3_5", + "test_config_external_volume_v2", + "test_config_external_volume_v2_x", + "test_config_external_volume_v3_4", + "test_config_external_volume_v3_x", + "test_config_list_services", + "test_config_list_volumes", + "test_config_quiet", + "test_config_quiet_with_error", + "test_config_restart", + "test_config_stdin", + "test_config_v1", + "test_config_v3", + "test_config_with_dot_env", + "test_config_with_dot_env_and_override_dir", + "test_config_with_env_file", + "test_config_with_hash_option", + "test_create", + "test_create_with_force_recreate", + "test_create_with_force_recreate_and_no_recreate", + "test_create_with_no_recreate", + "test_down", + "test_down_invalid_rmi_flag", + "test_down_signal", + "test_down_timeout", + "test_env_file_relative_to_compose_file", + "test_events_human_readable", + "test_events_json", + "test_exec_custom_user", + "test_exec_detach_long_form", + "test_exec_novalue_var_dotenv_file", + "test_exec_service_with_environment_overridden", + "test_exec_without_tty", + "test_exec_workdir", + "test_exit_code_from_signal_stop", + "test_expanded_port", + "test_forward_exitval", + "test_help", + "test_help_nonexistent", + "test_home_and_env_var_in_volume_path", + "test_host_not_reachable", + "test_host_not_reachable_volumes_from_container", + "test_host_not_reachable_volumes_from_container", + "test_images", + "test_images_default_composefile", + "test_images_tagless_image", + "test_images_use_service_tag", + "test_kill", + "test_kill_signal_sigstop", + "test_kill_stopped_service", + "test_logs_default", + "test_logs_follow", + "test_logs_follow_logs_from_new_containers", + "test_logs_follow_logs_from_restarted_containers", + "test_logs_invalid_service_name", + "test_logs_on_stopped_containers_exits", + "test_logs_tail", + "test_logs_timestamps", + "test_pause_no_containers", + "test_pause_unpause", + "test_port", + "test_port_with_scale", + "test_ps", + "test_ps_all", + "test_ps_alternate_composefile", + "test_ps_default_composefile", + "test_ps_services_filter_option", + "test_ps_services_filter_status", + "test_pull", + "test_pull_can_build", + "test_pull_with_digest", + "test_pull_with_ignore_pull_failures", + "test_pull_with_include_deps", + "test_pull_with_no_deps", + "test_pull_with_parallel_failure", + "test_pull_with_quiet", + "test_quiet_build", + "test_restart", + "test_restart_no_containers", + "test_restart_stopped_container", + "test_rm", + "test_rm_all", + "test_rm_stop", + "test_run_detached_connects_to_network", + "test_run_does_not_recreate_linked_containers", + "test_run_env_values_from_system", + "test_run_handles_sighup", + "test_run_handles_sigint", + "test_run_handles_sigterm", + "test_run_interactive_connects_to_network", + "test_run_label_flag", + "test_run_one_off_with_multiple_volumes", + "test_run_one_off_with_volume", + "test_run_one_off_with_volume_merge", + "test_run_rm", + "test_run_service_with_compose_file_entrypoint", + "test_run_service_with_compose_file_entrypoint_and_command_overridden", + "test_run_service_with_compose_file_entrypoint_and_empty_string_command", + "test_run_service_with_compose_file_entrypoint_overridden", + "test_run_service_with_dependencies", + "test_run_service_with_dockerfile_entrypoint", + "test_run_service_with_dockerfile_entrypoint_and_command_overridden", + "test_run_service_with_dockerfile_entrypoint_overridden", + "test_run_service_with_environment_overridden", + "test_run_service_with_explicitly_mapped_ip_ports", + "test_run_service_with_explicitly_mapped_ports", + "test_run_service_with_links", + "test_run_service_with_map_ports", + "test_run_service_with_scaled_dependencies", + "test_run_service_with_unset_entrypoint", + "test_run_service_with_use_aliases", + "test_run_service_with_user_overridden", + "test_run_service_with_user_overridden_short_form", + "test_run_service_with_workdir_overridden", + "test_run_service_with_workdir_overridden_short_form", + "test_run_service_without_links", + "test_run_service_without_map_ports", + "test_run_unicode_env_values_from_system", + "test_run_with_custom_name", + "test_run_with_expose_ports", + "test_run_with_no_deps", + "test_run_without_command", + "test_scale", + "test_scale_v2_2", + "test_shorthand_host_opt", + "test_shorthand_host_opt_interactive", + "test_start_no_containers", + "test_stop", + "test_stop_signal", + "test_top_processes_running", + "test_top_services_not_running", + "test_top_services_running", + "test_unpause_no_containers", + "test_up", + "test_up_attached", + "test_up_detached", + "test_up_detached_long_form", + "test_up_external_networks", + "test_up_handles_abort_on_container_exit", + "test_up_handles_abort_on_container_exit_code", + "test_up_handles_aborted_dependencies", + "test_up_handles_force_shutdown", + "test_up_handles_sigint", + "test_up_handles_sigterm", + "test_up_logging", + "test_up_logging_legacy", + "test_up_missing_network", + "test_up_no_ansi", + "test_up_no_services", + "test_up_no_start", + "test_up_no_start_remove_orphans", + "test_up_scale_reset", + "test_up_scale_scale_down", + "test_up_scale_scale_up", + "test_up_scale_to_zero", + "test_up_with_attach_dependencies", + "test_up_with_default_network_config", + "test_up_with_default_override_file", + "test_up_with_duplicate_override_yaml_files", + "test_up_with_extends", + "test_up_with_external_default_network", + "test_up_with_force_recreate", + "test_up_with_force_recreate_and_no_recreate", + "test_up_with_healthcheck", + "test_up_with_ignore_remove_orphans", + "test_up_with_links_v1", + "test_up_with_multiple_files", + "test_up_with_net_is_invalid", + "test_up_with_net_v1", + "test_up_with_network_aliases", + "test_up_with_network_internal", + "test_up_with_network_labels", + "test_up_with_network_mode", + "test_up_with_network_static_addresses", + "test_up_with_networks", + "test_up_with_no_deps", + "test_up_with_no_recreate", + "test_up_with_override_yaml", + "test_up_with_pid_mode", + "test_up_with_timeout", + "test_up_with_volume_labels", + "test_fail_on_both_host_and_context_opt", + "test_fail_run_on_inexistent_context", +] + + +def pytest_addoption(parser): + parser.addoption( + "--conformity", + action="store_true", + default=False, + help="Only runs tests that are not black listed as non conformity test. " + "The conformity tests check for compatibility with the Compose spec." + ) + parser.addoption( + "--binary", + default=tests.acceptance.cli_test.DOCKER_COMPOSE_EXECUTABLE, + help="Forces the execution of a binary in the PATH. Default is `docker-compose`." + ) + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--conformity"): + return + if config.getoption("--binary"): + tests.acceptance.cli_test.DOCKER_COMPOSE_EXECUTABLE = config.getoption("--binary") + + print("Binary -> {}".format(tests.acceptance.cli_test.DOCKER_COMPOSE_EXECUTABLE)) + skip_non_conformity = pytest.mark.skip(reason="skipping because that's not a conformity test") + for item in items: + if item.name in non_conformity_tests: + print("Skipping '{}' when running in compatibility mode".format(item.name)) + item.add_marker(skip_non_conformity)