From c69d8a3bd2584044348aa6a444cf22ed1fd5f43d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Mar 2016 15:49:42 -0800 Subject: [PATCH 01/16] Implement environment singleton to be accessed throughout the code Load and parse environment file from working dir Signed-off-by: Joffrey F --- compose/cli/command.py | 13 +++-- compose/cli/main.py | 2 +- compose/config/__init__.py | 1 + compose/config/config.py | 40 +++++++++----- compose/config/environment.py | 69 +++++++++++++++++++++++++ compose/config/interpolation.py | 23 +-------- tests/acceptance/cli_test.py | 4 +- tests/helpers.py | 13 +++++ tests/integration/service_test.py | 3 +- tests/unit/cli_test.py | 7 +-- tests/unit/config/config_test.py | 36 +++++++------ tests/unit/config/interpolation_test.py | 2 + tests/unit/interpolation_test.py | 2 +- 13 files changed, 151 insertions(+), 64 deletions(-) create mode 100644 compose/config/environment.py diff --git a/compose/cli/command.py b/compose/cli/command.py index 63d387f0c..3fcfbb4b3 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -21,7 +21,7 @@ log = logging.getLogger(__name__) def project_from_options(project_dir, options): return get_project( project_dir, - get_config_path_from_options(options), + get_config_path_from_options(project_dir, options), project_name=options.get('--project-name'), verbose=options.get('--verbose'), host=options.get('--host'), @@ -29,12 +29,13 @@ def project_from_options(project_dir, options): ) -def get_config_path_from_options(options): +def get_config_path_from_options(base_dir, options): file_option = options.get('--file') if file_option: return file_option - config_files = os.environ.get('COMPOSE_FILE') + environment = config.environment.get_instance(base_dir) + config_files = environment.get('COMPOSE_FILE') if config_files: return config_files.split(os.pathsep) return None @@ -57,8 +58,9 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(project_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) + environment = config.environment.get_instance(project_dir) - api_version = os.environ.get( + api_version = environment.get( 'COMPOSE_API_VERSION', API_VERSIONS[config_data.version]) client = get_client( @@ -73,7 +75,8 @@ def get_project_name(working_dir, project_name=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - project_name = project_name or os.environ.get('COMPOSE_PROJECT_NAME') + environment = config.environment.get_instance(working_dir) + project_name = project_name or environment.get('COMPOSE_PROJECT_NAME') if project_name: return normalize_name(project_name) diff --git a/compose/cli/main.py b/compose/cli/main.py index 6eada097f..3fa3e3a01 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -222,7 +222,7 @@ class TopLevelCommand(object): --services Print the service names, one per line. """ - config_path = get_config_path_from_options(config_options) + config_path = get_config_path_from_options(self.project_dir, config_options) compose_config = config.load(config.find(self.project_dir, config_path)) if options['--quiet']: diff --git a/compose/config/__init__.py b/compose/config/__init__.py index dd01f221e..7cf71eb98 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +from . import environment from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find diff --git a/compose/config/config.py b/compose/config/config.py index 961d0b57f..c9c4e3084 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -17,6 +17,7 @@ from cached_property import cached_property from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 from ..utils import build_string_dict +from .environment import Environment from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError @@ -211,7 +212,8 @@ def find(base_dir, filenames): if filenames == ['-']: return ConfigDetails( os.getcwd(), - [ConfigFile(None, yaml.safe_load(sys.stdin))]) + [ConfigFile(None, yaml.safe_load(sys.stdin))], + ) if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] @@ -221,7 +223,8 @@ def find(base_dir, filenames): log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( os.path.dirname(filenames[0]), - [ConfigFile.from_filename(f) for f in filenames]) + [ConfigFile.from_filename(f) for f in filenames], + ) def validate_config_version(config_files): @@ -288,6 +291,10 @@ def load(config_details): """ validate_config_version(config_details.config_files) + # load environment in working dir for later use in interpolation + # it is done here to avoid having to pass down working_dir + Environment.get_instance(config_details.working_dir) + processed_files = [ process_config_file(config_file) for config_file in config_details.config_files @@ -302,9 +309,8 @@ def load(config_details): config_details.config_files, 'get_networks', 'Network' ) service_dicts = load_services( - config_details.working_dir, - main_file, - [file.get_service_dicts() for file in config_details.config_files]) + config_details, main_file, + ) if main_file.version != V1: for service_dict in service_dicts: @@ -348,14 +354,16 @@ def load_mapping(config_files, get_func, entity_type): return mapping -def load_services(working_dir, config_file, service_configs): +def load_services(config_details, config_file): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( - working_dir, + config_details.working_dir, config_file.filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config, config_file) + resolver = ServiceExtendsResolver( + service_config, config_file + ) service_dict = process_service(resolver.run()) service_config = service_config._replace(config=service_dict) @@ -383,6 +391,10 @@ def load_services(working_dir, config_file, service_configs): for name in all_service_names } + service_configs = [ + file.get_service_dicts() for file in config_details.config_files + ] + service_config = service_configs[0] for next_config in service_configs[1:]: service_config = merge_services(service_config, next_config) @@ -462,8 +474,8 @@ class ServiceExtendsResolver(object): extends_file = ConfigFile.from_filename(config_path) validate_config_version([self.config_file, extends_file]) extended_file = process_config_file( - extends_file, - service_name=service_name) + extends_file, service_name=service_name + ) service_config = extended_file.get_service(service_name) return config_path, service_config, service_name @@ -476,7 +488,8 @@ class ServiceExtendsResolver(object): service_name, service_dict), self.config_file, - already_seen=self.already_seen + [self.signature]) + already_seen=self.already_seen + [self.signature], + ) service_config = resolver.run() other_service_dict = process_service(service_config) @@ -824,10 +837,11 @@ def parse_ulimits(ulimits): def resolve_env_var(key, val): + environment = Environment.get_instance() if val is not None: return key, val - elif key in os.environ: - return key, os.environ[key] + elif key in environment: + return key, environment[key] else: return key, None diff --git a/compose/config/environment.py b/compose/config/environment.py new file mode 100644 index 000000000..45f1c43fb --- /dev/null +++ b/compose/config/environment.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import logging +import os + +from .errors import ConfigurationError + +log = logging.getLogger(__name__) + + +class BlankDefaultDict(dict): + def __init__(self, *args, **kwargs): + super(BlankDefaultDict, self).__init__(*args, **kwargs) + self.missing_keys = [] + + def __getitem__(self, key): + try: + return super(BlankDefaultDict, self).__getitem__(key) + except KeyError: + if key not in self.missing_keys: + log.warn( + "The {} variable is not set. Defaulting to a blank string." + .format(key) + ) + self.missing_keys.append(key) + + return "" + + +class Environment(BlankDefaultDict): + __instance = None + + @classmethod + def get_instance(cls, base_dir='.'): + if cls.__instance: + return cls.__instance + + instance = cls(base_dir) + cls.__instance = instance + return instance + + @classmethod + def reset(cls): + cls.__instance = None + + def __init__(self, base_dir): + super(Environment, self).__init__() + self.load_environment_file(os.path.join(base_dir, '.env')) + self.update(os.environ) + + def load_environment_file(self, path): + if not os.path.exists(path): + return + mapping = {} + with open(path, 'r') as f: + for line in f.readlines(): + line = line.strip() + if '=' not in line: + raise ConfigurationError( + 'Invalid environment variable mapping in env file. ' + 'Missing "=" in "{0}"'.format(line) + ) + mapping.__setitem__(*line.split('=', 1)) + self.update(mapping) + + +def get_instance(base_dir=None): + return Environment.get_instance(base_dir) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 1e56ebb66..b76638d93 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -2,17 +2,17 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -import os from string import Template import six +from .environment import Environment from .errors import ConfigurationError log = logging.getLogger(__name__) def interpolate_environment_variables(config, section): - mapping = BlankDefaultDict(os.environ) + mapping = Environment.get_instance() def process_item(name, config_dict): return dict( @@ -60,25 +60,6 @@ def interpolate(string, mapping): raise InvalidInterpolation(string) -class BlankDefaultDict(dict): - def __init__(self, *args, **kwargs): - super(BlankDefaultDict, self).__init__(*args, **kwargs) - self.missing_keys = [] - - def __getitem__(self, key): - try: - return super(BlankDefaultDict, self).__getitem__(key) - except KeyError: - if key not in self.missing_keys: - log.warn( - "The {} variable is not set. Defaulting to a blank string." - .format(key) - ) - self.missing_keys.append(key) - - return "" - - class InvalidInterpolation(Exception): def __init__(self, string): self.string = string diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 707c24926..9d50ea99f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -15,7 +15,7 @@ from operator import attrgetter import yaml from docker import errors -from .. import mock +from ..helpers import clear_environment from compose.cli.command import get_project from compose.container import Container from compose.project import OneOffFilter @@ -1452,7 +1452,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn("FOO=1", containers[0].get('Config.Env')) - @mock.patch.dict(os.environ) + @clear_environment def test_home_and_env_var_in_volume_path(self): os.environ['VOLUME_NAME'] = 'my-volume' os.environ['HOME'] = '/tmp/home-dir' diff --git a/tests/helpers.py b/tests/helpers.py index dd0b668ed..2c3d5a98b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,9 +1,14 @@ from __future__ import absolute_import from __future__ import unicode_literals +import functools +import os + +from . import mock from compose.config.config import ConfigDetails from compose.config.config import ConfigFile from compose.config.config import load +from compose.config.environment import Environment def build_config(contents, **kwargs): @@ -14,3 +19,11 @@ def build_config_details(contents, working_dir='working_dir', filename='filename return ConfigDetails( working_dir, [ConfigFile(filename, contents)]) + + +def clear_environment(f): + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + Environment.reset() + with mock.patch.dict(os.environ): + f(self, *args, **kwargs) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index e2ef1161d..a1857b58a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,6 +12,7 @@ from six import StringIO from six import text_type from .. import mock +from ..helpers import clear_environment from .testcases import DockerClientTestCase from .testcases import get_links from .testcases import pull_busybox @@ -912,7 +913,7 @@ class ServiceTest(DockerClientTestCase): }.items(): self.assertEqual(env[k], v) - @mock.patch.dict(os.environ) + @clear_environment def test_resolve_env(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index e0ada460d..fd8aa95c1 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -11,6 +11,7 @@ import pytest from .. import mock from .. import unittest from ..helpers import build_config +from ..helpers import clear_environment from compose.cli.command import get_project from compose.cli.command import get_project_name from compose.cli.docopt_command import NoSuchCommand @@ -43,11 +44,11 @@ class CLITestCase(unittest.TestCase): project_name = get_project_name(None, project_name=name) self.assertEquals('explicitprojectname', project_name) + @clear_environment def test_project_name_from_environment_new_var(self): name = 'namefromenv' - with mock.patch.dict(os.environ): - os.environ['COMPOSE_PROJECT_NAME'] = name - project_name = get_project_name(None) + os.environ['COMPOSE_PROJECT_NAME'] = name + project_name = get_project_name(None) self.assertEquals(project_name, name) def test_project_name_with_empty_environment_var(self): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 04d82c811..9bd76fb9a 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -23,6 +23,7 @@ from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest +from tests.helpers import clear_environment DEFAULT_VERSION = V2_0 @@ -1581,7 +1582,7 @@ class PortsTest(unittest.TestCase): class InterpolationTest(unittest.TestCase): - @mock.patch.dict(os.environ) + @clear_environment def test_config_file_with_environment_variable(self): os.environ.update( IMAGE="busybox", @@ -1604,7 +1605,7 @@ class InterpolationTest(unittest.TestCase): } ]) - @mock.patch.dict(os.environ) + @clear_environment def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) os.environ.pop('BAR', None) @@ -1628,7 +1629,7 @@ class InterpolationTest(unittest.TestCase): self.assertIn('BAR', warnings[0]) self.assertIn('FOO', warnings[1]) - @mock.patch.dict(os.environ) + @clear_environment def test_invalid_interpolation(self): with self.assertRaises(config.ConfigurationError) as cm: config.load( @@ -1667,7 +1668,7 @@ class VolumeConfigTest(unittest.TestCase): d = make_service_dict('foo', {'build': '.', 'volumes': ['/data']}, working_dir='.') self.assertEqual(d['volumes'], ['/data']) - @mock.patch.dict(os.environ) + @clear_environment def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' @@ -1681,7 +1682,7 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') - @mock.patch.dict(os.environ) + @clear_environment def test_volume_binding_with_home(self): os.environ['HOME'] = '/home/user' d = make_service_dict('foo', {'build': '.', 'volumes': ['~:/container/path']}, working_dir='.') @@ -1739,7 +1740,7 @@ class VolumeConfigTest(unittest.TestCase): working_dir='c:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['c:\\Users\\me\\otherproject:/data']) - @mock.patch.dict(os.environ) + @clear_environment def test_home_directory_with_driver_does_not_expand(self): os.environ['NAME'] = 'surprise!' d = make_service_dict('foo', { @@ -2025,7 +2026,7 @@ class EnvTest(unittest.TestCase): def test_parse_environment_empty(self): self.assertEqual(config.parse_environment(None), {}) - @mock.patch.dict(os.environ) + @clear_environment def test_resolve_environment(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' @@ -2072,7 +2073,7 @@ class EnvTest(unittest.TestCase): assert 'Couldn\'t find env file' in exc.exconly() assert 'nonexistent.env' in exc.exconly() - @mock.patch.dict(os.environ) + @clear_environment def test_resolve_environment_from_env_file_with_empty_values(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' @@ -2087,7 +2088,7 @@ class EnvTest(unittest.TestCase): }, ) - @mock.patch.dict(os.environ) + @clear_environment def test_resolve_build_args(self): os.environ['env_arg'] = 'value2' @@ -2106,7 +2107,7 @@ class EnvTest(unittest.TestCase): ) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') - @mock.patch.dict(os.environ) + @clear_environment def test_resolve_path(self): os.environ['HOSTENV'] = '/tmp' os.environ['CONTAINERENV'] = '/host/tmp' @@ -2393,7 +2394,7 @@ class ExtendsTest(unittest.TestCase): assert 'net: container' in excinfo.exconly() assert 'cannot be extended' in excinfo.exconly() - @mock.patch.dict(os.environ) + @clear_environment def test_load_config_runs_interpolation_in_extended_service(self): os.environ.update(HOSTNAME_VALUE="penguin") expected_interpolated_value = "host-penguin" @@ -2465,6 +2466,7 @@ class ExtendsTest(unittest.TestCase): }, ])) + @clear_environment def test_extends_with_environment_and_env_files(self): tmpdir = py.test.ensuretemp('test_extends_with_environment') self.addCleanup(tmpdir.remove) @@ -2520,12 +2522,12 @@ class ExtendsTest(unittest.TestCase): }, }, ] - with mock.patch.dict(os.environ): - os.environ['SECRET'] = 'secret' - os.environ['THING'] = 'thing' - os.environ['COMMON_ENV_FILE'] = 'secret' - os.environ['TOP_ENV_FILE'] = 'secret' - config = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + + os.environ['SECRET'] = 'secret' + os.environ['THING'] = 'thing' + os.environ['COMMON_ENV_FILE'] = 'secret' + os.environ['TOP_ENV_FILE'] = 'secret' + config = load_from_filename(str(tmpdir.join('docker-compose.yml'))) assert config == expected diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 0691e8865..b4ba7b402 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -6,12 +6,14 @@ import os import mock import pytest +from compose.config.environment import Environment from compose.config.interpolation import interpolate_environment_variables @pytest.yield_fixture def mock_env(): with mock.patch.dict(os.environ): + Environment.reset() os.environ['USER'] = 'jenny' os.environ['FOO'] = 'bar' yield diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py index 317982a9b..b19fcdaca 100644 --- a/tests/unit/interpolation_test.py +++ b/tests/unit/interpolation_test.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import unittest -from compose.config.interpolation import BlankDefaultDict as bddict +from compose.config.environment import BlankDefaultDict as bddict from compose.config.interpolation import interpolate from compose.config.interpolation import InvalidInterpolation From bf8e501b5e7f255459dd232afa937d065b28f905 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 7 Mar 2016 14:42:17 -0800 Subject: [PATCH 02/16] Fix pre-commit config Signed-off-by: Joffrey F --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e37677c69..1ae45decb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ - id: end-of-file-fixer - id: flake8 - id: name-tests-test - exclude: 'tests/(helpers\.py|integration/testcases\.py)' + exclude: 'tests/(integration/testcases\.py)|(helpers\.py)' - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports From b9ca5188a21f99a2798c5451049e27be3e75ac27 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 7 Mar 2016 17:16:25 -0800 Subject: [PATCH 03/16] Remove Environment singleton, instead carry instance during config processing Project name and compose file detection also updated Signed-off-by: Joffrey F --- compose/cli/command.py | 6 +-- compose/config/config.py | 69 ++++++++++++++++++++------------- compose/config/environment.py | 22 +---------- compose/config/interpolation.py | 6 +-- 4 files changed, 48 insertions(+), 55 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 3fcfbb4b3..8c0bf07ff 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -34,7 +34,7 @@ def get_config_path_from_options(base_dir, options): if file_option: return file_option - environment = config.environment.get_instance(base_dir) + environment = config.environment.Environment(base_dir) config_files = environment.get('COMPOSE_FILE') if config_files: return config_files.split(os.pathsep) @@ -58,7 +58,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(project_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) - environment = config.environment.get_instance(project_dir) + environment = config.environment.Environment(project_dir) api_version = environment.get( 'COMPOSE_API_VERSION', @@ -75,7 +75,7 @@ def get_project_name(working_dir, project_name=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - environment = config.environment.get_instance(working_dir) + environment = config.environment.Environment(working_dir) project_name = project_name or environment.get('COMPOSE_PROJECT_NAME') if project_name: return normalize_name(project_name) diff --git a/compose/config/config.py b/compose/config/config.py index c9c4e3084..7db66004f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -114,14 +114,24 @@ DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' log = logging.getLogger(__name__) -class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files')): +class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files environment')): """ :param working_dir: the directory to use for relative paths in the config :type working_dir: string :param config_files: list of configuration files to load :type config_files: list of :class:`ConfigFile` + :param environment: computed environment values for this project + :type environment: :class:`environment.Environment` """ + def __new__(cls, working_dir, config_files): + return super(ConfigDetails, cls).__new__( + cls, + working_dir, + config_files, + Environment(working_dir), + ) + class ConfigFile(namedtuple('_ConfigFile', 'filename config')): """ @@ -291,12 +301,8 @@ def load(config_details): """ validate_config_version(config_details.config_files) - # load environment in working dir for later use in interpolation - # it is done here to avoid having to pass down working_dir - Environment.get_instance(config_details.working_dir) - processed_files = [ - process_config_file(config_file) + process_config_file(config_file, config_details.environment) for config_file in config_details.config_files ] config_details = config_details._replace(config_files=processed_files) @@ -362,7 +368,7 @@ def load_services(config_details, config_file): service_name, service_dict) resolver = ServiceExtendsResolver( - service_config, config_file + service_config, config_file, environment=config_details.environment ) service_dict = process_service(resolver.run()) @@ -371,7 +377,8 @@ def load_services(config_details, config_file): service_dict = finalize_service( service_config, service_names, - config_file.version) + config_file.version, + config_details.environment) return service_dict def build_services(service_config): @@ -402,16 +409,17 @@ def load_services(config_details, config_file): return build_services(service_config) -def interpolate_config_section(filename, config, section): +def interpolate_config_section(filename, config, section, environment): validate_config_section(filename, config, section) - return interpolate_environment_variables(config, section) + return interpolate_environment_variables(config, section, environment) -def process_config_file(config_file, service_name=None): +def process_config_file(config_file, environment, service_name=None): services = interpolate_config_section( config_file.filename, config_file.get_service_dicts(), - 'service') + 'service', + environment,) if config_file.version == V2_0: processed_config = dict(config_file.config) @@ -419,11 +427,13 @@ def process_config_file(config_file, service_name=None): processed_config['volumes'] = interpolate_config_section( config_file.filename, config_file.get_volumes(), - 'volume') + 'volume', + environment,) processed_config['networks'] = interpolate_config_section( config_file.filename, config_file.get_networks(), - 'network') + 'network', + environment,) if config_file.version == V1: processed_config = services @@ -440,11 +450,12 @@ def process_config_file(config_file, service_name=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, config_file, already_seen=None): + def __init__(self, service_config, config_file, environment=None, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] self.config_file = config_file + self.environment = environment or Environment(None) @property def signature(self): @@ -474,7 +485,7 @@ class ServiceExtendsResolver(object): extends_file = ConfigFile.from_filename(config_path) validate_config_version([self.config_file, extends_file]) extended_file = process_config_file( - extends_file, service_name=service_name + extends_file, self.environment, service_name=service_name ) service_config = extended_file.get_service(service_name) @@ -489,6 +500,7 @@ class ServiceExtendsResolver(object): service_dict), self.config_file, already_seen=self.already_seen + [self.signature], + environment=self.environment ) service_config = resolver.run() @@ -518,7 +530,7 @@ class ServiceExtendsResolver(object): return filename -def resolve_environment(service_dict): +def resolve_environment(service_dict, environment=None): """Unpack any environment variables from an env_file, if set. Interpolate environment values if set. """ @@ -527,12 +539,12 @@ def resolve_environment(service_dict): env.update(env_vars_from_file(env_file)) env.update(parse_environment(service_dict.get('environment'))) - return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) + return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(env)) -def resolve_build_args(build): +def resolve_build_args(build, environment): args = parse_build_arguments(build.get('args')) - return dict(resolve_env_var(k, v) for k, v in six.iteritems(args)) + return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(args)) def validate_extended_service_dict(service_dict, filename, service): @@ -611,11 +623,11 @@ def process_service(service_config): return service_dict -def finalize_service(service_config, service_names, version): +def finalize_service(service_config, service_names, version, environment): service_dict = dict(service_config.config) if 'environment' in service_dict or 'env_file' in service_dict: - service_dict['environment'] = resolve_environment(service_dict) + service_dict['environment'] = resolve_environment(service_dict, environment) service_dict.pop('env_file', None) if 'volumes_from' in service_dict: @@ -642,7 +654,7 @@ def finalize_service(service_config, service_names, version): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) - normalize_build(service_dict, service_config.working_dir) + normalize_build(service_dict, service_config.working_dir, environment) service_dict['name'] = service_config.name return normalize_v1_service_format(service_dict) @@ -836,11 +848,10 @@ def parse_ulimits(ulimits): return dict(ulimits) -def resolve_env_var(key, val): - environment = Environment.get_instance() +def resolve_env_var(key, val, environment): if val is not None: return key, val - elif key in environment: + elif environment and key in environment: return key, environment[key] else: return key, None @@ -880,7 +891,7 @@ def resolve_volume_path(working_dir, volume): return container_path -def normalize_build(service_dict, working_dir): +def normalize_build(service_dict, working_dir, environment): if 'build' in service_dict: build = {} @@ -890,7 +901,9 @@ def normalize_build(service_dict, working_dir): else: build.update(service_dict['build']) if 'args' in build: - build['args'] = build_string_dict(resolve_build_args(build)) + build['args'] = build_string_dict( + resolve_build_args(build, environment) + ) service_dict['build'] = build diff --git a/compose/config/environment.py b/compose/config/environment.py index 45f1c43fb..87b41223b 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -29,24 +29,10 @@ class BlankDefaultDict(dict): class Environment(BlankDefaultDict): - __instance = None - - @classmethod - def get_instance(cls, base_dir='.'): - if cls.__instance: - return cls.__instance - - instance = cls(base_dir) - cls.__instance = instance - return instance - - @classmethod - def reset(cls): - cls.__instance = None - def __init__(self, base_dir): super(Environment, self).__init__() - self.load_environment_file(os.path.join(base_dir, '.env')) + if base_dir: + self.load_environment_file(os.path.join(base_dir, '.env')) self.update(os.environ) def load_environment_file(self, path): @@ -63,7 +49,3 @@ class Environment(BlankDefaultDict): ) mapping.__setitem__(*line.split('=', 1)) self.update(mapping) - - -def get_instance(base_dir=None): - return Environment.get_instance(base_dir) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index b76638d93..63020d91a 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -6,17 +6,15 @@ from string import Template import six -from .environment import Environment from .errors import ConfigurationError log = logging.getLogger(__name__) -def interpolate_environment_variables(config, section): - mapping = Environment.get_instance() +def interpolate_environment_variables(config, section, environment): def process_item(name, config_dict): return dict( - (key, interpolate_value(name, key, val, section, mapping)) + (key, interpolate_value(name, key, val, section, environment)) for key, val in (config_dict or {}).items() ) From 5831b869e879b1350f0610f41b2dbd58c5e6285f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 7 Mar 2016 17:18:19 -0800 Subject: [PATCH 04/16] Update tests for new environment handling Signed-off-by: Joffrey F --- .pre-commit-config.yaml | 2 +- tests/acceptance/cli_test.py | 4 +-- tests/helpers.py | 13 --------- tests/integration/service_test.py | 3 +-- tests/integration/testcases.py | 3 ++- tests/unit/cli_test.py | 3 +-- tests/unit/config/config_test.py | 36 +++++++++++++------------ tests/unit/config/interpolation_test.py | 9 ++++--- 8 files changed, 32 insertions(+), 41 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ae45decb..462fcc4c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ - id: end-of-file-fixer - id: flake8 - id: name-tests-test - exclude: 'tests/(integration/testcases\.py)|(helpers\.py)' + exclude: 'tests/integration/testcases\.py' - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9d50ea99f..707c24926 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -15,7 +15,7 @@ from operator import attrgetter import yaml from docker import errors -from ..helpers import clear_environment +from .. import mock from compose.cli.command import get_project from compose.container import Container from compose.project import OneOffFilter @@ -1452,7 +1452,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn("FOO=1", containers[0].get('Config.Env')) - @clear_environment + @mock.patch.dict(os.environ) def test_home_and_env_var_in_volume_path(self): os.environ['VOLUME_NAME'] = 'my-volume' os.environ['HOME'] = '/tmp/home-dir' diff --git a/tests/helpers.py b/tests/helpers.py index 2c3d5a98b..dd0b668ed 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,14 +1,9 @@ from __future__ import absolute_import from __future__ import unicode_literals -import functools -import os - -from . import mock from compose.config.config import ConfigDetails from compose.config.config import ConfigFile from compose.config.config import load -from compose.config.environment import Environment def build_config(contents, **kwargs): @@ -19,11 +14,3 @@ def build_config_details(contents, working_dir='working_dir', filename='filename return ConfigDetails( working_dir, [ConfigFile(filename, contents)]) - - -def clear_environment(f): - @functools.wraps(f) - def wrapper(self, *args, **kwargs): - Environment.reset() - with mock.patch.dict(os.environ): - f(self, *args, **kwargs) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a1857b58a..e2ef1161d 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,7 +12,6 @@ from six import StringIO from six import text_type from .. import mock -from ..helpers import clear_environment from .testcases import DockerClientTestCase from .testcases import get_links from .testcases import pull_busybox @@ -913,7 +912,7 @@ class ServiceTest(DockerClientTestCase): }.items(): self.assertEqual(env[k], v) - @clear_environment + @mock.patch.dict(os.environ) def test_resolve_env(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8e2f25937..aaa002f34 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -12,6 +12,7 @@ from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 +from compose.config.environment import Environment from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output @@ -89,7 +90,7 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - kwargs['environment'] = resolve_environment(kwargs) + kwargs['environment'] = resolve_environment(kwargs, Environment(None)) labels = dict(kwargs.setdefault('labels', {})) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index fd8aa95c1..d8e0b33fb 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -11,7 +11,6 @@ import pytest from .. import mock from .. import unittest from ..helpers import build_config -from ..helpers import clear_environment from compose.cli.command import get_project from compose.cli.command import get_project_name from compose.cli.docopt_command import NoSuchCommand @@ -44,7 +43,7 @@ class CLITestCase(unittest.TestCase): project_name = get_project_name(None, project_name=name) self.assertEquals('explicitprojectname', project_name) - @clear_environment + @mock.patch.dict(os.environ) def test_project_name_from_environment_new_var(self): name = 'namefromenv' os.environ['COMPOSE_PROJECT_NAME'] = name diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 9bd76fb9a..6dc7dbcad 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -17,13 +17,13 @@ from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 +from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest -from tests.helpers import clear_environment DEFAULT_VERSION = V2_0 @@ -1582,7 +1582,7 @@ class PortsTest(unittest.TestCase): class InterpolationTest(unittest.TestCase): - @clear_environment + @mock.patch.dict(os.environ) def test_config_file_with_environment_variable(self): os.environ.update( IMAGE="busybox", @@ -1605,7 +1605,7 @@ class InterpolationTest(unittest.TestCase): } ]) - @clear_environment + @mock.patch.dict(os.environ) def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) os.environ.pop('BAR', None) @@ -1621,7 +1621,7 @@ class InterpolationTest(unittest.TestCase): None, ) - with mock.patch('compose.config.interpolation.log') as log: + with mock.patch('compose.config.environment.log') as log: config.load(config_details) self.assertEqual(2, log.warn.call_count) @@ -1629,7 +1629,7 @@ class InterpolationTest(unittest.TestCase): self.assertIn('BAR', warnings[0]) self.assertIn('FOO', warnings[1]) - @clear_environment + @mock.patch.dict(os.environ) def test_invalid_interpolation(self): with self.assertRaises(config.ConfigurationError) as cm: config.load( @@ -1668,7 +1668,7 @@ class VolumeConfigTest(unittest.TestCase): d = make_service_dict('foo', {'build': '.', 'volumes': ['/data']}, working_dir='.') self.assertEqual(d['volumes'], ['/data']) - @clear_environment + @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' @@ -1682,7 +1682,7 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') - @clear_environment + @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): os.environ['HOME'] = '/home/user' d = make_service_dict('foo', {'build': '.', 'volumes': ['~:/container/path']}, working_dir='.') @@ -1740,7 +1740,7 @@ class VolumeConfigTest(unittest.TestCase): working_dir='c:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['c:\\Users\\me\\otherproject:/data']) - @clear_environment + @mock.patch.dict(os.environ) def test_home_directory_with_driver_does_not_expand(self): os.environ['NAME'] = 'surprise!' d = make_service_dict('foo', { @@ -2026,7 +2026,7 @@ class EnvTest(unittest.TestCase): def test_parse_environment_empty(self): self.assertEqual(config.parse_environment(None), {}) - @clear_environment + @mock.patch.dict(os.environ) def test_resolve_environment(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' @@ -2042,7 +2042,7 @@ class EnvTest(unittest.TestCase): }, } self.assertEqual( - resolve_environment(service_dict), + resolve_environment(service_dict, Environment(None)), {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': None}, ) @@ -2073,13 +2073,15 @@ class EnvTest(unittest.TestCase): assert 'Couldn\'t find env file' in exc.exconly() assert 'nonexistent.env' in exc.exconly() - @clear_environment + @mock.patch.dict(os.environ) def test_resolve_environment_from_env_file_with_empty_values(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' os.environ['ENV_DEF'] = 'E3' self.assertEqual( - resolve_environment({'env_file': ['tests/fixtures/env/resolve.env']}), + resolve_environment( + {'env_file': ['tests/fixtures/env/resolve.env']}, Environment(None) + ), { 'FILE_DEF': u'bär', 'FILE_DEF_EMPTY': '', @@ -2088,7 +2090,7 @@ class EnvTest(unittest.TestCase): }, ) - @clear_environment + @mock.patch.dict(os.environ) def test_resolve_build_args(self): os.environ['env_arg'] = 'value2' @@ -2102,12 +2104,12 @@ class EnvTest(unittest.TestCase): } } self.assertEqual( - resolve_build_args(build), + resolve_build_args(build, Environment(build['context'])), {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': None}, ) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') - @clear_environment + @mock.patch.dict(os.environ) def test_resolve_path(self): os.environ['HOSTENV'] = '/tmp' os.environ['CONTAINERENV'] = '/host/tmp' @@ -2394,7 +2396,7 @@ class ExtendsTest(unittest.TestCase): assert 'net: container' in excinfo.exconly() assert 'cannot be extended' in excinfo.exconly() - @clear_environment + @mock.patch.dict(os.environ) def test_load_config_runs_interpolation_in_extended_service(self): os.environ.update(HOSTNAME_VALUE="penguin") expected_interpolated_value = "host-penguin" @@ -2466,7 +2468,7 @@ class ExtendsTest(unittest.TestCase): }, ])) - @clear_environment + @mock.patch.dict(os.environ) def test_extends_with_environment_and_env_files(self): tmpdir = py.test.ensuretemp('test_extends_with_environment') self.addCleanup(tmpdir.remove) diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index b4ba7b402..f83caea91 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -13,7 +13,6 @@ from compose.config.interpolation import interpolate_environment_variables @pytest.yield_fixture def mock_env(): with mock.patch.dict(os.environ): - Environment.reset() os.environ['USER'] = 'jenny' os.environ['FOO'] = 'bar' yield @@ -44,7 +43,9 @@ def test_interpolate_environment_variables_in_services(mock_env): } } } - assert interpolate_environment_variables(services, 'service') == expected + assert interpolate_environment_variables( + services, 'service', Environment(None) + ) == expected def test_interpolate_environment_variables_in_volumes(mock_env): @@ -68,4 +69,6 @@ def test_interpolate_environment_variables_in_volumes(mock_env): }, 'other': {}, } - assert interpolate_environment_variables(volumes, 'volume') == expected + assert interpolate_environment_variables( + volumes, 'volume', Environment(None) + ) == expected From fd020ed2cfa2fb2b03a31a3fe87d6da47edfd713 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 7 Mar 2016 17:46:55 -0800 Subject: [PATCH 05/16] Tests use updated get_config_paths_from_options signature Signed-off-by: Joffrey F --- .pre-commit-config.yaml | 2 +- tests/unit/cli/command_test.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 462fcc4c6..0e7b9d5f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ - id: end-of-file-fixer - id: flake8 - id: name-tests-test - exclude: 'tests/integration/testcases\.py' + exclude: 'tests/(integration/testcases\.py|helpers\.py)' - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index b524a5f3b..11fea16f6 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -15,24 +15,24 @@ class TestGetConfigPathFromOptions(object): def test_path_from_options(self): paths = ['one.yml', 'two.yml'] opts = {'--file': paths} - assert get_config_path_from_options(opts) == paths + assert get_config_path_from_options('.', opts) == paths def test_single_path_from_env(self): with mock.patch.dict(os.environ): os.environ['COMPOSE_FILE'] = 'one.yml' - assert get_config_path_from_options({}) == ['one.yml'] + assert get_config_path_from_options('.', {}) == ['one.yml'] @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix separator') def test_multiple_path_from_env(self): with mock.patch.dict(os.environ): os.environ['COMPOSE_FILE'] = 'one.yml:two.yml' - assert get_config_path_from_options({}) == ['one.yml', 'two.yml'] + assert get_config_path_from_options('.', {}) == ['one.yml', 'two.yml'] @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows separator') def test_multiple_path_from_env_windows(self): with mock.patch.dict(os.environ): os.environ['COMPOSE_FILE'] = 'one.yml;two.yml' - assert get_config_path_from_options({}) == ['one.yml', 'two.yml'] + assert get_config_path_from_options('.', {}) == ['one.yml', 'two.yml'] def test_no_path(self): - assert not get_config_path_from_options({}) + assert not get_config_path_from_options('.', {}) From 1801f83bb83f59fd508e5f9ad85b4c14d1f9d1d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 8 Mar 2016 16:54:14 -0800 Subject: [PATCH 06/16] Environment class cleanup Signed-off-by: Joffrey F --- compose/cli/command.py | 6 +-- compose/config/config.py | 35 ++----------- compose/config/environment.py | 69 +++++++++++++++---------- tests/integration/testcases.py | 2 +- tests/unit/config/config_test.py | 6 +-- tests/unit/config/interpolation_test.py | 4 +- tests/unit/interpolation_test.py | 2 +- 7 files changed, 58 insertions(+), 66 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 8c0bf07ff..07293f74a 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -34,7 +34,7 @@ def get_config_path_from_options(base_dir, options): if file_option: return file_option - environment = config.environment.Environment(base_dir) + environment = config.environment.Environment.from_env_file(base_dir) config_files = environment.get('COMPOSE_FILE') if config_files: return config_files.split(os.pathsep) @@ -58,7 +58,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(project_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) - environment = config.environment.Environment(project_dir) + environment = config.environment.Environment.from_env_file(project_dir) api_version = environment.get( 'COMPOSE_API_VERSION', @@ -75,7 +75,7 @@ def get_project_name(working_dir, project_name=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - environment = config.environment.Environment(working_dir) + environment = config.environment.Environment.from_env_file(working_dir) project_name = project_name or environment.get('COMPOSE_PROJECT_NAME') if project_name: return normalize_name(project_name) diff --git a/compose/config/config.py b/compose/config/config.py index 7db66004f..a50efdf8e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,7 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -import codecs import functools import logging import operator @@ -17,7 +16,9 @@ from cached_property import cached_property from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 from ..utils import build_string_dict +from .environment import env_vars_from_file from .environment import Environment +from .environment import split_env from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError @@ -129,7 +130,7 @@ class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files envir cls, working_dir, config_files, - Environment(working_dir), + Environment.from_env_file(working_dir), ) @@ -314,9 +315,7 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) - service_dicts = load_services( - config_details, main_file, - ) + service_dicts = load_services(config_details, main_file) if main_file.version != V1: for service_dict in service_dicts: @@ -455,7 +454,7 @@ class ServiceExtendsResolver(object): self.working_dir = service_config.working_dir self.already_seen = already_seen or [] self.config_file = config_file - self.environment = environment or Environment(None) + self.environment = environment or Environment() @property def signature(self): @@ -802,15 +801,6 @@ def merge_environment(base, override): return env -def split_env(env): - if isinstance(env, six.binary_type): - env = env.decode('utf-8', 'replace') - if '=' in env: - return env.split('=', 1) - else: - return env, None - - def split_label(label): if '=' in label: return label.split('=', 1) @@ -857,21 +847,6 @@ def resolve_env_var(key, val, environment): return key, None -def env_vars_from_file(filename): - """ - Read in a line delimited file of environment variables. - """ - if not os.path.exists(filename): - raise ConfigurationError("Couldn't find env file: %s" % filename) - env = {} - for line in codecs.open(filename, 'r', 'utf-8'): - line = line.strip() - if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v - return env - - def resolve_volume_paths(working_dir, service_dict): return [ resolve_volume_path(working_dir, volume) diff --git a/compose/config/environment.py b/compose/config/environment.py index 87b41223b..8066c50fc 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -1,22 +1,62 @@ from __future__ import absolute_import from __future__ import unicode_literals +import codecs import logging import os +import six + from .errors import ConfigurationError log = logging.getLogger(__name__) -class BlankDefaultDict(dict): +def split_env(env): + if isinstance(env, six.binary_type): + env = env.decode('utf-8', 'replace') + if '=' in env: + return env.split('=', 1) + else: + return env, None + + +def env_vars_from_file(filename): + """ + Read in a line delimited file of environment variables. + """ + if not os.path.exists(filename): + raise ConfigurationError("Couldn't find env file: %s" % filename) + env = {} + for line in codecs.open(filename, 'r', 'utf-8'): + line = line.strip() + if line and not line.startswith('#'): + k, v = split_env(line) + env[k] = v + return env + + +class Environment(dict): def __init__(self, *args, **kwargs): - super(BlankDefaultDict, self).__init__(*args, **kwargs) + super(Environment, self).__init__(*args, **kwargs) self.missing_keys = [] + self.update(os.environ) + + @classmethod + def from_env_file(cls, base_dir): + result = cls() + if base_dir is None: + return result + env_file_path = os.path.join(base_dir, '.env') + try: + result.update(env_vars_from_file(env_file_path)) + except ConfigurationError: + pass + return result def __getitem__(self, key): try: - return super(BlankDefaultDict, self).__getitem__(key) + return super(Environment, self).__getitem__(key) except KeyError: if key not in self.missing_keys: log.warn( @@ -26,26 +66,3 @@ class BlankDefaultDict(dict): self.missing_keys.append(key) return "" - - -class Environment(BlankDefaultDict): - def __init__(self, base_dir): - super(Environment, self).__init__() - if base_dir: - self.load_environment_file(os.path.join(base_dir, '.env')) - self.update(os.environ) - - def load_environment_file(self, path): - if not os.path.exists(path): - return - mapping = {} - with open(path, 'r') as f: - for line in f.readlines(): - line = line.strip() - if '=' not in line: - raise ConfigurationError( - 'Invalid environment variable mapping in env file. ' - 'Missing "=" in "{0}"'.format(line) - ) - mapping.__setitem__(*line.split('=', 1)) - self.update(mapping) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index aaa002f34..98e0540f1 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -90,7 +90,7 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - kwargs['environment'] = resolve_environment(kwargs, Environment(None)) + kwargs['environment'] = resolve_environment(kwargs, Environment()) labels = dict(kwargs.setdefault('labels', {})) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 6dc7dbcad..daf724a88 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2042,7 +2042,7 @@ class EnvTest(unittest.TestCase): }, } self.assertEqual( - resolve_environment(service_dict, Environment(None)), + resolve_environment(service_dict, Environment()), {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': None}, ) @@ -2080,7 +2080,7 @@ class EnvTest(unittest.TestCase): os.environ['ENV_DEF'] = 'E3' self.assertEqual( resolve_environment( - {'env_file': ['tests/fixtures/env/resolve.env']}, Environment(None) + {'env_file': ['tests/fixtures/env/resolve.env']}, Environment() ), { 'FILE_DEF': u'bär', @@ -2104,7 +2104,7 @@ class EnvTest(unittest.TestCase): } } self.assertEqual( - resolve_build_args(build, Environment(build['context'])), + resolve_build_args(build, Environment.from_env_file(build['context'])), {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': None}, ) diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index f83caea91..282ba779a 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -44,7 +44,7 @@ def test_interpolate_environment_variables_in_services(mock_env): } } assert interpolate_environment_variables( - services, 'service', Environment(None) + services, 'service', Environment() ) == expected @@ -70,5 +70,5 @@ def test_interpolate_environment_variables_in_volumes(mock_env): 'other': {}, } assert interpolate_environment_variables( - volumes, 'volume', Environment(None) + volumes, 'volume', Environment() ) == expected diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py index b19fcdaca..c3050c2ca 100644 --- a/tests/unit/interpolation_test.py +++ b/tests/unit/interpolation_test.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import unittest -from compose.config.environment import BlankDefaultDict as bddict +from compose.config.environment import Environment as bddict from compose.config.interpolation import interpolate from compose.config.interpolation import InvalidInterpolation From d55fc85feadf51e753c06b34f7cc200305915a54 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 8 Mar 2016 16:55:05 -0800 Subject: [PATCH 07/16] Added default env file test. Signed-off-by: Joffrey F --- tests/fixtures/default-env-file/.env | 4 ++++ tests/fixtures/default-env-file/docker-compose.yml | 6 ++++++ tests/unit/config/config_test.py | 13 +++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 tests/fixtures/default-env-file/.env create mode 100644 tests/fixtures/default-env-file/docker-compose.yml diff --git a/tests/fixtures/default-env-file/.env b/tests/fixtures/default-env-file/.env new file mode 100644 index 000000000..996c886cb --- /dev/null +++ b/tests/fixtures/default-env-file/.env @@ -0,0 +1,4 @@ +IMAGE=alpine:latest +COMMAND=true +PORT1=5643 +PORT2=9999 \ No newline at end of file diff --git a/tests/fixtures/default-env-file/docker-compose.yml b/tests/fixtures/default-env-file/docker-compose.yml new file mode 100644 index 000000000..aa8e4409e --- /dev/null +++ b/tests/fixtures/default-env-file/docker-compose.yml @@ -0,0 +1,6 @@ +web: + image: ${IMAGE} + command: ${COMMAND} + ports: + - $PORT1 + - $PORT2 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index daf724a88..913cbed9c 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1582,6 +1582,19 @@ class PortsTest(unittest.TestCase): class InterpolationTest(unittest.TestCase): + @mock.patch.dict(os.environ) + def test_config_file_with_environment_file(self): + service_dicts = config.load( + config.find('tests/fixtures/default-env-file', None) + ).services + + self.assertEqual(service_dicts[0], { + 'name': 'web', + 'image': 'alpine:latest', + 'ports': ['5643', '9999'], + 'command': 'true' + }) + @mock.patch.dict(os.environ) def test_config_file_with_environment_variable(self): os.environ.update( From f48da96e8b5a732399d5ab18e32c53156934e694 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 8 Mar 2016 17:18:04 -0800 Subject: [PATCH 08/16] Test get_project_name from env file Signed-off-by: Joffrey F --- compose/config/environment.py | 2 +- tests/unit/cli_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 8066c50fc..17eacf1f5 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -49,7 +49,7 @@ class Environment(dict): return result env_file_path = os.path.join(base_dir, '.env') try: - result.update(env_vars_from_file(env_file_path)) + return cls(env_vars_from_file(env_file_path)) except ConfigurationError: pass return result diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index d8e0b33fb..bd35dc06f 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -3,6 +3,8 @@ from __future__ import absolute_import from __future__ import unicode_literals import os +import shutil +import tempfile import docker import py @@ -57,6 +59,22 @@ class CLITestCase(unittest.TestCase): project_name = get_project_name(base_dir) self.assertEquals('simplecomposefile', project_name) + @mock.patch.dict(os.environ) + def test_project_name_with_environment_file(self): + base_dir = tempfile.mkdtemp() + try: + name = 'namefromenvfile' + with open(os.path.join(base_dir, '.env'), 'w') as f: + f.write('COMPOSE_PROJECT_NAME={}'.format(name)) + project_name = get_project_name(base_dir) + assert project_name == name + + # Environment has priority over .env file + os.environ['COMPOSE_PROJECT_NAME'] = 'namefromenv' + assert get_project_name(base_dir) == os.environ['COMPOSE_PROJECT_NAME'] + finally: + shutil.rmtree(base_dir) + def test_get_project(self): base_dir = 'tests/fixtures/longer-filename-composefile' project = get_project(base_dir) From 21aa7a0448703e3e59bc39300a82f68ded659f45 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 8 Mar 2016 17:48:23 -0800 Subject: [PATCH 09/16] Documentation for .env file Signed-off-by: Joffrey F --- docs/env-file.md | 38 ++++++++++++++++++++++++++++++++++++++ docs/index.md | 1 + 2 files changed, 39 insertions(+) create mode 100644 docs/env-file.md diff --git a/docs/env-file.md b/docs/env-file.md new file mode 100644 index 000000000..3dec592f3 --- /dev/null +++ b/docs/env-file.md @@ -0,0 +1,38 @@ + + + +# Environment file + +Compose supports declaring default environment variables in an environment +file named `.env` and placed in the same folder as your +[compose file](compose-file.md). + +Compose expects each line in an env file to be in `VAR=VAL` format. Lines +beginning with `#` (i.e. comments) are ignored, as are blank lines. + +> Note: Values present in the environment at runtime will always override +> those defined inside the `.env` file. + +Those environment variables will be used for +[variable substitution](compose-file.md#variable-substitution) in your Compose +file, but can also be used to define the following +[CLI variables](reference/envvars.md): + +- `COMPOSE_PROJECT_NAME` +- `COMPOSE_FILE` +- `COMPOSE_API_VERSION` + +## More Compose documentation + +- [User guide](index.md) +- [Command line reference](./reference/index.md) +- [Compose file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index f5d84218f..f1b710794 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,6 +23,7 @@ Compose is a tool for defining and running multi-container Docker applications. - [Frequently asked questions](faq.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) +- [Environment file](env-file.md) To see a detailed list of changes for past and current releases of Docker Compose, please refer to the From dcdcf4869b6df77e16e243ace9e49c136d336b78 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 10 Mar 2016 17:29:01 -0800 Subject: [PATCH 10/16] Mention environment file in envvars.md Signed-off-by: Joffrey F --- docs/reference/envvars.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/reference/envvars.md b/docs/reference/envvars.md index e1170be90..6f7fb7919 100644 --- a/docs/reference/envvars.md +++ b/docs/reference/envvars.md @@ -17,6 +17,9 @@ Several environment variables are available for you to configure the Docker Comp Variables starting with `DOCKER_` are the same as those used to configure the Docker command-line client. If you're using `docker-machine`, then the `eval "$(docker-machine env my-docker-vm)"` command should set them to their correct values. (In this example, `my-docker-vm` is the name of a machine you created.) +> Note: Some of these variables can also be provided using an +> [environment file](../env-file.md) + ## COMPOSE\_PROJECT\_NAME Sets the project name. This value is prepended along with the service name to the container container on start up. For example, if you project name is `myapp` and it includes two services `db` and `web` then compose starts containers named `myapp_db_1` and `myapp_web_1` respectively. @@ -81,3 +84,4 @@ it failed. Defaults to 60 seconds. - [User guide](../index.md) - [Installing Compose](../install.md) - [Compose file reference](../compose-file.md) +- [Environment file](../env-file.md) From 0ff53d9668670efe99736e045d4476bccf9435ca Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 14 Mar 2016 13:02:55 -0700 Subject: [PATCH 11/16] Less verbose environment invocation Signed-off-by: Joffrey F --- compose/cli/command.py | 7 ++++--- docs/env-file.md | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 07293f74a..d726c7b3d 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -9,6 +9,7 @@ import six from . import verbose_proxy from .. import config +from ..config.environment import Environment from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client @@ -34,7 +35,7 @@ def get_config_path_from_options(base_dir, options): if file_option: return file_option - environment = config.environment.Environment.from_env_file(base_dir) + environment = Environment.from_env_file(base_dir) config_files = environment.get('COMPOSE_FILE') if config_files: return config_files.split(os.pathsep) @@ -58,7 +59,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(project_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) - environment = config.environment.Environment.from_env_file(project_dir) + environment = Environment.from_env_file(project_dir) api_version = environment.get( 'COMPOSE_API_VERSION', @@ -75,7 +76,7 @@ def get_project_name(working_dir, project_name=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - environment = config.environment.Environment.from_env_file(working_dir) + environment = Environment.from_env_file(working_dir) project_name = project_name or environment.get('COMPOSE_PROJECT_NAME') if project_name: return normalize_name(project_name) diff --git a/docs/env-file.md b/docs/env-file.md index 3dec592f3..6d12d228c 100644 --- a/docs/env-file.md +++ b/docs/env-file.md @@ -20,7 +20,8 @@ Compose expects each line in an env file to be in `VAR=VAL` format. Lines beginning with `#` (i.e. comments) are ignored, as are blank lines. > Note: Values present in the environment at runtime will always override -> those defined inside the `.env` file. +> those defined inside the `.env` file. Similarly, values passed via +> command-line arguments take precedence as well. Those environment variables will be used for [variable substitution](compose-file.md#variable-substitution) in your Compose From 36f1b4589cd0dfd343c0b597abe56824d95cea09 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Mar 2016 16:08:07 -0700 Subject: [PATCH 12/16] Limit occurrences of creating an environment object. .env file is always read from the project_dir Signed-off-by: Joffrey F --- compose/cli/command.py | 23 ++++++++++++++--------- compose/cli/main.py | 10 ++++++++-- compose/config/config.py | 14 +++++++------- tests/helpers.py | 3 ++- tests/unit/cli/command_test.py | 20 +++++++++++++++----- tests/unit/config/config_test.py | 14 +++++++++++--- 6 files changed, 57 insertions(+), 27 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index d726c7b3d..73eccc96c 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -20,22 +20,23 @@ log = logging.getLogger(__name__) def project_from_options(project_dir, options): + environment = Environment.from_env_file(project_dir) return get_project( project_dir, - get_config_path_from_options(project_dir, options), + get_config_path_from_options(project_dir, options, environment), project_name=options.get('--project-name'), verbose=options.get('--verbose'), host=options.get('--host'), tls_config=tls_config_from_options(options), + environment=environment ) -def get_config_path_from_options(base_dir, options): +def get_config_path_from_options(base_dir, options, environment): file_option = options.get('--file') if file_option: return file_option - environment = Environment.from_env_file(base_dir) config_files = environment.get('COMPOSE_FILE') if config_files: return config_files.split(os.pathsep) @@ -55,11 +56,14 @@ def get_client(verbose=False, version=None, tls_config=None, host=None): def get_project(project_dir, config_path=None, project_name=None, verbose=False, - host=None, tls_config=None): - config_details = config.find(project_dir, config_path) - project_name = get_project_name(config_details.working_dir, project_name) + host=None, tls_config=None, environment=None): + if not environment: + environment = Environment.from_env_file(project_dir) + config_details = config.find(project_dir, config_path, environment) + project_name = get_project_name( + config_details.working_dir, project_name, environment + ) config_data = config.load(config_details) - environment = Environment.from_env_file(project_dir) api_version = environment.get( 'COMPOSE_API_VERSION', @@ -72,11 +76,12 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, return Project.from_config(project_name, config_data, client) -def get_project_name(working_dir, project_name=None): +def get_project_name(working_dir, project_name=None, environment=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - environment = Environment.from_env_file(working_dir) + if not environment: + environment = Environment.from_env_file(working_dir) project_name = project_name or environment.get('COMPOSE_PROJECT_NAME') if project_name: return normalize_name(project_name) diff --git a/compose/cli/main.py b/compose/cli/main.py index 3fa3e3a01..8348b8c37 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -17,6 +17,7 @@ from .. import __version__ from ..config import config from ..config import ConfigurationError from ..config import parse_environment +from ..config.environment import Environment from ..config.serialize import serialize_config from ..const import DEFAULT_TIMEOUT from ..const import IS_WINDOWS_PLATFORM @@ -222,8 +223,13 @@ class TopLevelCommand(object): --services Print the service names, one per line. """ - config_path = get_config_path_from_options(self.project_dir, config_options) - compose_config = config.load(config.find(self.project_dir, config_path)) + environment = Environment.from_env_file(self.project_dir) + config_path = get_config_path_from_options( + self.project_dir, config_options, environment + ) + compose_config = config.load( + config.find(self.project_dir, config_path, environment) + ) if options['--quiet']: return diff --git a/compose/config/config.py b/compose/config/config.py index a50efdf8e..47cb23312 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -124,13 +124,11 @@ class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files envir :param environment: computed environment values for this project :type environment: :class:`environment.Environment` """ - - def __new__(cls, working_dir, config_files): + def __new__(cls, working_dir, config_files, environment=None): + if environment is None: + environment = Environment.from_env_file(working_dir) return super(ConfigDetails, cls).__new__( - cls, - working_dir, - config_files, - Environment.from_env_file(working_dir), + cls, working_dir, config_files, environment ) @@ -219,11 +217,12 @@ class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name conf config) -def find(base_dir, filenames): +def find(base_dir, filenames, environment): if filenames == ['-']: return ConfigDetails( os.getcwd(), [ConfigFile(None, yaml.safe_load(sys.stdin))], + environment ) if filenames: @@ -235,6 +234,7 @@ def find(base_dir, filenames): return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile.from_filename(f) for f in filenames], + environment ) diff --git a/tests/helpers.py b/tests/helpers.py index dd0b668ed..4b422a6a0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -13,4 +13,5 @@ def build_config(contents, **kwargs): def build_config_details(contents, working_dir='working_dir', filename='filename.yml'): return ConfigDetails( working_dir, - [ConfigFile(filename, contents)]) + [ConfigFile(filename, contents)], + ) diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 11fea16f6..3502d6369 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -6,6 +6,7 @@ import os import pytest from compose.cli.command import get_config_path_from_options +from compose.config.environment import Environment from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -15,24 +16,33 @@ class TestGetConfigPathFromOptions(object): def test_path_from_options(self): paths = ['one.yml', 'two.yml'] opts = {'--file': paths} - assert get_config_path_from_options('.', opts) == paths + environment = Environment.from_env_file('.') + assert get_config_path_from_options('.', opts, environment) == paths def test_single_path_from_env(self): with mock.patch.dict(os.environ): os.environ['COMPOSE_FILE'] = 'one.yml' - assert get_config_path_from_options('.', {}) == ['one.yml'] + environment = Environment.from_env_file('.') + assert get_config_path_from_options('.', {}, environment) == ['one.yml'] @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix separator') def test_multiple_path_from_env(self): with mock.patch.dict(os.environ): os.environ['COMPOSE_FILE'] = 'one.yml:two.yml' - assert get_config_path_from_options('.', {}) == ['one.yml', 'two.yml'] + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', {}, environment + ) == ['one.yml', 'two.yml'] @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows separator') def test_multiple_path_from_env_windows(self): with mock.patch.dict(os.environ): os.environ['COMPOSE_FILE'] = 'one.yml;two.yml' - assert get_config_path_from_options('.', {}) == ['one.yml', 'two.yml'] + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', {}, environment + ) == ['one.yml', 'two.yml'] def test_no_path(self): - assert not get_config_path_from_options('.', {}) + environment = Environment.from_env_file('.') + assert not get_config_path_from_options('.', {}, environment) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 913cbed9c..111528715 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1584,8 +1584,11 @@ class PortsTest(unittest.TestCase): class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_config_file_with_environment_file(self): + project_dir = 'tests/fixtures/default-env-file' service_dicts = config.load( - config.find('tests/fixtures/default-env-file', None) + config.find( + project_dir, None, Environment.from_env_file(project_dir) + ) ).services self.assertEqual(service_dicts[0], { @@ -1597,6 +1600,7 @@ class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_config_file_with_environment_variable(self): + project_dir = 'tests/fixtures/environment-interpolation' os.environ.update( IMAGE="busybox", HOST_PORT="80", @@ -1604,7 +1608,9 @@ class InterpolationTest(unittest.TestCase): ) service_dicts = config.load( - config.find('tests/fixtures/environment-interpolation', None), + config.find( + project_dir, None, Environment.from_env_file(project_dir) + ) ).services self.assertEqual(service_dicts, [ @@ -2149,7 +2155,9 @@ class EnvTest(unittest.TestCase): def load_from_filename(filename): - return config.load(config.find('.', [filename])).services + return config.load( + config.find('.', [filename], Environment.from_env_file('.')) + ).services class ExtendsTest(unittest.TestCase): From c7afe16419945d74f956fa065f7b9f79712ed626 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Mar 2016 16:33:58 -0700 Subject: [PATCH 13/16] Account for case-insensitive env on windows platform Signed-off-by: Joffrey F --- compose/config/environment.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/config/environment.py b/compose/config/environment.py index 17eacf1f5..7f7269b7c 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -7,6 +7,7 @@ import os import six +from ..const import IS_WINDOWS_PLATFORM from .errors import ConfigurationError log = logging.getLogger(__name__) @@ -58,6 +59,11 @@ class Environment(dict): try: return super(Environment, self).__getitem__(key) except KeyError: + if IS_WINDOWS_PLATFORM: + try: + return super(Environment, self).__getitem__(key.upper()) + except KeyError: + pass if key not in self.missing_keys: log.warn( "The {} variable is not set. Defaulting to a blank string." From b99037b4a61e10c9377dd707e35860cec298a268 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Mar 2016 18:32:13 -0700 Subject: [PATCH 14/16] Add support for DOCKER_* variables in .env file Signed-off-by: Joffrey F --- compose/cli/command.py | 9 ++++++--- compose/cli/docker_client.py | 13 ++++++++----- compose/const.py | 2 +- docs/env-file.md | 8 ++++++-- tests/integration/testcases.py | 2 +- tests/unit/cli/docker_client_test.py | 4 ++-- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 73eccc96c..b7160deec 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -43,8 +43,11 @@ def get_config_path_from_options(base_dir, options, environment): return None -def get_client(verbose=False, version=None, tls_config=None, host=None): - client = docker_client(version=version, tls_config=tls_config, host=host) +def get_client(environment, verbose=False, version=None, tls_config=None, host=None): + client = docker_client( + version=version, tls_config=tls_config, host=host, + environment=environment + ) if verbose: version_info = six.iteritems(client.version()) log.info(get_version_info('full')) @@ -70,7 +73,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, API_VERSIONS[config_data.version]) client = get_client( verbose=verbose, version=api_version, tls_config=tls_config, - host=host + host=host, environment=environment ) return Project.from_config(project_name, config_data, client) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index deb568660..f782a1ae6 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -2,7 +2,6 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -import os from docker import Client from docker.errors import TLSParameterError @@ -42,17 +41,17 @@ def tls_config_from_options(options): return None -def docker_client(version=None, tls_config=None, host=None): +def docker_client(environment, version=None, tls_config=None, host=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ - if 'DOCKER_CLIENT_TIMEOUT' in os.environ: + if 'DOCKER_CLIENT_TIMEOUT' in environment: log.warn("The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. " "Please use COMPOSE_HTTP_TIMEOUT instead.") try: - kwargs = kwargs_from_env(assert_hostname=False) + kwargs = kwargs_from_env(assert_hostname=False, environment=environment) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " @@ -67,6 +66,10 @@ def docker_client(version=None, tls_config=None, host=None): if version: kwargs['version'] = version - kwargs['timeout'] = HTTP_TIMEOUT + timeout = environment.get('COMPOSE_HTTP_TIMEOUT') + if timeout: + kwargs['timeout'] = int(timeout) + else: + kwargs['timeout'] = HTTP_TIMEOUT return Client(**kwargs) diff --git a/compose/const.py b/compose/const.py index db5e2fb4f..9e00d96e9 100644 --- a/compose/const.py +++ b/compose/const.py @@ -5,7 +5,7 @@ import os import sys DEFAULT_TIMEOUT = 10 -HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) +HTTP_TIMEOUT = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60)) IMAGE_EVENTS = ['delete', 'import', 'pull', 'push', 'tag', 'untag'] IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' diff --git a/docs/env-file.md b/docs/env-file.md index 6d12d228c..a285a7908 100644 --- a/docs/env-file.md +++ b/docs/env-file.md @@ -28,9 +28,13 @@ Those environment variables will be used for file, but can also be used to define the following [CLI variables](reference/envvars.md): -- `COMPOSE_PROJECT_NAME` -- `COMPOSE_FILE` - `COMPOSE_API_VERSION` +- `COMPOSE_FILE` +- `COMPOSE_HTTP_TIMEOUT` +- `COMPOSE_PROJECT_NAME` +- `DOCKER_CERT_PATH` +- `DOCKER_HOST` +- `DOCKER_TLS_VERIFY` ## More Compose documentation diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 98e0540f1..e8b2f35dc 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -61,7 +61,7 @@ class DockerClientTestCase(unittest.TestCase): else: version = API_VERSIONS[V2_0] - cls.client = docker_client(version) + cls.client = docker_client(Environment(), version) def tearDown(self): for c in self.client.containers( diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index b55f1d179..56bab19c3 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -17,12 +17,12 @@ class DockerClientTestCase(unittest.TestCase): def test_docker_client_no_home(self): with mock.patch.dict(os.environ): del os.environ['HOME'] - docker_client() + docker_client(os.environ) def test_docker_client_with_custom_timeout(self): timeout = 300 with mock.patch('compose.cli.docker_client.HTTP_TIMEOUT', 300): - client = docker_client() + client = docker_client(os.environ) self.assertEqual(client.timeout, int(timeout)) From 1506f997def80fdfc4cf6e0377a1cabeaad35d43 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 22 Mar 2016 11:43:03 -0700 Subject: [PATCH 15/16] Better windows support for Environment class Signed-off-by: Joffrey F --- compose/config/environment.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/compose/config/environment.py b/compose/config/environment.py index 7f7269b7c..8d2f5e365 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -72,3 +72,19 @@ class Environment(dict): self.missing_keys.append(key) return "" + + def __contains__(self, key): + result = super(Environment, self).__contains__(key) + if IS_WINDOWS_PLATFORM: + return ( + result or super(Environment, self).__contains__(key.upper()) + ) + return result + + def get(self, key, *args, **kwargs): + if IS_WINDOWS_PLATFORM: + return super(Environment, self).get( + key, + super(Environment, self).get(key.upper(), *args, **kwargs) + ) + return super(Environment, self).get(key, *args, **kwargs) From 12ad3ff30194e8b4cd5d5e8874fffba297f09dc4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 22 Mar 2016 15:42:30 -0700 Subject: [PATCH 16/16] Injecting os.environ in Environment instance happens outside of init method Signed-off-by: Joffrey F --- compose/config/config.py | 4 ++-- compose/config/environment.py | 21 ++++++++++++--------- tests/integration/testcases.py | 4 +++- tests/unit/config/config_test.py | 11 ++++++++--- tests/unit/config/interpolation_test.py | 8 ++++---- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 47cb23312..dc3f56ea9 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -449,12 +449,12 @@ def process_config_file(config_file, environment, service_name=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, config_file, environment=None, already_seen=None): + def __init__(self, service_config, config_file, environment, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] self.config_file = config_file - self.environment = environment or Environment() + self.environment = environment @property def signature(self): diff --git a/compose/config/environment.py b/compose/config/environment.py index 8d2f5e365..ad5c0b3da 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -41,19 +41,22 @@ class Environment(dict): def __init__(self, *args, **kwargs): super(Environment, self).__init__(*args, **kwargs) self.missing_keys = [] - self.update(os.environ) @classmethod def from_env_file(cls, base_dir): - result = cls() - if base_dir is None: + def _initialize(): + result = cls() + if base_dir is None: + return result + env_file_path = os.path.join(base_dir, '.env') + try: + return cls(env_vars_from_file(env_file_path)) + except ConfigurationError: + pass return result - env_file_path = os.path.join(base_dir, '.env') - try: - return cls(env_vars_from_file(env_file_path)) - except ConfigurationError: - pass - return result + instance = _initialize() + instance.update(os.environ) + return instance def __getitem__(self, key): try: diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index e8b2f35dc..8d69d5319 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -90,7 +90,9 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - kwargs['environment'] = resolve_environment(kwargs, Environment()) + kwargs['environment'] = resolve_environment( + kwargs, Environment.from_env_file(None) + ) labels = dict(kwargs.setdefault('labels', {})) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 111528715..2bbbe6145 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -37,7 +37,9 @@ def make_service_dict(name, service_dict, working_dir, filename=None): filename=filename, name=name, config=service_dict), - config.ConfigFile(filename=filename, config={})) + config.ConfigFile(filename=filename, config={}), + environment=Environment.from_env_file(working_dir) + ) return config.process_service(resolver.run()) @@ -2061,7 +2063,9 @@ class EnvTest(unittest.TestCase): }, } self.assertEqual( - resolve_environment(service_dict, Environment()), + resolve_environment( + service_dict, Environment.from_env_file(None) + ), {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': None}, ) @@ -2099,7 +2103,8 @@ class EnvTest(unittest.TestCase): os.environ['ENV_DEF'] = 'E3' self.assertEqual( resolve_environment( - {'env_file': ['tests/fixtures/env/resolve.env']}, Environment() + {'env_file': ['tests/fixtures/env/resolve.env']}, + Environment.from_env_file(None) ), { 'FILE_DEF': u'bär', diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 282ba779a..42b5db6e9 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -20,7 +20,7 @@ def mock_env(): def test_interpolate_environment_variables_in_services(mock_env): services = { - 'servivea': { + 'servicea': { 'image': 'example:${USER}', 'volumes': ['$FOO:/target'], 'logging': { @@ -32,7 +32,7 @@ def test_interpolate_environment_variables_in_services(mock_env): } } expected = { - 'servivea': { + 'servicea': { 'image': 'example:jenny', 'volumes': ['bar:/target'], 'logging': { @@ -44,7 +44,7 @@ def test_interpolate_environment_variables_in_services(mock_env): } } assert interpolate_environment_variables( - services, 'service', Environment() + services, 'service', Environment.from_env_file(None) ) == expected @@ -70,5 +70,5 @@ def test_interpolate_environment_variables_in_volumes(mock_env): 'other': {}, } assert interpolate_environment_variables( - volumes, 'volume', Environment() + volumes, 'volume', Environment.from_env_file(None) ) == expected