Updated COMPOSE_ENV_FILES in env files

Updated the docker compose command to recursively search through any specified .env files for the COMPOSE_ENV_FILES parameter.

- When found, the paths are followed recursively to extend the list of .env files used in the command.
- A cache is kept that prevents circular dependencies.
- Recursion is depth first.

Signed-off-by: Aron Kyle <aron.kyle@lightspeedhq.com>

.gitignore

@@ -3,3 +3,4 @@ bin/
 coverage.out
 covdatafiles/
 .DS_Store
+/.idea

cmd/compose/compose.go

@@ -647,12 +647,12 @@ func RootCommand(dockerCli command.Cli, backend Backend) *cobra.Command { //noli
 	c.Flags().MarkHidden("verbose") //nolint:errcheck
 	return c
 }
-
 func setEnvWithDotEnv(opts ProjectOptions) error {
 	options, err := cli.NewProjectOptions(opts.ConfigPaths,
 		cli.WithWorkingDirectory(opts.ProjectDir),
 		cli.WithOsEnv,
 		cli.WithEnvFiles(opts.EnvFiles...),
+		WithExtendedEnvFiles,
 		cli.WithDotEnv,
 	)
 	if err != nil {
@@ -672,6 +672,88 @@ func setEnvWithDotEnv(opts ProjectOptions) error {
 	return err
 }
 
+func WithExtendedEnvFiles(o *cli.ProjectOptions) error {
+
+	absEnvFiles := make([]string, 0)
+	fileLookupCache := make(map[string]string)
+
+	for _, file := range o.EnvFiles {
+		absFile, err := filepath.Abs(file)
+		if err != nil {
+			return err
+		}
+		absEnvFiles = append(absEnvFiles, absFile)
+		fileLookupCache[absFile] = absFile
+
+		recursedFiles, err := recurseEnvFiles(absFile, fileLookupCache)
+		if err != nil {
+			return err
+		}
+		absEnvFiles = append(absEnvFiles, recursedFiles...)
+	}
+	o.EnvFiles = absEnvFiles
+
+	return nil
+}
+
+func recurseEnvFiles(envFile string, fileLookup map[string]string) ([]string, error) {
+
+	newEnvFiles := make([]string, 0)
+
+	_, err := os.Stat(envFile)
+	// This indicates that the specified file does not exist
+	// In this specific case it's safe to ignore loading the file i.e. the file is optional.
+	if os.IsNotExist(err) {
+		return newEnvFiles, nil
+	}
+
+	// Parse the .env file
+	envFromFile, err := dotenv.GetEnvFromFile(make(map[string]string), []string{envFile})
+	if err != nil {
+		return nil, err
+	}
+
+	// If the file contains a COMPOSE_ENV_FILES key, add the files to the list.
+	// Remote any files that don't exist i.e. the file is optional.
+	// Filter any files we've already seen in the fileLookup.
+	// Depth first recursion into the new files
+	if extraEnvFiles, ok := envFromFile[ComposeEnvFiles]; ok {
+
+		for _, newFile := range strings.Split(extraEnvFiles, ",") {
+			// Handle relative paths
+			if !filepath.IsAbs(newFile) {
+				newFile, err = filepath.Abs(filepath.Join(filepath.Dir(envFile), newFile))
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			_, err := os.Stat(newFile)
+			// This indicates that the specified file does not exist
+			// In this specific case it's safe to ignore using the file as an env file
+			// i.e. the file is optional.
+			if os.IsNotExist(err) {
+				continue
+			}
+
+			// if we haven't seen this file before, add it to the list
+			// and recurse into it
+			if _, ok := fileLookup[newFile]; !ok {
+				newEnvFiles = append(newEnvFiles, newFile)
+				fileLookup[newFile] = newFile
+				recursedFiles, recurseErr := recurseEnvFiles(newFile, fileLookup)
+				if recurseErr != nil {
+					return nil, recurseErr
+				}
+
+				newEnvFiles = append(newEnvFiles, recursedFiles...)
+			}
+		}
+	}
+
+	return newEnvFiles, nil
+}
+
 var printerModes = []string{
 	ui.ModeAuto,
 	ui.ModeTTY,

pkg/e2e/compose_environment_test.go

@@ -171,6 +171,41 @@ func TestEnvPriority(t *testing.T) {
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File")
 	})
 
+	// Recursing through multiple env files based on the COMPOSE_ENV_FILES variable
+	// Chain of env files:
+	// 1. .env
+	// 2. .env.2
+	// 3. .env.3
+	t.Run("recurse env files with COMPOSE_ENV_FILES", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-recursion/compose.yaml",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-recursion")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File 3")
+	})
+
+	// Recursing through multiple env files based on the COMPOSE_ENV_FILES variable
+	// Chain of env files:
+	// 1. .env.3
+	// 2. .env
+	// 3. .env.2
+	t.Run("recurse env files with COMPOSE_ENV_FILES with --env-file", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-recursion/compose.yaml",
+			"--env-file", "./fixtures/environment/env-recursion/.env.3",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-recursion")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File 2")
+	})
+	// Recursing through multiple env files based on the COMPOSE_ENV_FILES variable with missing file
+	// Chain of env files:
+	// 1. .env.test-missing
+	// 2. .env.test-missing.2
+	// 3. .env.test-missing.idontexist -> Skipped because it does not exist
+	// 4. .env.test-missing.3
+	t.Run("recurse env files with COMPOSE_ENV_FILES with --env-file and missing file", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-recursion/compose.yaml",
+			"--env-file", "./fixtures/environment/env-recursion/.env.test-missing",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-recursion")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File Test Missing 3")
+	})
+
 	// No Compose file & no env variable, using an empty override env file
 	// 1. Command Line (docker compose run --env <KEY[=VAL]>)
 	// 2. Compose File (service::environment section)

pkg/e2e/fixtures/environment/env-recursion/.env

@@ -0,0 +1,2 @@
+COMPOSE_ENV_FILES=./.env.2
+WHEREAMI="Env File"

pkg/e2e/fixtures/environment/env-recursion/.env.2

@@ -0,0 +1,2 @@
+COMPOSE_ENV_FILES=.env.3,.env.2
+WHEREAMI="Env File 2"

pkg/e2e/fixtures/environment/env-recursion/.env.3

@@ -0,0 +1,2 @@
+COMPOSE_ENV_FILES=.env
+WHEREAMI="Env File 3"

pkg/e2e/fixtures/environment/env-recursion/.env.test-missing

@@ -0,0 +1,2 @@
+COMPOSE_ENV_FILES=.env.test-missing.2
+WHEREAMI="Env File Test Missing"

pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.2

@@ -0,0 +1,2 @@
+COMPOSE_ENV_FILES=.env.test-missing.idontexist,.env.test-missing.3
+WHEREAMI="Env File Test Missing 2"

pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.3

@@ -0,0 +1 @@
+WHEREAMI="Env File Test Missing 3"

pkg/e2e/fixtures/environment/env-recursion/Dockerfile

@@ -0,0 +1,17 @@
+#   Copyright 2020 Docker Compose CLI authors
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM alpine
+ENV  WHEREAMI=Dockerfile
+CMD ["printenv", "WHEREAMI"]

pkg/e2e/fixtures/environment/env-recursion/compose.yaml

@@ -0,0 +1,5 @@
+services:
+  env-compose-recursion:
+    image: env-compose-recursion
+    build:
+      context: .