include: add experimental support for Git resources (#10811)

Requires setting `COMPOSE_EXPERIMENTAL_GIT_REMOTE=1`.

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

cmd/compose/compose.go

@@ -29,6 +29,7 @@ import (
 	"github.com/compose-spec/compose-go/dotenv"
 	buildx "github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/command"
+	"github.com/docker/compose/v2/pkg/remote"
 
 	"github.com/compose-spec/compose-go/cli"
 	"github.com/compose-spec/compose-go/types"
@@ -134,7 +135,25 @@ func (o *ProjectOptions) WithProject(fn ProjectFunc) func(cmd *cobra.Command, ar
 // WithServices creates a cobra run command from a ProjectFunc based on configured project options and selected services
 func (o *ProjectOptions) WithServices(fn ProjectServicesFunc) func(cmd *cobra.Command, args []string) error {
 	return Adapt(func(ctx context.Context, args []string) error {
-		project, err := o.ToProject(args, cli.WithResolvedPaths(true), cli.WithDiscardEnvFile)
+		options := []cli.ProjectOptionsFn{
+			cli.WithResolvedPaths(true),
+			cli.WithDiscardEnvFile,
+			cli.WithContext(ctx),
+		}
+
+		enabled, err := remote.GitRemoteLoaderEnabled()
+		if err != nil {
+			return err
+		}
+		if enabled {
+			git, err := remote.NewGitRemoteLoader()
+			if err != nil {
+				return err
+			}
+			options = append(options, cli.WithResourceLoader(git))
+		}
+
+		project, err := o.ToProject(args, options...)
 		if err != nil {
 			return err
 		}

cmd/compose/config.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/compose-spec/compose-go/cli"
 	"github.com/compose-spec/compose-go/types"
+	"github.com/docker/compose/v2/pkg/remote"
 	"github.com/spf13/cobra"
 
 	"github.com/docker/compose/v2/pkg/api"
@@ -49,14 +50,21 @@ type configOptions struct {
 	noConsistency       bool
 }
 
-func (o *configOptions) ToProject(services []string) (*types.Project, error) {
+func (o *configOptions) ToProject(ctx context.Context, services []string) (*types.Project, error) {
+	git, err := remote.NewGitRemoteLoader()
+	if err != nil {
+		return nil, err
+	}
+
 	return o.ProjectOptions.ToProject(services,
 		cli.WithInterpolation(!o.noInterpolate),
 		cli.WithResolvedPaths(!o.noResolvePath),
 		cli.WithNormalization(!o.noNormalize),
 		cli.WithConsistency(!o.noConsistency),
 		cli.WithProfiles(o.Profiles),
-		cli.WithDiscardEnvFile)
+		cli.WithDiscardEnvFile,
+		cli.WithContext(ctx),
+		cli.WithResourceLoader(git))
 }
 
 func configCommand(p *ProjectOptions, streams api.Streams, backend api.Service) *cobra.Command {
@@ -82,19 +90,19 @@ func configCommand(p *ProjectOptions, streams api.Streams, backend api.Service)
 		}),
 		RunE: Adapt(func(ctx context.Context, args []string) error {
 			if opts.services {
-				return runServices(streams, opts)
+				return runServices(ctx, streams, opts)
 			}
 			if opts.volumes {
-				return runVolumes(streams, opts)
+				return runVolumes(ctx, streams, opts)
 			}
 			if opts.hash != "" {
-				return runHash(streams, opts)
+				return runHash(ctx, streams, opts)
 			}
 			if opts.profiles {
-				return runProfiles(streams, opts, args)
+				return runProfiles(ctx, streams, opts, args)
 			}
 			if opts.images {
-				return runConfigImages(streams, opts, args)
+				return runConfigImages(ctx, streams, opts, args)
 			}
 
 			return runConfig(ctx, streams, backend, opts, args)
@@ -122,7 +130,7 @@ func configCommand(p *ProjectOptions, streams api.Streams, backend api.Service)
 
 func runConfig(ctx context.Context, streams api.Streams, backend api.Service, opts configOptions, services []string) error {
 	var content []byte
-	project, err := opts.ToProject(services)
+	project, err := opts.ToProject(ctx, services)
 	if err != nil {
 		return err
 	}
@@ -151,8 +159,8 @@ func runConfig(ctx context.Context, streams api.Streams, backend api.Service, op
 	return err
 }
 
-func runServices(streams api.Streams, opts configOptions) error {
+func runServices(ctx context.Context, streams api.Streams, opts configOptions) error {
-	project, err := opts.ToProject(nil)
+	project, err := opts.ToProject(ctx, nil)
 	if err != nil {
 		return err
 	}
@@ -162,8 +170,8 @@ func runServices(streams api.Streams, opts configOptions) error {
 	})
 }
 
-func runVolumes(streams api.Streams, opts configOptions) error {
+func runVolumes(ctx context.Context, streams api.Streams, opts configOptions) error {
-	project, err := opts.ToProject(nil)
+	project, err := opts.ToProject(ctx, nil)
 	if err != nil {
 		return err
 	}
@@ -173,12 +181,12 @@ func runVolumes(streams api.Streams, opts configOptions) error {
 	return nil
 }
 
-func runHash(streams api.Streams, opts configOptions) error {
+func runHash(ctx context.Context, streams api.Streams, opts configOptions) error {
 	var services []string
 	if opts.hash != "*" {
 		services = append(services, strings.Split(opts.hash, ",")...)
 	}
-	project, err := opts.ToProject(nil)
+	project, err := opts.ToProject(ctx, nil)
 	if err != nil {
 		return err
 	}
@@ -205,9 +213,9 @@ func runHash(streams api.Streams, opts configOptions) error {
 	return nil
 }
 
-func runProfiles(streams api.Streams, opts configOptions, services []string) error {
+func runProfiles(ctx context.Context, streams api.Streams, opts configOptions, services []string) error {
 	set := map[string]struct{}{}
-	project, err := opts.ToProject(services)
+	project, err := opts.ToProject(ctx, services)
 	if err != nil {
 		return err
 	}
@@ -227,8 +235,8 @@ func runProfiles(streams api.Streams, opts configOptions, services []string) err
 	return nil
 }
 
-func runConfigImages(streams api.Streams, opts configOptions, services []string) error {
+func runConfigImages(ctx context.Context, streams api.Streams, opts configOptions, services []string) error {
-	project, err := opts.ToProject(services)
+	project, err := opts.ToProject(ctx, services)
 	if err != nil {
 		return err
 	}

pkg/remote/git.go

@@ -0,0 +1,194 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package remote
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+
+	"github.com/compose-spec/compose-go/cli"
+	"github.com/compose-spec/compose-go/loader"
+	"github.com/compose-spec/compose-go/types"
+	"github.com/docker/compose/v2/pkg/api"
+	"github.com/moby/buildkit/util/gitutil"
+	"github.com/pkg/errors"
+)
+
+func GitRemoteLoaderEnabled() (bool, error) {
+	if v := os.Getenv("COMPOSE_EXPERIMENTAL_GIT_REMOTE"); v != "" {
+		enabled, err := strconv.ParseBool(v)
+		if err != nil {
+			return false, errors.Wrap(err, "COMPOSE_EXPERIMENTAL_GIT_REMOTE environment variable expects boolean value")
+		}
+		return enabled, err
+	}
+	return false, nil
+}
+
+func NewGitRemoteLoader() (loader.ResourceLoader, error) {
+	var base string
+	if cacheHome := os.Getenv("XDG_CACHE_HOME"); cacheHome != "" {
+		base = cacheHome
+	} else {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return nil, err
+		}
+		base = filepath.Join(home, ".cache")
+	}
+	cache := filepath.Join(base, "docker-compose")
+
+	err := os.MkdirAll(cache, 0o700)
+	return gitRemoteLoader{
+		cache: cache,
+	}, err
+}
+
+type gitRemoteLoader struct {
+	cache string
+}
+
+func (g gitRemoteLoader) Accept(path string) bool {
+	_, err := gitutil.ParseGitRef(path)
+	return err == nil
+}
+
+var commitSHA = regexp.MustCompile(`^[a-f0-9]{40}$`)
+
+func (g gitRemoteLoader) Load(ctx context.Context, path string) (string, error) {
+	ref, err := gitutil.ParseGitRef(path)
+	if err != nil {
+		return "", err
+	}
+
+	if ref.Commit == "" {
+		ref.Commit = "HEAD" // default branch
+	}
+
+	if !commitSHA.MatchString(ref.Commit) {
+		cmd := exec.CommandContext(ctx, "git", "ls-remote", "--exit-code", ref.Remote, ref.Commit)
+		cmd.Env = g.gitCommandEnv()
+		out, err := cmd.Output()
+		if err != nil {
+			if cmd.ProcessState.ExitCode() == 2 {
+				return "", errors.Wrapf(err, "repository does not contain ref %s, output: %q", path, string(out))
+			}
+			return "", err
+		}
+		if len(out) < 40 {
+			return "", fmt.Errorf("unexpected git command output: %q", string(out))
+		}
+		sha := string(out[:40])
+		if !commitSHA.MatchString(sha) {
+			return "", fmt.Errorf("invalid commit sha %q", sha)
+		}
+		ref.Commit = sha
+	}
+
+	local := filepath.Join(g.cache, ref.Commit)
+	if _, err := os.Stat(local); os.IsNotExist(err) {
+		err = g.checkout(ctx, local, ref)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if ref.SubDir != "" {
+		local = filepath.Join(local, ref.SubDir)
+	}
+	stat, err := os.Stat(local)
+	if err != nil {
+		return "", err
+	}
+	if stat.IsDir() {
+		local, err = findFile(cli.DefaultFileNames, local)
+	}
+	return local, err
+}
+
+func (g gitRemoteLoader) checkout(ctx context.Context, path string, ref *gitutil.GitRef) error {
+	err := os.MkdirAll(path, 0o700)
+	if err != nil {
+		return err
+	}
+	err = exec.CommandContext(ctx, "git", "init", path).Run()
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.CommandContext(ctx, "git", "remote", "add", "origin", ref.Remote)
+	cmd.Dir = path
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	cmd = exec.CommandContext(ctx, "git", "fetch", "--depth=1", "origin", ref.Commit)
+	cmd.Env = g.gitCommandEnv()
+	cmd.Dir = path
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	cmd = exec.CommandContext(ctx, "git", "checkout", ref.Commit)
+	cmd.Dir = path
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (g gitRemoteLoader) gitCommandEnv() []string {
+	env := types.NewMapping(os.Environ())
+	if env["GIT_TERMINAL_PROMPT"] == "" {
+		// Disable prompting for passwords by Git until user explicitly asks for it.
+		env["GIT_TERMINAL_PROMPT"] = "0"
+	}
+	if env["GIT_SSH"] == "" && env["GIT_SSH_COMMAND"] == "" {
+		// Disable any ssh connection pooling by Git and do not attempt to prompt the user.
+		env["GIT_SSH_COMMAND"] = "ssh -o ControlMaster=no -o BatchMode=yes"
+	}
+	v := values(env)
+	return v
+}
+
+func findFile(names []string, pwd string) (string, error) {
+	for _, n := range names {
+		f := filepath.Join(pwd, n)
+		if fi, err := os.Stat(f); err == nil && !fi.IsDir() {
+			return f, nil
+		}
+	}
+	return "", api.ErrNotFound
+}
+
+var _ loader.ResourceLoader = gitRemoteLoader{}
+
+func values(m types.Mapping) []string {
+	values := make([]string, 0, len(m))
+	for k, v := range m {
+		values = append(values, fmt.Sprintf("%s=%s", k, v))
+	}
+	return values
+}