From dd34f7a22b7d4e8156b48559cf14039445614418 Mon Sep 17 00:00:00 2001 From: Nicolas De loof Date: Fri, 18 Aug 2023 15:16:45 +0200 Subject: [PATCH] include: add experimental support for Git resources (#10811) Requires setting `COMPOSE_EXPERIMENTAL_GIT_REMOTE=1`. Signed-off-by: Nicolas De Loof --- cmd/compose/compose.go | 21 ++++- cmd/compose/config.go | 44 ++++++---- pkg/remote/git.go | 194 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 19 deletions(-) create mode 100644 pkg/remote/git.go diff --git a/cmd/compose/compose.go b/cmd/compose/compose.go index 0f4016f30..256b30d5a 100644 --- a/cmd/compose/compose.go +++ b/cmd/compose/compose.go @@ -29,6 +29,7 @@ import ( "github.com/compose-spec/compose-go/dotenv" buildx "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" + "github.com/docker/compose/v2/pkg/remote" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" @@ -134,7 +135,25 @@ func (o *ProjectOptions) WithProject(fn ProjectFunc) func(cmd *cobra.Command, ar // WithServices creates a cobra run command from a ProjectFunc based on configured project options and selected services func (o *ProjectOptions) WithServices(fn ProjectServicesFunc) func(cmd *cobra.Command, args []string) error { return Adapt(func(ctx context.Context, args []string) error { - project, err := o.ToProject(args, cli.WithResolvedPaths(true), cli.WithDiscardEnvFile) + options := []cli.ProjectOptionsFn{ + cli.WithResolvedPaths(true), + cli.WithDiscardEnvFile, + cli.WithContext(ctx), + } + + enabled, err := remote.GitRemoteLoaderEnabled() + if err != nil { + return err + } + if enabled { + git, err := remote.NewGitRemoteLoader() + if err != nil { + return err + } + options = append(options, cli.WithResourceLoader(git)) + } + + project, err := o.ToProject(args, options...) if err != nil { return err } diff --git a/cmd/compose/config.go b/cmd/compose/config.go index 16031ffeb..1383558c3 100644 --- a/cmd/compose/config.go +++ b/cmd/compose/config.go @@ -26,6 +26,7 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" + "github.com/docker/compose/v2/pkg/remote" "github.com/spf13/cobra" "github.com/docker/compose/v2/pkg/api" @@ -49,14 +50,21 @@ type configOptions struct { noConsistency bool } -func (o *configOptions) ToProject(services []string) (*types.Project, error) { +func (o *configOptions) ToProject(ctx context.Context, services []string) (*types.Project, error) { + git, err := remote.NewGitRemoteLoader() + if err != nil { + return nil, err + } + return o.ProjectOptions.ToProject(services, cli.WithInterpolation(!o.noInterpolate), cli.WithResolvedPaths(!o.noResolvePath), cli.WithNormalization(!o.noNormalize), cli.WithConsistency(!o.noConsistency), cli.WithProfiles(o.Profiles), - cli.WithDiscardEnvFile) + cli.WithDiscardEnvFile, + cli.WithContext(ctx), + cli.WithResourceLoader(git)) } func configCommand(p *ProjectOptions, streams api.Streams, backend api.Service) *cobra.Command { @@ -82,19 +90,19 @@ func configCommand(p *ProjectOptions, streams api.Streams, backend api.Service) }), RunE: Adapt(func(ctx context.Context, args []string) error { if opts.services { - return runServices(streams, opts) + return runServices(ctx, streams, opts) } if opts.volumes { - return runVolumes(streams, opts) + return runVolumes(ctx, streams, opts) } if opts.hash != "" { - return runHash(streams, opts) + return runHash(ctx, streams, opts) } if opts.profiles { - return runProfiles(streams, opts, args) + return runProfiles(ctx, streams, opts, args) } if opts.images { - return runConfigImages(streams, opts, args) + return runConfigImages(ctx, streams, opts, args) } return runConfig(ctx, streams, backend, opts, args) @@ -122,7 +130,7 @@ func configCommand(p *ProjectOptions, streams api.Streams, backend api.Service) func runConfig(ctx context.Context, streams api.Streams, backend api.Service, opts configOptions, services []string) error { var content []byte - project, err := opts.ToProject(services) + project, err := opts.ToProject(ctx, services) if err != nil { return err } @@ -151,8 +159,8 @@ func runConfig(ctx context.Context, streams api.Streams, backend api.Service, op return err } -func runServices(streams api.Streams, opts configOptions) error { - project, err := opts.ToProject(nil) +func runServices(ctx context.Context, streams api.Streams, opts configOptions) error { + project, err := opts.ToProject(ctx, nil) if err != nil { return err } @@ -162,8 +170,8 @@ func runServices(streams api.Streams, opts configOptions) error { }) } -func runVolumes(streams api.Streams, opts configOptions) error { - project, err := opts.ToProject(nil) +func runVolumes(ctx context.Context, streams api.Streams, opts configOptions) error { + project, err := opts.ToProject(ctx, nil) if err != nil { return err } @@ -173,12 +181,12 @@ func runVolumes(streams api.Streams, opts configOptions) error { return nil } -func runHash(streams api.Streams, opts configOptions) error { +func runHash(ctx context.Context, streams api.Streams, opts configOptions) error { var services []string if opts.hash != "*" { services = append(services, strings.Split(opts.hash, ",")...) } - project, err := opts.ToProject(nil) + project, err := opts.ToProject(ctx, nil) if err != nil { return err } @@ -205,9 +213,9 @@ func runHash(streams api.Streams, opts configOptions) error { return nil } -func runProfiles(streams api.Streams, opts configOptions, services []string) error { +func runProfiles(ctx context.Context, streams api.Streams, opts configOptions, services []string) error { set := map[string]struct{}{} - project, err := opts.ToProject(services) + project, err := opts.ToProject(ctx, services) if err != nil { return err } @@ -227,8 +235,8 @@ func runProfiles(streams api.Streams, opts configOptions, services []string) err return nil } -func runConfigImages(streams api.Streams, opts configOptions, services []string) error { - project, err := opts.ToProject(services) +func runConfigImages(ctx context.Context, streams api.Streams, opts configOptions, services []string) error { + project, err := opts.ToProject(ctx, services) if err != nil { return err } diff --git a/pkg/remote/git.go b/pkg/remote/git.go new file mode 100644 index 000000000..a32e230b7 --- /dev/null +++ b/pkg/remote/git.go @@ -0,0 +1,194 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package remote + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + + "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/loader" + "github.com/compose-spec/compose-go/types" + "github.com/docker/compose/v2/pkg/api" + "github.com/moby/buildkit/util/gitutil" + "github.com/pkg/errors" +) + +func GitRemoteLoaderEnabled() (bool, error) { + if v := os.Getenv("COMPOSE_EXPERIMENTAL_GIT_REMOTE"); v != "" { + enabled, err := strconv.ParseBool(v) + if err != nil { + return false, errors.Wrap(err, "COMPOSE_EXPERIMENTAL_GIT_REMOTE environment variable expects boolean value") + } + return enabled, err + } + return false, nil +} + +func NewGitRemoteLoader() (loader.ResourceLoader, error) { + var base string + if cacheHome := os.Getenv("XDG_CACHE_HOME"); cacheHome != "" { + base = cacheHome + } else { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + base = filepath.Join(home, ".cache") + } + cache := filepath.Join(base, "docker-compose") + + err := os.MkdirAll(cache, 0o700) + return gitRemoteLoader{ + cache: cache, + }, err +} + +type gitRemoteLoader struct { + cache string +} + +func (g gitRemoteLoader) Accept(path string) bool { + _, err := gitutil.ParseGitRef(path) + return err == nil +} + +var commitSHA = regexp.MustCompile(`^[a-f0-9]{40}$`) + +func (g gitRemoteLoader) Load(ctx context.Context, path string) (string, error) { + ref, err := gitutil.ParseGitRef(path) + if err != nil { + return "", err + } + + if ref.Commit == "" { + ref.Commit = "HEAD" // default branch + } + + if !commitSHA.MatchString(ref.Commit) { + cmd := exec.CommandContext(ctx, "git", "ls-remote", "--exit-code", ref.Remote, ref.Commit) + cmd.Env = g.gitCommandEnv() + out, err := cmd.Output() + if err != nil { + if cmd.ProcessState.ExitCode() == 2 { + return "", errors.Wrapf(err, "repository does not contain ref %s, output: %q", path, string(out)) + } + return "", err + } + if len(out) < 40 { + return "", fmt.Errorf("unexpected git command output: %q", string(out)) + } + sha := string(out[:40]) + if !commitSHA.MatchString(sha) { + return "", fmt.Errorf("invalid commit sha %q", sha) + } + ref.Commit = sha + } + + local := filepath.Join(g.cache, ref.Commit) + if _, err := os.Stat(local); os.IsNotExist(err) { + err = g.checkout(ctx, local, ref) + if err != nil { + return "", err + } + } + + if ref.SubDir != "" { + local = filepath.Join(local, ref.SubDir) + } + stat, err := os.Stat(local) + if err != nil { + return "", err + } + if stat.IsDir() { + local, err = findFile(cli.DefaultFileNames, local) + } + return local, err +} + +func (g gitRemoteLoader) checkout(ctx context.Context, path string, ref *gitutil.GitRef) error { + err := os.MkdirAll(path, 0o700) + if err != nil { + return err + } + err = exec.CommandContext(ctx, "git", "init", path).Run() + if err != nil { + return err + } + + cmd := exec.CommandContext(ctx, "git", "remote", "add", "origin", ref.Remote) + cmd.Dir = path + err = cmd.Run() + if err != nil { + return err + } + + cmd = exec.CommandContext(ctx, "git", "fetch", "--depth=1", "origin", ref.Commit) + cmd.Env = g.gitCommandEnv() + cmd.Dir = path + err = cmd.Run() + if err != nil { + return err + } + + cmd = exec.CommandContext(ctx, "git", "checkout", ref.Commit) + cmd.Dir = path + err = cmd.Run() + if err != nil { + return err + } + return nil +} + +func (g gitRemoteLoader) gitCommandEnv() []string { + env := types.NewMapping(os.Environ()) + if env["GIT_TERMINAL_PROMPT"] == "" { + // Disable prompting for passwords by Git until user explicitly asks for it. + env["GIT_TERMINAL_PROMPT"] = "0" + } + if env["GIT_SSH"] == "" && env["GIT_SSH_COMMAND"] == "" { + // Disable any ssh connection pooling by Git and do not attempt to prompt the user. + env["GIT_SSH_COMMAND"] = "ssh -o ControlMaster=no -o BatchMode=yes" + } + v := values(env) + return v +} + +func findFile(names []string, pwd string) (string, error) { + for _, n := range names { + f := filepath.Join(pwd, n) + if fi, err := os.Stat(f); err == nil && !fi.IsDir() { + return f, nil + } + } + return "", api.ErrNotFound +} + +var _ loader.ResourceLoader = gitRemoteLoader{} + +func values(m types.Mapping) []string { + values := make([]string, 0, len(m)) + for k, v := range m { + values = append(values, fmt.Sprintf("%s=%s", k, v)) + } + return values +}