From 16c8099cf8bc8cd51d03c5d72fa606d7c5b56bf6 Mon Sep 17 00:00:00 2001 From: jhrotko Date: Mon, 12 Feb 2024 14:52:58 +0000 Subject: [PATCH] Use listener for file metadata Signed-off-by: jhrotko --- cmd/compose/build.go | 2 +- cmd/compose/completion.go | 4 ++-- cmd/compose/compose.go | 27 ++++++++++++++++++--------- cmd/compose/config.go | 15 ++++++++------- cmd/compose/publish.go | 2 +- cmd/compose/pull.go | 2 +- cmd/compose/push.go | 2 +- cmd/compose/run.go | 2 +- cmd/compose/scale.go | 2 +- cmd/compose/viz.go | 2 +- cmd/compose/watch.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/tracing/attributes.go | 15 +++++++++++---- pkg/compose/build.go | 4 ++-- pkg/compose/scale.go | 2 +- pkg/compose/up.go | 2 +- 17 files changed, 54 insertions(+), 37 deletions(-) diff --git a/cmd/compose/build.go b/cmd/compose/build.go index e96fbfb1c..bb1db159a 100644 --- a/cmd/compose/build.go +++ b/cmd/compose/build.go @@ -136,7 +136,7 @@ func buildCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) } func runBuild(ctx context.Context, dockerCli command.Cli, backend api.Service, opts buildOptions, services []string) error { - project, err := opts.ToProject(ctx, dockerCli, services, cli.WithResolvedPaths(true), cli.WithoutEnvironmentResolution) + project, _, err := opts.ToProject(ctx, dockerCli, services, cli.WithResolvedPaths(true), cli.WithoutEnvironmentResolution) if err != nil { return err } diff --git a/cmd/compose/completion.go b/cmd/compose/completion.go index 21351d254..71da360c1 100644 --- a/cmd/compose/completion.go +++ b/cmd/compose/completion.go @@ -37,7 +37,7 @@ func noCompletion() validArgsFn { func completeServiceNames(dockerCli command.Cli, p *ProjectOptions) validArgsFn { return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { p.Offline = true - project, err := p.ToProject(cmd.Context(), dockerCli, nil) + project, _, err := p.ToProject(cmd.Context(), dockerCli, nil) if err != nil { return nil, cobra.ShellCompDirectiveNoFileComp } @@ -72,7 +72,7 @@ func completeProjectNames(backend api.Service) func(cmd *cobra.Command, args []s func completeProfileNames(dockerCli command.Cli, p *ProjectOptions) validArgsFn { return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { p.Offline = true - project, err := p.ToProject(cmd.Context(), dockerCli, nil) + project, _, err := p.ToProject(cmd.Context(), dockerCli, nil) if err != nil { return nil, cobra.ShellCompDirectiveNoFileComp } diff --git a/cmd/compose/compose.go b/cmd/compose/compose.go index 5d3d62916..6d923076b 100644 --- a/cmd/compose/compose.go +++ b/cmd/compose/compose.go @@ -36,6 +36,7 @@ import ( "github.com/docker/cli/cli-plugins/manager" "github.com/docker/cli/cli/command" "github.com/docker/compose/v2/cmd/formatter" + "github.com/docker/compose/v2/internal/tracing" "github.com/docker/compose/v2/pkg/api" "github.com/docker/compose/v2/pkg/compose" ui "github.com/docker/compose/v2/pkg/progress" @@ -141,11 +142,13 @@ func (o *ProjectOptions) WithServices(dockerCli command.Cli, fn ProjectServicesF cli.WithDiscardEnvFile, } - project, err := o.ToProject(ctx, dockerCli, args, options...) + project, metrics, err := o.ToProject(ctx, dockerCli, args, options...) if err != nil { return err } + ctx = context.WithValue(ctx, tracing.Metrics{}, metrics) + return fn(ctx, project, args) }) } @@ -166,7 +169,7 @@ func (o *ProjectOptions) projectOrName(ctx context.Context, dockerCli command.Cl name := o.ProjectName var project *types.Project if len(o.ConfigPaths) > 0 || o.ProjectName == "" { - p, err := o.ToProject(ctx, dockerCli, services, cli.WithDiscardEnvFile) + p, _, err := o.ToProject(ctx, dockerCli, services, cli.WithDiscardEnvFile) if err != nil { envProjectName := os.Getenv(ComposeProjectName) if envProjectName != "" { @@ -190,14 +193,15 @@ func (o *ProjectOptions) toProjectName(ctx context.Context, dockerCli command.Cl return envProjectName, nil } - project, err := o.ToProject(ctx, dockerCli, nil) + project, _, err := o.ToProject(ctx, dockerCli, nil) if err != nil { return "", err } return project.Name, nil } -func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, services []string, po ...cli.ProjectOptionsFn) (*types.Project, error) { +func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, services []string, po ...cli.ProjectOptionsFn) (*types.Project, tracing.Metrics, error) { + var metrics tracing.Metrics if !o.Offline { po = append(po, o.remoteLoaders(dockerCli)...) } @@ -206,25 +210,30 @@ func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, s options, err := o.toProjectOptions(po...) if err != nil { - return nil, compose.WrapComposeError(err) + return nil, metrics, compose.WrapComposeError(err) } + options.WithListeners(func(event string, metadata map[string]any) { + if event == "extends" { + metrics.CountExtends++ + } + }) if o.Compatibility || utils.StringToBool(options.Environment[ComposeCompatibility]) { api.Separator = "_" } project, err := cli.ProjectFromOptions(options) if err != nil { - return nil, compose.WrapComposeError(err) + return nil, metrics, compose.WrapComposeError(err) } if project.Name == "" { - return nil, errors.New("project name can't be empty. Use `--project-name` to set a valid name") + return nil, metrics, errors.New("project name can't be empty. Use `--project-name` to set a valid name") } project, err = project.WithServicesEnabled(services...) if err != nil { - return nil, err + return nil, metrics, err } for name, s := range project.Services { @@ -245,7 +254,7 @@ func (o *ProjectOptions) ToProject(ctx context.Context, dockerCli command.Cli, s project = project.WithoutUnnecessaryResources() project, err = project.WithSelectedServices(services) - return project, err + return project, metrics, err } func (o *ProjectOptions) remoteLoaders(dockerCli command.Cli) []cli.ProjectOptionsFn { diff --git a/cmd/compose/config.go b/cmd/compose/config.go index b0386be72..c76303753 100644 --- a/cmd/compose/config.go +++ b/cmd/compose/config.go @@ -29,6 +29,7 @@ import ( "github.com/docker/cli/cli/command" "github.com/spf13/cobra" + "github.com/docker/compose/v2/internal/tracing" "github.com/docker/compose/v2/pkg/api" "github.com/docker/compose/v2/pkg/compose" ) @@ -50,7 +51,7 @@ type configOptions struct { noConsistency bool } -func (o *configOptions) ToProject(ctx context.Context, dockerCli command.Cli, services []string, po ...cli.ProjectOptionsFn) (*types.Project, error) { +func (o *configOptions) ToProject(ctx context.Context, dockerCli command.Cli, services []string, po ...cli.ProjectOptionsFn) (*types.Project, tracing.Metrics, error) { po = append(po, cli.WithInterpolation(!o.noInterpolate), cli.WithResolvedPaths(!o.noResolvePath), @@ -124,7 +125,7 @@ func configCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service func runConfig(ctx context.Context, dockerCli command.Cli, backend api.Service, opts configOptions, services []string) error { var content []byte - project, err := opts.ToProject(ctx, dockerCli, services) + project, _, err := opts.ToProject(ctx, dockerCli, services) if err != nil { return err } @@ -154,7 +155,7 @@ func runConfig(ctx context.Context, dockerCli command.Cli, backend api.Service, } func runServices(ctx context.Context, dockerCli command.Cli, opts configOptions) error { - project, err := opts.ToProject(ctx, dockerCli, nil, cli.WithoutEnvironmentResolution) + project, _, err := opts.ToProject(ctx, dockerCli, nil, cli.WithoutEnvironmentResolution) if err != nil { return err } @@ -166,7 +167,7 @@ func runServices(ctx context.Context, dockerCli command.Cli, opts configOptions) } func runVolumes(ctx context.Context, dockerCli command.Cli, opts configOptions) error { - project, err := opts.ToProject(ctx, dockerCli, nil, cli.WithoutEnvironmentResolution) + project, _, err := opts.ToProject(ctx, dockerCli, nil, cli.WithoutEnvironmentResolution) if err != nil { return err } @@ -181,7 +182,7 @@ func runHash(ctx context.Context, dockerCli command.Cli, opts configOptions) err if opts.hash != "*" { services = append(services, strings.Split(opts.hash, ",")...) } - project, err := opts.ToProject(ctx, dockerCli, nil, cli.WithoutEnvironmentResolution) + project, _, err := opts.ToProject(ctx, dockerCli, nil, cli.WithoutEnvironmentResolution) if err != nil { return err } @@ -217,7 +218,7 @@ func runHash(ctx context.Context, dockerCli command.Cli, opts configOptions) err func runProfiles(ctx context.Context, dockerCli command.Cli, opts configOptions, services []string) error { set := map[string]struct{}{} - project, err := opts.ToProject(ctx, dockerCli, services, cli.WithoutEnvironmentResolution) + project, _, err := opts.ToProject(ctx, dockerCli, services, cli.WithoutEnvironmentResolution) if err != nil { return err } @@ -238,7 +239,7 @@ func runProfiles(ctx context.Context, dockerCli command.Cli, opts configOptions, } func runConfigImages(ctx context.Context, dockerCli command.Cli, opts configOptions, services []string) error { - project, err := opts.ToProject(ctx, dockerCli, services, cli.WithoutEnvironmentResolution) + project, _, err := opts.ToProject(ctx, dockerCli, services, cli.WithoutEnvironmentResolution) if err != nil { return err } diff --git a/cmd/compose/publish.go b/cmd/compose/publish.go index 32a202085..48b0e2d58 100644 --- a/cmd/compose/publish.go +++ b/cmd/compose/publish.go @@ -50,7 +50,7 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic } func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service, opts publishOptions, repository string) error { - project, err := opts.ToProject(ctx, dockerCli, nil) + project, _, err := opts.ToProject(ctx, dockerCli, nil) if err != nil { return err } diff --git a/cmd/compose/pull.go b/cmd/compose/pull.go index 3f350ee86..3d3ef5442 100644 --- a/cmd/compose/pull.go +++ b/cmd/compose/pull.go @@ -94,7 +94,7 @@ func (opts pullOptions) apply(project *types.Project, services []string) (*types } func runPull(ctx context.Context, dockerCli command.Cli, backend api.Service, opts pullOptions, services []string) error { - project, err := opts.ToProject(ctx, dockerCli, services) + project, _, err := opts.ToProject(ctx, dockerCli, services) if err != nil { return err } diff --git a/cmd/compose/push.go b/cmd/compose/push.go index 603c2092f..177f9f2ec 100644 --- a/cmd/compose/push.go +++ b/cmd/compose/push.go @@ -54,7 +54,7 @@ func pushCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) } func runPush(ctx context.Context, dockerCli command.Cli, backend api.Service, opts pushOptions, services []string) error { - project, err := opts.ToProject(ctx, dockerCli, services) + project, _, err := opts.ToProject(ctx, dockerCli, services) if err != nil { return err } diff --git a/cmd/compose/run.go b/cmd/compose/run.go index 19db6e32e..29fcea254 100644 --- a/cmd/compose/run.go +++ b/cmd/compose/run.go @@ -156,7 +156,7 @@ func runCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) * return nil }), RunE: Adapt(func(ctx context.Context, args []string) error { - project, err := p.ToProject(ctx, dockerCli, []string{options.Service}, cgo.WithResolvedPaths(true), cgo.WithDiscardEnvFile) + project, _, err := p.ToProject(ctx, dockerCli, []string{options.Service}, cgo.WithResolvedPaths(true), cgo.WithDiscardEnvFile) if err != nil { return err } diff --git a/cmd/compose/scale.go b/cmd/compose/scale.go index ea852dabb..0ec4799a9 100644 --- a/cmd/compose/scale.go +++ b/cmd/compose/scale.go @@ -61,7 +61,7 @@ func scaleCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) func runScale(ctx context.Context, dockerCli command.Cli, backend api.Service, opts scaleOptions, serviceReplicaTuples map[string]int) error { services := maps.Keys(serviceReplicaTuples) - project, err := opts.ToProject(ctx, dockerCli, services) + project, _, err := opts.ToProject(ctx, dockerCli, services) if err != nil { return err } diff --git a/cmd/compose/viz.go b/cmd/compose/viz.go index 4410136f5..d97504e38 100644 --- a/cmd/compose/viz.go +++ b/cmd/compose/viz.go @@ -65,7 +65,7 @@ func vizCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) * func runViz(ctx context.Context, dockerCli command.Cli, backend api.Service, opts *vizOptions) error { _, _ = fmt.Fprintln(os.Stderr, "viz command is EXPERIMENTAL") - project, err := opts.ToProject(ctx, dockerCli, nil) + project, _, err := opts.ToProject(ctx, dockerCli, nil) if err != nil { return err } diff --git a/cmd/compose/watch.go b/cmd/compose/watch.go index cf39a2075..e4f942726 100644 --- a/cmd/compose/watch.go +++ b/cmd/compose/watch.go @@ -63,7 +63,7 @@ func watchCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) } func runWatch(ctx context.Context, dockerCli command.Cli, backend api.Service, watchOpts watchOptions, buildOpts buildOptions, services []string) error { - project, err := watchOpts.ToProject(ctx, dockerCli, nil) + project, _, err := watchOpts.ToProject(ctx, dockerCli, nil) if err != nil { return err } diff --git a/go.mod b/go.mod index 24de12787..c946369eb 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Microsoft/go-winio v0.6.1 github.com/buger/goterm v1.0.4 - github.com/compose-spec/compose-go/v2 v2.0.0-rc.5 + github.com/compose-spec/compose-go/v2 v2.0.0-rc.6 github.com/containerd/console v1.0.3 github.com/containerd/containerd v1.7.12 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 0cb6c7f79..9ed3efa6f 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= -github.com/compose-spec/compose-go/v2 v2.0.0-rc.5 h1:YoGsuVzxve1m5SdCfZqI8wJoMVZWu7SelHoqiCqb+iQ= -github.com/compose-spec/compose-go/v2 v2.0.0-rc.5/go.mod h1:bEPizBkIojlQ20pi2vNluBa58tevvj0Y18oUSHPyfdc= +github.com/compose-spec/compose-go/v2 v2.0.0-rc.6 h1:sv9W3U0IEYqgGqTbSDpU2c8cttWQmlbJ0D6jdt//Dv8= +github.com/compose-spec/compose-go/v2 v2.0.0-rc.6/go.mod h1:bEPizBkIojlQ20pi2vNluBa58tevvj0Y18oUSHPyfdc= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= diff --git a/internal/tracing/attributes.go b/internal/tracing/attributes.go index 5eb470c6c..23a9b1349 100644 --- a/internal/tracing/attributes.go +++ b/internal/tracing/attributes.go @@ -17,6 +17,7 @@ package tracing import ( + "context" "crypto/sha256" "encoding/json" "fmt" @@ -34,6 +35,9 @@ import ( // SpanOptions is a small helper type to make it easy to share the options helpers between // downstream functions that accept slices of trace.SpanStartOption and trace.EventOption. type SpanOptions []trace.SpanStartEventOption +type Metrics struct { + CountExtends int +} func (s SpanOptions) SpanStartOptions() []trace.SpanStartOption { out := make([]trace.SpanStartOption, len(s)) @@ -56,18 +60,15 @@ func (s SpanOptions) EventOptions() []trace.EventOption { // For convenience, it's returned as a SpanOptions object to allow it to be // passed directly to the wrapping helper methods in this package such as // SpanWrapFunc. -func ProjectOptions(proj *types.Project) SpanOptions { +func ProjectOptions(ctx context.Context, proj *types.Project) SpanOptions { if proj == nil { return nil } - capabilities, gpu, tpu := proj.ServicesWithCapabilities() attrs := []attribute.KeyValue{ attribute.String("project.name", proj.Name), attribute.String("project.dir", proj.WorkingDir), attribute.StringSlice("project.compose_files", proj.ComposeFiles), - attribute.StringSlice("project.services.active", proj.ServiceNames()), - attribute.StringSlice("project.services.disabled", proj.DisabledServiceNames()), attribute.StringSlice("project.profiles", proj.Profiles), attribute.StringSlice("project.volumes", proj.VolumeNames()), attribute.StringSlice("project.networks", proj.NetworkNames()), @@ -75,12 +76,18 @@ func ProjectOptions(proj *types.Project) SpanOptions { attribute.StringSlice("project.configs", proj.ConfigNames()), attribute.StringSlice("project.extensions", keys(proj.Extensions)), attribute.StringSlice("project.includes", flattenIncludeReferences(proj.IncludeReferences)), + attribute.StringSlice("project.services.active", proj.ServiceNames()), + attribute.StringSlice("project.services.disabled", proj.DisabledServiceNames()), attribute.StringSlice("project.services.build", proj.ServicesWithBuild()), attribute.StringSlice("project.services.depends_on", proj.ServicesWithDependsOn()), attribute.StringSlice("project.services.capabilities", capabilities), attribute.StringSlice("project.services.capabilities.gpu", gpu), attribute.StringSlice("project.services.capabilities.tpu", tpu), } + if metrics, ok := ctx.Value(Metrics{}).(Metrics); ok { + attrs = append(attrs, attribute.Int("project.services.extends", metrics.CountExtends)) + } + if projHash, ok := projectHash(proj); ok { attrs = append(attrs, attribute.String("project.hash", projHash)) } diff --git a/pkg/compose/build.go b/pkg/compose/build.go index b4ce4c80a..fcac6495b 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -221,7 +221,7 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. return err } - err = tracing.SpanWrapFunc("project/pull", tracing.ProjectOptions(project), + err = tracing.SpanWrapFunc("project/pull", tracing.ProjectOptions(ctx, project), func(ctx context.Context) error { return s.pullRequiredImages(ctx, project, images, quietPull) }, @@ -231,7 +231,7 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. } if buildOpts != nil { - err = tracing.SpanWrapFunc("project/build", tracing.ProjectOptions(project), + err = tracing.SpanWrapFunc("project/build", tracing.ProjectOptions(ctx, project), func(ctx context.Context) error { builtImages, err := s.build(ctx, project, *buildOpts, images) if err != nil { diff --git a/pkg/compose/scale.go b/pkg/compose/scale.go index 1e869ea72..985cc129d 100644 --- a/pkg/compose/scale.go +++ b/pkg/compose/scale.go @@ -25,7 +25,7 @@ import ( ) func (s *composeService) Scale(ctx context.Context, project *types.Project, options api.ScaleOptions) error { - return progress.Run(ctx, tracing.SpanWrapFunc("project/scale", tracing.ProjectOptions(project), func(ctx context.Context) error { + return progress.Run(ctx, tracing.SpanWrapFunc("project/scale", tracing.ProjectOptions(ctx, project), func(ctx context.Context) error { err := s.create(ctx, project, api.CreateOptions{Services: options.Services}) if err != nil { return err diff --git a/pkg/compose/up.go b/pkg/compose/up.go index be6bbdb14..c1dafb594 100644 --- a/pkg/compose/up.go +++ b/pkg/compose/up.go @@ -32,7 +32,7 @@ import ( ) func (s *composeService) Up(ctx context.Context, project *types.Project, options api.UpOptions) error { //nolint:gocyclo - err := progress.Run(ctx, tracing.SpanWrapFunc("project/up", tracing.ProjectOptions(project), func(ctx context.Context) error { + err := progress.Run(ctx, tracing.SpanWrapFunc("project/up", tracing.ProjectOptions(ctx, project), func(ctx context.Context) error { w := progress.ContextWriter(ctx) w.HasMore(options.Start.Attach == nil) err := s.create(ctx, project, options.Create)