Merge pull request #10831 from milas/instrument-up

trace: instrument `compose up` at a high-level

internal/tracing/attributes.go

@@ -0,0 +1,152 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package tracing
+
+import (
+	"strings"
+	"time"
+
+	"github.com/compose-spec/compose-go/types"
+	moby "github.com/docker/docker/api/types"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// SpanOptions is a small helper type to make it easy to share the options helpers between
+// downstream functions that accept slices of trace.SpanStartOption and trace.EventOption.
+type SpanOptions []trace.SpanStartEventOption
+
+func (s SpanOptions) SpanStartOptions() []trace.SpanStartOption {
+	out := make([]trace.SpanStartOption, len(s))
+	for i := range s {
+		out[i] = s[i]
+	}
+	return out
+}
+
+func (s SpanOptions) EventOptions() []trace.EventOption {
+	out := make([]trace.EventOption, len(s))
+	for i := range s {
+		out[i] = s[i]
+	}
+	return out
+}
+
+// ProjectOptions returns common attributes from a Compose project.
+//
+// For convenience, it's returned as a SpanOptions object to allow it to be
+// passed directly to the wrapping helper methods in this package such as
+// SpanWrapFunc.
+func ProjectOptions(proj *types.Project) SpanOptions {
+	if proj == nil {
+		return nil
+	}
+
+	disabledServiceNames := make([]string, len(proj.DisabledServices))
+	for i := range proj.DisabledServices {
+		disabledServiceNames[i] = proj.DisabledServices[i].Name
+	}
+
+	attrs := []attribute.KeyValue{
+		attribute.String("project.name", proj.Name),
+		attribute.String("project.dir", proj.WorkingDir),
+		attribute.StringSlice("project.compose_files", proj.ComposeFiles),
+		attribute.StringSlice("project.services.active", proj.ServiceNames()),
+		attribute.StringSlice("project.services.disabled", disabledServiceNames),
+		attribute.StringSlice("project.profiles", proj.Profiles),
+		attribute.StringSlice("project.volumes", proj.VolumeNames()),
+		attribute.StringSlice("project.networks", proj.NetworkNames()),
+		attribute.StringSlice("project.secrets", proj.SecretNames()),
+		attribute.StringSlice("project.configs", proj.ConfigNames()),
+		attribute.StringSlice("project.extensions", keys(proj.Extensions)),
+	}
+	return []trace.SpanStartEventOption{
+		trace.WithAttributes(attrs...),
+	}
+}
+
+// ServiceOptions returns common attributes from a Compose service.
+//
+// For convenience, it's returned as a SpanOptions object to allow it to be
+// passed directly to the wrapping helper methods in this package such as
+// SpanWrapFunc.
+func ServiceOptions(service types.ServiceConfig) SpanOptions {
+	attrs := []attribute.KeyValue{
+		attribute.String("service.name", service.Name),
+		attribute.String("service.image", service.Image),
+		attribute.StringSlice("service.networks", keys(service.Networks)),
+	}
+
+	configNames := make([]string, len(service.Configs))
+	for i := range service.Configs {
+		configNames[i] = service.Configs[i].Source
+	}
+	attrs = append(attrs, attribute.StringSlice("service.configs", configNames))
+
+	secretNames := make([]string, len(service.Secrets))
+	for i := range service.Secrets {
+		secretNames[i] = service.Secrets[i].Source
+	}
+	attrs = append(attrs, attribute.StringSlice("service.secrets", secretNames))
+
+	volNames := make([]string, len(service.Volumes))
+	for i := range service.Volumes {
+		volNames[i] = service.Volumes[i].Source
+	}
+	attrs = append(attrs, attribute.StringSlice("service.volumes", volNames))
+
+	return []trace.SpanStartEventOption{
+		trace.WithAttributes(attrs...),
+	}
+}
+
+// ContainerOptions returns common attributes from a Moby container.
+//
+// For convenience, it's returned as a SpanOptions object to allow it to be
+// passed directly to the wrapping helper methods in this package such as
+// SpanWrapFunc.
+func ContainerOptions(container moby.Container) SpanOptions {
+	attrs := []attribute.KeyValue{
+		attribute.String("container.id", container.ID),
+		attribute.String("container.image", container.Image),
+		unixTimeAttr("container.created_at", container.Created),
+	}
+
+	if len(container.Names) != 0 {
+		attrs = append(attrs, attribute.String("container.name", strings.TrimPrefix(container.Names[0], "/")))
+	}
+
+	return []trace.SpanStartEventOption{
+		trace.WithAttributes(attrs...),
+	}
+}
+
+func keys[T any](m map[string]T) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}
+
+func timeAttr(key string, value time.Time) attribute.KeyValue {
+	return attribute.String(key, value.Format(time.RFC3339))
+}
+
+func unixTimeAttr(key string, value int64) attribute.KeyValue {
+	return timeAttr(key, time.Unix(value, 0).UTC())
+}

internal/tracing/wrap.go

@@ -0,0 +1,91 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// SpanWrapFunc wraps a function that takes a context with a trace.Span, marking the status as codes.Error if the
+// wrapped function returns an error.
+//
+// The context passed to the function is created from the span to ensure correct propagation.
+//
+// NOTE: This function is nearly identical to SpanWrapFuncForErrGroup, except the latter is designed specially for
+// convenience with errgroup.Group due to its prevalence throughout the codebase. The code is duplicated to avoid
+// adding even more levels of function wrapping/indirection.
+func SpanWrapFunc(spanName string, opts SpanOptions, fn func(ctx context.Context) error) func(context.Context) error {
+	return func(ctx context.Context) error {
+		ctx, span := Tracer.Start(ctx, spanName, opts.SpanStartOptions()...)
+		defer span.End()
+
+		if err := fn(ctx); err != nil {
+			span.SetStatus(codes.Error, err.Error())
+			return err
+		}
+
+		span.SetStatus(codes.Ok, "")
+		return nil
+	}
+}
+
+// SpanWrapFuncForErrGroup wraps a function that takes a context with a trace.Span, marking the status as codes.Error
+// if the wrapped function returns an error.
+//
+// The context passed to the function is created from the span to ensure correct propagation.
+//
+// NOTE: This function is nearly identical to SpanWrapFunc, except this function is designed specially for
+// convenience with errgroup.Group due to its prevalence throughout the codebase. The code is duplicated to avoid
+// adding even more levels of function wrapping/indirection.
+func SpanWrapFuncForErrGroup(ctx context.Context, spanName string, opts SpanOptions, fn func(ctx context.Context) error) func() error {
+	return func() error {
+		ctx, span := Tracer.Start(ctx, spanName, opts.SpanStartOptions()...)
+		defer span.End()
+
+		if err := fn(ctx); err != nil {
+			span.SetStatus(codes.Error, err.Error())
+			return err
+		}
+
+		span.SetStatus(codes.Ok, "")
+		return nil
+	}
+}
+
+// EventWrapFuncForErrGroup invokes a function and records an event, optionally including the returned
+// error as the "exception message" on the event.
+//
+// This is intended for lightweight usage to wrap errgroup.Group calls where a full span is not desired.
+func EventWrapFuncForErrGroup(ctx context.Context, eventName string, opts SpanOptions, fn func(ctx context.Context) error) func() error {
+	return func() error {
+		span := trace.SpanFromContext(ctx)
+		eventOpts := opts.EventOptions()
+
+		err := fn(ctx)
+
+		if err != nil {
+			eventOpts = append(eventOpts, trace.WithAttributes(semconv.ExceptionMessage(err.Error())))
+		}
+		span.AddEvent(eventName, eventOpts...)
+
+		return err
+	}
+}

pkg/compose/build.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/docker/compose/v2/internal/tracing"
+
 	"github.com/docker/buildx/controller/pb"
 
 	"github.com/compose-spec/compose-go/types"
@@ -170,7 +172,11 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types.
 		return err
 	}
 
-	err = s.pullRequiredImages(ctx, project, images, quietPull)
+	err = tracing.SpanWrapFunc("project/pull", tracing.ProjectOptions(project),
+		func(ctx context.Context) error {
+			return s.pullRequiredImages(ctx, project, images, quietPull)
+		},
+	)(ctx)
 	if err != nil {
 		return err
 	}
@@ -186,16 +192,24 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types.
 	}
 
 	if buildRequired {
-		builtImages, err := s.build(ctx, project, api.BuildOptions{
-			Progress: mode,
-		})
+		err = tracing.SpanWrapFunc("project/build", tracing.ProjectOptions(project),
+			func(ctx context.Context) error {
+				builtImages, err := s.build(ctx, project, api.BuildOptions{
+					Progress: mode,
+				})
+				if err != nil {
+					return err
+				}
+
+				for name, digest := range builtImages {
+					images[name] = digest
+				}
+				return nil
+			},
+		)(ctx)
 		if err != nil {
 			return err
 		}
-
-		for name, digest := range builtImages {
-			images[name] = digest
-		}
 	}
 
 	// set digest as com.docker.compose.image label so we can detect outdated containers

pkg/compose/convergence.go

@@ -25,8 +25,12 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+
 	"github.com/compose-spec/compose-go/types"
 	"github.com/containerd/containerd/platforms"
+	"github.com/docker/compose/v2/internal/tracing"
 	moby "github.com/docker/docker/api/types"
 	containerType "github.com/docker/docker/api/types/container"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -93,17 +97,19 @@ func (c *convergence) apply(ctx context.Context, project *types.Project, options
 			return err
 		}
 
-		strategy := options.RecreateDependencies
-		if utils.StringContains(options.Services, name) {
-			strategy = options.Recreate
-		}
-		err = c.ensureService(ctx, project, service, strategy, options.Inherit, options.Timeout)
-		if err != nil {
-			return err
-		}
+		return tracing.SpanWrapFunc("service/apply", tracing.ServiceOptions(service), func(ctx context.Context) error {
+			strategy := options.RecreateDependencies
+			if utils.StringContains(options.Services, name) {
+				strategy = options.Recreate
+			}
+			err = c.ensureService(ctx, project, service, strategy, options.Inherit, options.Timeout)
+			if err != nil {
+				return err
+			}
 
-		c.updateProject(project, name)
-		return nil
+			c.updateProject(project, name)
+			return nil
+		})(ctx)
 	})
 }
 
@@ -179,7 +185,8 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project,
 		if i >= expected {
 			// Scale Down
 			container := container
-			eg.Go(func() error {
+			traceOpts := append(tracing.ServiceOptions(service), tracing.ContainerOptions(container)...)
+			eg.Go(tracing.SpanWrapFuncForErrGroup(ctx, "service/scale/down", traceOpts, func(ctx context.Context) error {
 				timeoutInSecond := utils.DurationSecondToInt(timeout)
 				err := c.service.apiClient().ContainerStop(ctx, container.ID, containerType.StopOptions{
 					Timeout: timeoutInSecond,
@@ -188,7 +195,7 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project,
 					return err
 				}
 				return c.service.apiClient().ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{})
-			})
+			}))
 			continue
 		}
 
@@ -198,11 +205,11 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project,
 		}
 		if mustRecreate {
 			i, container := i, container
-			eg.Go(func() error {
+			eg.Go(tracing.SpanWrapFuncForErrGroup(ctx, "container/recreate", tracing.ContainerOptions(container), func(ctx context.Context) error {
 				recreated, err := c.service.recreateContainer(ctx, project, service, container, inherit, timeout)
 				updated[i] = recreated
 				return err
-			})
+			}))
 			continue
 		}
 
@@ -218,9 +225,9 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project,
 			w.Event(progress.CreatedEvent(name))
 		default:
 			container := container
-			eg.Go(func() error {
+			eg.Go(tracing.EventWrapFuncForErrGroup(ctx, "service/start", tracing.ContainerOptions(container), func(ctx context.Context) error {
 				return c.service.startContainer(ctx, container)
-			})
+			}))
 		}
 		updated[i] = container
 	}
@@ -231,7 +238,8 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project,
 		number := next + i
 		name := getContainerName(project.Name, service, number)
 		i := i
-		eg.Go(func() error {
+		eventOpts := tracing.SpanOptions{trace.WithAttributes(attribute.String("container.name", name))}
+		eg.Go(tracing.EventWrapFuncForErrGroup(ctx, "service/scale/up", eventOpts, func(ctx context.Context) error {
 			opts := createOptions{
 				AutoRemove:        false,
 				AttachStdin:       false,
@@ -241,7 +249,7 @@ func (c *convergence) ensureService(ctx context.Context, project *types.Project,
 			container, err := c.service.createContainer(ctx, project, service, name, number, opts)
 			updated[actual+i] = container
 			return err
-		})
+		}))
 		continue
 	}
 

pkg/compose/up.go

@@ -23,6 +23,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/docker/compose/v2/internal/tracing"
+
 	"github.com/compose-spec/compose-go/types"
 	"github.com/docker/cli/cli"
 	"github.com/docker/compose/v2/pkg/api"
@@ -31,7 +33,7 @@ import (
 )
 
 func (s *composeService) Up(ctx context.Context, project *types.Project, options api.UpOptions) error {
-	err := progress.Run(ctx, func(ctx context.Context) error {
+	err := progress.Run(ctx, tracing.SpanWrapFunc("project/up", tracing.ProjectOptions(project), func(ctx context.Context) error {
 		err := s.create(ctx, project, options.Create)
 		if err != nil {
 			return err
@@ -40,7 +42,7 @@ func (s *composeService) Up(ctx context.Context, project *types.Project, options
 			return s.start(ctx, project.Name, options.Start, nil)
 		}
 		return nil
-	}, s.stdinfo())
+	}), s.stdinfo())
 	if err != nil {
 		return err
 	}