trace: add OTEL initialization (#10526)

This is a bunch of OTEL initialization code. It's all in
`internal/` because there are re-usable parts here, but Compose
isn't the right spot. Once we've stabilized the interfaces a bit
and the need arises, we can move it to a separate module.

Currently, a single span is produced to wrap the root Compose
command.

Compose will respect the standard OTEL environment variables
as well as OTEL metadata from the Docker context. Both can be
used simultaneously. The latter is intended for local system
integration and is restricted to Unix sockets / named pipes.

None of this is enabled by default. It requires setting the
`COMPOSE_EXPERIMENTAL_OTEL=1` environment variable to
gate it during development.

Signed-off-by: Milas Bowman <milas.bowman@docker.com>
This commit is contained in:
Milas Bowman 2023-06-08 16:46:07 -04:00 committed by GitHub
parent e63ab14b1e
commit 3c8a56dbf3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 534 additions and 23 deletions

View File

@ -17,23 +17,33 @@
package main
import (
"context"
"os"
"time"
dockercli "github.com/docker/cli/cli"
"github.com/docker/cli/cli-plugins/manager"
"github.com/docker/cli/cli-plugins/plugin"
"github.com/docker/cli/cli/command"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/docker/compose/v2/cmd/compatibility"
commands "github.com/docker/compose/v2/cmd/compose"
"github.com/docker/compose/v2/internal"
"github.com/docker/compose/v2/internal/tracing"
"github.com/docker/compose/v2/pkg/api"
"github.com/docker/compose/v2/pkg/compose"
)
func pluginMain() {
plugin.Run(func(dockerCli command.Cli) *cobra.Command {
var tracingShutdown tracing.ShutdownFunc
var cmdSpan trace.Span
serviceProxy := api.NewServiceProxy().WithService(compose.NewComposeService(dockerCli))
cmd := commands.RootCommand(dockerCli, serviceProxy)
originalPreRun := cmd.PersistentPreRunE
@ -41,11 +51,52 @@ func pluginMain() {
if err := plugin.PersistentPreRunE(cmd, args); err != nil {
return err
}
// the call to plugin.PersistentPreRunE is what actually
// initializes the command.Cli instance, so this is the earliest
// that tracing can be practically initialized (in the future,
// this could ideally happen in coordination with docker/cli)
tracingShutdown, _ = tracing.InitTracing(dockerCli)
ctx := cmd.Context()
ctx, cmdSpan = tracing.Tracer.Start(
ctx, "cli/"+cmd.Name(),
trace.WithAttributes(
attribute.String("compose.version", internal.Version),
attribute.String("docker.context", dockerCli.CurrentContext()),
),
)
cmd.SetContext(ctx)
if originalPreRun != nil {
return originalPreRun(cmd, args)
}
return nil
}
// manually wrap RunE instead of using PersistentPostRunE because the
// latter only runs when RunE does _not_ return an error, but the
// tracing clean-up logic should always be invoked
originalPersistentPostRunE := cmd.PersistentPostRunE
cmd.PersistentPostRunE = func(cmd *cobra.Command, args []string) (err error) {
defer func() {
if cmdSpan != nil {
if err != nil && !errors.Is(err, context.Canceled) {
cmdSpan.SetStatus(codes.Error, "CLI command returned error")
cmdSpan.RecordError(err)
}
cmdSpan.End()
}
if tracingShutdown != nil {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = tracingShutdown(ctx)
}
}()
if originalPersistentPostRunE != nil {
return originalPersistentPostRunE(cmd, args)
}
return nil
}
cmd.SetFlagErrorFunc(func(c *cobra.Command, err error) error {
return dockercli.StatusError{
StatusCode: compose.CommandSyntaxFailure.ExitCode,

18
go.mod
View File

@ -4,6 +4,7 @@ go 1.20
require (
github.com/AlecAivazis/survey/v2 v2.3.6
github.com/Microsoft/go-winio v0.5.2
github.com/buger/goterm v1.0.4
github.com/compose-spec/compose-go v1.14.0
github.com/containerd/console v1.0.3
@ -16,12 +17,15 @@ require (
github.com/docker/docker v24.0.2+incompatible
github.com/docker/go-connections v0.4.0
github.com/docker/go-units v0.5.0
github.com/fsnotify/fsevents v0.1.1
github.com/golang/mock v1.6.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-version v1.6.0
github.com/jonboulle/clockwork v0.4.0
github.com/mattn/go-shellwords v1.0.12
github.com/mitchellh/mapstructure v1.5.0
github.com/moby/buildkit v0.11.7-0.20230519102302-348e79dfed17
github.com/moby/patternmatcher v0.5.0
github.com/moby/term v0.5.0
github.com/morikuni/aec v1.0.0
github.com/opencontainers/go-digest v1.0.0
@ -34,15 +38,19 @@ require (
github.com/theupdateframework/notary v0.7.0
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375
go.opentelemetry.io/otel v1.15.1
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1
go.opentelemetry.io/otel/sdk v1.4.1
go.opentelemetry.io/otel/trace v1.15.1
go.uber.org/goleak v1.2.1
golang.org/x/sync v0.2.0
google.golang.org/grpc v1.53.0
gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.4.0
)
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.3 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.5 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.0 // indirect
@ -70,7 +78,6 @@ require (
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/fsnotify/fsevents v0.1.1
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fvbommel/sortorder v1.0.2 // indirect
github.com/go-logr/logr v1.2.4 // indirect
@ -95,7 +102,6 @@ require (
github.com/imdario/mergo v0.3.15 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/gorm v1.9.11 // indirect
github.com/jonboulle/clockwork v0.4.0
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.16.5 // indirect
@ -107,7 +113,6 @@ require (
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/patternmatcher v0.5.0
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
@ -140,13 +145,9 @@ require (
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.29.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.4.1 // indirect
go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
go.opentelemetry.io/otel/metric v0.27.0 // indirect
go.opentelemetry.io/otel/sdk v1.4.1 // indirect
go.opentelemetry.io/otel/trace v1.15.1 // indirect
go.opentelemetry.io/proto/otlp v0.12.0 // indirect
golang.org/x/crypto v0.7.0 // indirect
golang.org/x/net v0.8.0 // indirect
@ -157,7 +158,6 @@ require (
golang.org/x/time v0.1.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.29.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect

View File

@ -0,0 +1,44 @@
//go:build !windows
/*
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"context"
"fmt"
"net"
"strings"
"syscall"
)
const maxUnixSocketPathSize = len(syscall.RawSockaddrUnix{}.Path)
func DialInMemory(ctx context.Context, addr string) (net.Conn, error) {
if !strings.HasPrefix(addr, "unix://") {
return nil, fmt.Errorf("not a Unix socket address: %s", addr)
}
addr = strings.TrimPrefix(addr, "unix://")
if len(addr) > maxUnixSocketPathSize {
//goland:noinspection GoErrorStringFormat
return nil, fmt.Errorf("Unix socket address is too long: %s", addr)
}
var d net.Dialer
return d.DialContext(ctx, "unix", addr)
}

View File

@ -0,0 +1,35 @@
/*
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"context"
"fmt"
"net"
"strings"
"github.com/Microsoft/go-winio"
)
func DialInMemory(ctx context.Context, addr string) (net.Conn, error) {
if !strings.HasPrefix(addr, "npipe://") {
return nil, fmt.Errorf("not a named pipe address: %s", addr)
}
addr = strings.TrimPrefix(addr, "npipe://")
return winio.DialPipeContext(ctx, addr)
}

View File

@ -0,0 +1,122 @@
/*
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"context"
"fmt"
"os"
"time"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/context/store"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
const otelConfigFieldName = "otel"
// traceClientFromDockerContext creates a gRPC OTLP client based on metadata
// from the active Docker CLI context.
func traceClientFromDockerContext(dockerCli command.Cli, otelEnv envMap) (otlptrace.Client, error) {
// attempt to extract an OTEL config from the Docker context to enable
// automatic integration with Docker Desktop;
cfg, err := ConfigFromDockerContext(dockerCli.ContextStore(), dockerCli.CurrentContext())
if err != nil {
return nil, fmt.Errorf("loading otel config from docker context metadata: %v", err)
}
if cfg.Endpoint == "" {
return nil, nil
}
// HACK: unfortunately _all_ public OTEL initialization functions
// implicitly read from the OS env, so temporarily unset them all and
// restore afterwards
defer func() {
for k, v := range otelEnv {
if err := os.Setenv(k, v); err != nil {
panic(fmt.Errorf("restoring env for %q: %v", k, err))
}
}
}()
for k := range otelEnv {
if err := os.Unsetenv(k); err != nil {
return nil, fmt.Errorf("stashing env for %q: %v", k, err)
}
}
dialCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
conn, err := grpc.DialContext(
dialCtx,
cfg.Endpoint,
grpc.WithContextDialer(DialInMemory),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithBlock(),
)
if err != nil {
return nil, fmt.Errorf("initializing otel connection from docker context metadata: %v", err)
}
client := otlptracegrpc.NewClient(otlptracegrpc.WithGRPCConn(conn))
return client, nil
}
// ConfigFromDockerContext inspects extra metadata included as part of the
// specified Docker context to try and extract a valid OTLP client configuration.
func ConfigFromDockerContext(st store.Store, name string) (OTLPConfig, error) {
meta, err := st.GetMetadata(name)
if err != nil {
return OTLPConfig{}, err
}
var otelCfg interface{}
switch m := meta.Metadata.(type) {
case command.DockerContext:
otelCfg = m.AdditionalFields[otelConfigFieldName]
case map[string]interface{}:
otelCfg = m[otelConfigFieldName]
}
otelMap, ok := otelCfg.(map[string]interface{})
if !ok {
return OTLPConfig{}, fmt.Errorf(
"unexpected type for field %q: %T (expected: %T)",
otelConfigFieldName,
otelCfg,
otelMap,
)
}
// keys from https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/
cfg := OTLPConfig{
Endpoint: valueOrDefault[string](otelMap, "OTEL_EXPORTER_OTLP_ENDPOINT"),
}
return cfg, nil
}
// valueOrDefault returns the type-cast value at the specified key in the map
// if present and the correct type; otherwise, it returns the default value for
// T.
func valueOrDefault[T any](m map[string]interface{}, key string) T {
if v, ok := m[key].(T); ok {
return v
}
return *new(T)
}

View File

@ -1,5 +1,5 @@
/*
Copyright 2020 Docker Compose CLI authors
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,22 +14,16 @@
limitations under the License.
*/
package compose
package tracing
import (
"github.com/moby/buildkit/util/tracing/detect"
"go.opentelemetry.io/otel"
_ "github.com/moby/buildkit/util/tracing/detect/delegated" //nolint:blank-imports
_ "github.com/moby/buildkit/util/tracing/env" //nolint:blank-imports
)
func init() {
detect.ServiceName = "compose"
// do not log tracing errors to stdio
otel.SetErrorHandler(skipErrors{})
}
// skipErrors is a no-op otel.ErrorHandler.
type skipErrors struct{}
func (skipErrors) Handle(err error) {}
// Handle does nothing, ignoring any errors passed to it.
func (skipErrors) Handle(_ error) {}
var _ otel.ErrorHandler = skipErrors{}

54
internal/tracing/mux.go Normal file
View File

@ -0,0 +1,54 @@
/*
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"context"
"log"
"github.com/hashicorp/go-multierror"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
type MuxExporter struct {
exporters []sdktrace.SpanExporter
}
func (m MuxExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error {
var eg multierror.Group
for i := range m.exporters {
exporter := m.exporters[i]
eg.Go(func() error {
return exporter.ExportSpans(ctx, spans)
})
}
if err := eg.Wait(); err != nil {
log.Fatal(err)
}
return nil
}
func (m MuxExporter) Shutdown(ctx context.Context) error {
var eg multierror.Group
for i := range m.exporters {
exporter := m.exporters[i]
eg.Go(func() error {
return exporter.Shutdown(ctx)
})
}
return eg.Wait()
}

151
internal/tracing/tracing.go Normal file
View File

@ -0,0 +1,151 @@
/*
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"context"
"errors"
"fmt"
"os"
"strconv"
"strings"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/tracing/detect"
_ "github.com/moby/buildkit/util/tracing/detect/delegated" //nolint:blank-imports
_ "github.com/moby/buildkit/util/tracing/env" //nolint:blank-imports
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
)
func init() {
detect.ServiceName = "compose"
// do not log tracing errors to stdio
otel.SetErrorHandler(skipErrors{})
}
var Tracer = otel.Tracer("compose")
// OTLPConfig contains the necessary values to initialize an OTLP client
// manually.
//
// This supports a minimal set of options based on what is necessary for
// automatic OTEL configuration from Docker context metadata.
type OTLPConfig struct {
Endpoint string
}
// ShutdownFunc flushes and stops an OTEL exporter.
type ShutdownFunc func(ctx context.Context) error
// envMap is a convenience type for OS environment variables.
type envMap map[string]string
func InitTracing(dockerCli command.Cli) (ShutdownFunc, error) {
// set global propagator to tracecontext (the default is no-op).
otel.SetTextMapPropagator(propagation.TraceContext{})
if v, _ := strconv.ParseBool(os.Getenv("COMPOSE_EXPERIMENTAL_OTEL")); !v {
return nil, nil
}
return InitProvider(dockerCli)
}
func InitProvider(dockerCli command.Cli) (ShutdownFunc, error) {
ctx := context.Background()
var errs []error
var exporters []sdktrace.SpanExporter
envClient, otelEnv := traceClientFromEnv()
if envClient != nil {
if envExporter, err := otlptrace.New(ctx, envClient); err != nil {
errs = append(errs, err)
} else if envExporter != nil {
exporters = append(exporters, envExporter)
}
}
if dcClient, err := traceClientFromDockerContext(dockerCli, otelEnv); err != nil {
errs = append(errs, err)
} else if dcClient != nil {
if dcExporter, err := otlptrace.New(ctx, dcClient); err != nil {
errs = append(errs, err)
} else if dcExporter != nil {
exporters = append(exporters, dcExporter)
}
}
if len(errs) != 0 {
return nil, errors.Join(errs...)
}
res, err := resource.New(
ctx,
resource.WithAttributes(
semconv.ServiceName("compose"),
),
)
if err != nil {
return nil, fmt.Errorf("failed to create resource: %v", err)
}
muxExporter := MuxExporter{exporters: exporters}
sp := sdktrace.NewSimpleSpanProcessor(muxExporter)
tracerProvider := sdktrace.NewTracerProvider(
sdktrace.WithSampler(sdktrace.AlwaysSample()),
sdktrace.WithResource(res),
sdktrace.WithSpanProcessor(sp),
)
otel.SetTracerProvider(tracerProvider)
// Shutdown will flush any remaining spans and shut down the exporter.
return tracerProvider.Shutdown, nil
}
// traceClientFromEnv creates a GRPC OTLP client based on OS environment
// variables.
//
// https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/
func traceClientFromEnv() (otlptrace.Client, envMap) {
hasOtelEndpointInEnv := false
otelEnv := make(map[string]string)
for _, kv := range os.Environ() {
k, v, ok := strings.Cut(kv, "=")
if !ok {
continue
}
if strings.HasPrefix(k, "OTEL_") {
otelEnv[k] = v
if strings.HasSuffix(k, "ENDPOINT") {
hasOtelEndpointInEnv = true
}
}
}
if !hasOtelEndpointInEnv {
return nil, nil
}
client := otlptracegrpc.NewClient()
return client, otelEnv
}

View File

@ -0,0 +1,60 @@
/*
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing_test
import (
"testing"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/context/store"
"github.com/stretchr/testify/require"
"github.com/docker/compose/v2/internal/tracing"
)
var testStoreCfg = store.NewConfig(
func() interface{} {
return &map[string]interface{}{}
},
)
func TestExtractOtelFromContext(t *testing.T) {
if testing.Short() {
t.Skip("Requires filesystem access")
}
dir := t.TempDir()
st := store.New(dir, testStoreCfg)
err := st.CreateOrUpdate(store.Metadata{
Name: "test",
Metadata: command.DockerContext{
Description: t.Name(),
AdditionalFields: map[string]interface{}{
"otel": map[string]interface{}{
"OTEL_EXPORTER_OTLP_ENDPOINT": "localhost:1234",
},
},
},
Endpoints: make(map[string]interface{}),
})
require.NoError(t, err)
cfg, err := tracing.ConfigFromDockerContext(st, "test")
require.NoError(t, err)
require.Equal(t, "localhost:1234", cfg.Endpoint)
}

View File

@ -1,5 +1,5 @@
/*
Copyright 2020 Docker Compose CLI authors
Copyright 2023 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.