diff --git a/cmd/compose/publish.go b/cmd/compose/publish.go index 14b22c880..174c8fa86 100644 --- a/cmd/compose/publish.go +++ b/cmd/compose/publish.go @@ -34,6 +34,7 @@ type publishOptions struct { ociVersion string withEnvironment bool assumeYes bool + app bool } func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command { @@ -53,6 +54,7 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI image/artifact specification version (automatically determined by default)") flags.BoolVar(&opts.withEnvironment, "with-env", false, "Include environment variables in the published OCI artifact") flags.BoolVarP(&opts.assumeYes, "yes", "y", false, `Assume "yes" as answer to all prompts`) + flags.BoolVar(&opts.app, "app", false, "Published compose application (includes referenced images)") flags.SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { // assumeYes was introduced by mistake as `--y` if name == "y" { @@ -76,7 +78,8 @@ func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service, } return backend.Publish(ctx, project, repository, api.PublishOptions{ - ResolveImageDigests: opts.resolveImageDigests, + ResolveImageDigests: opts.resolveImageDigests || opts.app, + Application: opts.app, OCIVersion: api.OCIVersion(opts.ociVersion), WithEnvironment: opts.withEnvironment, AssumeYes: opts.assumeYes, diff --git a/docs/reference/compose_publish.md b/docs/reference/compose_publish.md index 8e5d18133..9a82fc260 100644 --- a/docs/reference/compose_publish.md +++ b/docs/reference/compose_publish.md @@ -7,6 +7,7 @@ Publish compose application | Name | Type | Default | Description | |:--------------------------|:---------|:--------|:-------------------------------------------------------------------------------| +| `--app` | `bool` | | Published compose application (includes referenced images) | | `--dry-run` | `bool` | | Execute command in dry run mode | | `--oci-version` | `string` | | OCI image/artifact specification version (automatically determined by default) | | `--resolve-image-digests` | `bool` | | Pin image tags to digests | diff --git a/docs/reference/docker_compose_alpha_publish.yaml b/docs/reference/docker_compose_alpha_publish.yaml index 2c9224939..9ab6a20c4 100644 --- a/docs/reference/docker_compose_alpha_publish.yaml +++ b/docs/reference/docker_compose_alpha_publish.yaml @@ -5,6 +5,16 @@ usage: docker compose alpha publish [OPTIONS] REPOSITORY[:TAG] pname: docker compose alpha plink: docker_compose_alpha.yaml options: + - option: app + value_type: bool + default_value: "false" + description: Published compose application (includes referenced images) + deprecated: false + hidden: false + experimental: false + experimentalcli: false + kubernetes: false + swarm: false - option: oci-version value_type: string description: | diff --git a/docs/reference/docker_compose_publish.yaml b/docs/reference/docker_compose_publish.yaml index 44a7a46dd..e939a8ff8 100644 --- a/docs/reference/docker_compose_publish.yaml +++ b/docs/reference/docker_compose_publish.yaml @@ -5,6 +5,16 @@ usage: docker compose publish [OPTIONS] REPOSITORY[:TAG] pname: docker compose plink: docker_compose.yaml options: + - option: app + value_type: bool + default_value: "false" + description: Published compose application (includes referenced images) + deprecated: false + hidden: false + experimental: false + experimentalcli: false + kubernetes: false + swarm: false - option: oci-version value_type: string description: | diff --git a/internal/oci/push.go b/internal/oci/push.go index 5aa4f80b3..caa63881c 100644 --- a/internal/oci/push.go +++ b/internal/oci/push.go @@ -28,7 +28,6 @@ import ( "github.com/containerd/containerd/v2/core/remotes" pusherrors "github.com/containerd/containerd/v2/core/remotes/errors" - "github.com/containerd/errdefs" "github.com/distribution/reference" "github.com/docker/compose/v2/pkg/api" "github.com/opencontainers/go-digest" @@ -94,12 +93,12 @@ func DescriptorForEnvFile(path string, content []byte) v1.Descriptor { } } -func PushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) error { +func PushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) (v1.Descriptor, error) { // Check if we need an extra empty layer for the manifest config if ociVersion == api.OCIVersion1_1 || ociVersion == "" { err := push(ctx, resolver, named, v1.DescriptorEmptyJSON) if err != nil { - return err + return v1.Descriptor{}, err } } // prepare to push the manifest by pushing the layers @@ -107,7 +106,7 @@ func PushManifest(ctx context.Context, resolver remotes.Resolver, named referenc for i := range layers { layerDescriptors[i] = layers[i] if err := push(ctx, resolver, named, layers[i]); err != nil { - return err + return v1.Descriptor{}, err } } @@ -119,13 +118,13 @@ func PushManifest(ctx context.Context, resolver remotes.Resolver, named referenc // try to push in the OCI 1.1 format but fallback to OCI 1.0 on 4xx errors // (other than auth) since it's most likely the result of the registry not // having support - err := createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_1) + descriptor, err := createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_1) var pushErr pusherrors.ErrUnexpectedStatus if errors.As(err, &pushErr) && isNonAuthClientError(pushErr.StatusCode) { // TODO(milas): show a warning here (won't work with logrus) return createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_0) } - return err + return descriptor, err } func push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, descriptor v1.Descriptor) error { @@ -134,37 +133,21 @@ func push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, d return err } - pusher, err := resolver.Pusher(ctx, fullRef.String()) - if err != nil { - return err - } - push, err := pusher.Push(ctx, descriptor) - if errdefs.IsAlreadyExists(err) { - return nil - } - if err != nil { - return err - } - defer func() { - _ = push.Close() - }() - - _, err = push.Write(descriptor.Data) - return err + return Push(ctx, resolver, fullRef, descriptor) } -func createAndPushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) error { - toPush, err := generateManifest(layers, ociVersion) +func createAndPushManifest(ctx context.Context, resolver remotes.Resolver, named reference.Named, layers []v1.Descriptor, ociVersion api.OCIVersion) (v1.Descriptor, error) { + descriptor, toPush, err := generateManifest(layers, ociVersion) if err != nil { - return err + return v1.Descriptor{}, err } for _, p := range toPush { err = push(ctx, resolver, named, p) if err != nil { - return err + return v1.Descriptor{}, err } } - return nil + return descriptor, nil } func isNonAuthClientError(statusCode int) bool { @@ -175,7 +158,7 @@ func isNonAuthClientError(statusCode int) bool { return !slices.Contains(clientAuthStatusCodes, statusCode) } -func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.Descriptor, error) { +func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) (v1.Descriptor, []v1.Descriptor, error) { var toPush []v1.Descriptor var config v1.Descriptor var artifactType string @@ -205,10 +188,9 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.De case api.OCIVersion1_1: config = v1.DescriptorEmptyJSON artifactType = ComposeProjectArtifactType - // N.B. the descriptor has the data embedded in it toPush = append(toPush, config) default: - return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat) + return v1.Descriptor{}, nil, fmt.Errorf("unsupported OCI version: %s", ociCompat) } manifest, err := json.Marshal(v1.Manifest{ @@ -222,7 +204,7 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.De }, }) if err != nil { - return nil, err + return v1.Descriptor{}, nil, err } manifestDescriptor := v1.Descriptor{ @@ -236,5 +218,5 @@ func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]v1.De Data: manifest, } toPush = append(toPush, manifestDescriptor) - return toPush, nil + return manifestDescriptor, toPush, nil } diff --git a/internal/oci/resolver.go b/internal/oci/resolver.go index 744e0b05e..a71d335ba 100644 --- a/internal/oci/resolver.go +++ b/internal/oci/resolver.go @@ -19,12 +19,17 @@ package oci import ( "context" "io" + "net/url" + "strings" "github.com/containerd/containerd/v2/core/remotes" "github.com/containerd/containerd/v2/core/remotes/docker" + "github.com/containerd/containerd/v2/pkg/labels" + "github.com/containerd/errdefs" "github.com/distribution/reference" "github.com/docker/cli/cli/config/configfile" "github.com/docker/compose/v2/internal/registry" + "github.com/moby/buildkit/util/contentutil" spec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -70,3 +75,60 @@ func Get(ctx context.Context, resolver remotes.Resolver, ref reference.Named) (s } return descriptor, content, nil } + +func Copy(ctx context.Context, resolver remotes.Resolver, image reference.Named, named reference.Named) (spec.Descriptor, error) { + src, desc, err := resolver.Resolve(ctx, image.String()) + if err != nil { + return spec.Descriptor{}, err + } + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + // set LabelDistributionSource so push will actually use a registry mount + refspec := reference.TrimNamed(image).String() + u, err := url.Parse("dummy://" + refspec) + if err != nil { + return spec.Descriptor{}, err + } + source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/") + desc.Annotations[labels.LabelDistributionSource+"."+source] = repo + + p, err := resolver.Pusher(ctx, named.Name()) + if err != nil { + return spec.Descriptor{}, err + } + f, err := resolver.Fetcher(ctx, src) + if err != nil { + return spec.Descriptor{}, err + } + + err = contentutil.CopyChain(ctx, + contentutil.FromPusher(p), + contentutil.FromFetcher(f), desc) + return desc, err +} + +func Push(ctx context.Context, resolver remotes.Resolver, ref reference.Named, descriptor spec.Descriptor) error { + pusher, err := resolver.Pusher(ctx, ref.String()) + if err != nil { + return err + } + ctx = remotes.WithMediaTypeKeyPrefix(ctx, ComposeYAMLMediaType, "artifact-") + ctx = remotes.WithMediaTypeKeyPrefix(ctx, ComposeEnvFileMediaType, "artifact-") + ctx = remotes.WithMediaTypeKeyPrefix(ctx, ComposeEmptyConfigMediaType, "config-") + ctx = remotes.WithMediaTypeKeyPrefix(ctx, spec.MediaTypeEmptyJSON, "config-") + + push, err := pusher.Push(ctx, descriptor) + if errdefs.IsAlreadyExists(err) { + return nil + } + if err != nil { + return err + } + defer func() { + _ = push.Close() + }() + + _, err = push.Write(descriptor.Data) + return err +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 83fda7f50..c82c8618a 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -444,9 +444,10 @@ const ( // PublishOptions group options of the Publish API type PublishOptions struct { ResolveImageDigests bool + Application bool WithEnvironment bool - AssumeYes bool + AssumeYes bool OCIVersion OCIVersion } diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index 1580858dd..8f3fcb819 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/json" "errors" "fmt" "io" @@ -36,6 +37,8 @@ import ( "github.com/docker/compose/v2/pkg/compose/transform" "github.com/docker/compose/v2/pkg/progress" "github.com/docker/compose/v2/pkg/prompt" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -45,6 +48,7 @@ func (s *composeService) Publish(ctx context.Context, project *types.Project, re }, s.stdinfo(), "Publishing") } +//nolint:gocyclo func (s *composeService) publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { accept, err := s.preChecks(project, options) if err != nil { @@ -106,7 +110,7 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re Status: progress.Working, }) if !s.dryRun { - err = oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion) + descriptor, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion) if err != nil { w.Event(progress.Event{ ID: repository, @@ -115,6 +119,50 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re }) return err } + + if options.Application { + manifests := []v1.Descriptor{} + for _, service := range project.Services { + ref, err := reference.ParseDockerRef(service.Image) + if err != nil { + return err + } + + manifest, err := oci.Copy(ctx, resolver, ref, named) + if err != nil { + return err + } + manifests = append(manifests, manifest) + } + + descriptor.Data = nil + index, err := json.Marshal(v1.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: v1.MediaTypeImageIndex, + Manifests: manifests, + Subject: &descriptor, + Annotations: map[string]string{ + "com.docker.compose.version": api.ComposeVersion, + }, + }) + if err != nil { + return err + } + imagesDescriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageIndex, + ArtifactType: oci.ComposeProjectArtifactType, + Digest: digest.FromString(string(index)), + Size: int64(len(index)), + Annotations: map[string]string{ + "com.docker.compose.version": api.ComposeVersion, + }, + Data: index, + } + err = oci.Push(ctx, resolver, reference.TrimNamed(named), imagesDescriptor) + if err != nil { + return err + } + } } w.Event(progress.Event{ ID: repository, diff --git a/pkg/remote/oci.go b/pkg/remote/oci.go index 2833b3ad1..11699a1ea 100644 --- a/pkg/remote/oci.go +++ b/pkg/remote/oci.go @@ -26,11 +26,12 @@ import ( "strings" "github.com/compose-spec/compose-go/v2/loader" + "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/core/remotes" "github.com/distribution/reference" "github.com/docker/cli/cli/command" "github.com/docker/compose/v2/internal/oci" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + spec "github.com/opencontainers/image-spec/specs-go/v1" ) const ( @@ -67,6 +68,7 @@ func (g ociRemoteLoader) Accept(path string) bool { return strings.HasPrefix(path, OciPrefix) } +//nolint:gocyclo func (g ociRemoteLoader) Load(ctx context.Context, path string) (string, error) { enabled, err := ociRemoteLoaderEnabled() if err != nil { @@ -91,7 +93,7 @@ func (g ociRemoteLoader) Load(ctx context.Context, path string) (string, error) descriptor, content, err := oci.Get(ctx, resolver, ref) if err != nil { - return "", err + return "", fmt.Errorf("failed to pull OCI resource %q: %w", ref, err) } cache, err := cacheDir() @@ -101,7 +103,35 @@ func (g ociRemoteLoader) Load(ctx context.Context, path string) (string, error) local = filepath.Join(cache, descriptor.Digest.Hex()) if _, err = os.Stat(local); os.IsNotExist(err) { - var manifest v1.Manifest + + // a Compose application bundle is published as image index + if images.IsIndexType(descriptor.MediaType) { + var index spec.Index + err = json.Unmarshal(content, &index) + if err != nil { + return "", err + } + found := false + for _, manifest := range index.Manifests { + if manifest.ArtifactType != oci.ComposeProjectArtifactType { + continue + } + found = true + digested, err := reference.WithDigest(ref, manifest.Digest) + if err != nil { + return "", err + } + descriptor, content, err = oci.Get(ctx, resolver, digested) + if err != nil { + return "", fmt.Errorf("failed to pull OCI resource %q: %w", ref, err) + } + } + if !found { + return "", fmt.Errorf("OCI index %s doesn't refer to compose artifacts", ref) + } + } + + var manifest spec.Manifest err = json.Unmarshal(content, &manifest) if err != nil { return "", err @@ -123,7 +153,7 @@ func (g ociRemoteLoader) Dir(path string) string { return g.known[path] } -func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, manifest v1.Manifest, ref reference.Named, resolver remotes.Resolver) error { //nolint:gocyclo +func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, manifest spec.Manifest, ref reference.Named, resolver remotes.Resolver) error { //nolint:gocyclo err := os.MkdirAll(local, 0o700) if err != nil { return err @@ -173,7 +203,7 @@ func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, man return nil } -func writeComposeFile(layer v1.Descriptor, i int, f *os.File, content []byte) error { +func writeComposeFile(layer spec.Descriptor, i int, f *os.File, content []byte) error { if _, ok := layer.Annotations["com.docker.compose.file"]; i > 0 && ok { _, err := f.Write([]byte("\n---\n")) if err != nil { @@ -184,7 +214,7 @@ func writeComposeFile(layer v1.Descriptor, i int, f *os.File, content []byte) er return err } -func writeEnvFile(layer v1.Descriptor, local string, content []byte) error { +func writeEnvFile(layer spec.Descriptor, local string, content []byte) error { envfilePath, ok := layer.Annotations["com.docker.compose.envfile"] if !ok { return fmt.Errorf("missing annotation com.docker.compose.envfile in layer %q", layer.Digest)