move around OCI logic, auto fallback/retry 1.1 -> 1.0

Signed-off-by: Milas Bowman <milas.bowman@docker.com>
2023-12-04 22:39:23 -05:00 · 2023-12-04 22:39:23 -05:00 · 7c8ff36d78
parent 111ad3b039
commit 7c8ff36d78
7 changed files with 236 additions and 213 deletions
--- a/cmd/compose/publish.go
+++ b/cmd/compose/publish.go
@ -28,6 +28,7 @@ import (
 type publishOptions struct {
 	*ProjectOptions
 	resolveImageDigests bool
+	ociVersion          string
 }

 func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command {
@ -44,6 +45,7 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic
 	}
 	flags := cmd.Flags()
 	flags.BoolVar(&opts.resolveImageDigests, "resolve-image-digests", false, "Pin image tags to digests.")
+	flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI Image specification version (automatically determined by default)")
 	return cmd
 }

@ -55,5 +57,6 @@ func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service,

 	return backend.Publish(ctx, project, repository, api.PublishOptions{
 		ResolveImageDigests: opts.resolveImageDigests,
+		OCIVersion:          api.OCIVersion(opts.ociVersion),
 	})
 }
--- a/docs/reference/compose_alpha_publish.md
+++ b/docs/reference/compose_alpha_publish.md
@ -5,10 +5,11 @@ Publish compose application

 ### Options

-| Name                      | Type | Default | Description                     |
-|:--------------------------|:-----|:--------|:--------------------------------|
-| `--dry-run`               |      |         | Execute command in dry run mode |
-| `--resolve-image-digests` |      |         | Pin image tags to digests.      |
+| Name                      | Type     | Default | Description                                                           |
+|:--------------------------|:---------|:--------|:----------------------------------------------------------------------|
+| `--dry-run`               |          |         | Execute command in dry run mode                                       |
+| `--oci-version`           | `string` |         | OCI Image specification version (automatically determined by default) |
+| `--resolve-image-digests` |          |         | Pin image tags to digests.                                            |


 <!---MARKER_GEN_END-->
--- a/docs/reference/docker_compose_alpha_publish.yaml
+++ b/docs/reference/docker_compose_alpha_publish.yaml
@ -5,6 +5,16 @@ usage: docker compose alpha publish [OPTIONS] [REPOSITORY]
 pname: docker compose alpha
 plink: docker_compose_alpha.yaml
 options:
+    - option: oci-version
+      value_type: string
+      description: |
+        OCI Image specification version (automatically determined by default)
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
    - option: resolve-image-digests
      value_type: bool
      default_value: "false"
--- a/internal/ocipush/push.go
+++ b/internal/ocipush/push.go
@ -0,0 +1,183 @@
+/*
+   Copyright 2023 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ocipush
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	pusherrors "github.com/containerd/containerd/remotes/errors"
+	"github.com/distribution/reference"
+	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/compose/v2/pkg/api"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// clientAuthStatusCodes are client (4xx) errors that are authentication
+// related.
+var clientAuthStatusCodes = []int{
+	http.StatusUnauthorized,
+	http.StatusForbidden,
+	http.StatusProxyAuthRequired,
+}
+
+type Pushable struct {
+	Descriptor v1.Descriptor
+	Data       []byte
+}
+
+func DescriptorForComposeFile(path string, content []byte) v1.Descriptor {
+	return v1.Descriptor{
+		MediaType: "application/vnd.docker.compose.file+yaml",
+		Digest:    digest.FromString(string(content)),
+		Size:      int64(len(content)),
+		Annotations: map[string]string{
+			"com.docker.compose.version": api.ComposeVersion,
+			"com.docker.compose.file":    filepath.Base(path),
+		},
+	}
+}
+
+func PushManifest(
+	ctx context.Context,
+	resolver *imagetools.Resolver,
+	named reference.Named,
+	layers []Pushable,
+	ociVersion api.OCIVersion,
+) error {
+	// prepare to push the manifest by pushing the layers
+	layerDescriptors := make([]v1.Descriptor, len(layers))
+	for i := range layers {
+		layerDescriptors[i] = layers[i].Descriptor
+		if err := resolver.Push(ctx, named, layers[i].Descriptor, layers[i].Data); err != nil {
+			return err
+		}
+	}
+
+	if ociVersion != "" {
+		// if a version was explicitly specified, use it
+		return createAndPushManifest(ctx, resolver, named, layerDescriptors, ociVersion)
+	}
+
+	// try to push in the OCI 1.1 format but fallback to OCI 1.0 on 4xx errors
+	// (other than auth) since it's most likely the result of the registry not
+	// having support
+	err := createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_1)
+	var pushErr pusherrors.ErrUnexpectedStatus
+	if errors.As(err, &pushErr) && isNonAuthClientError(pushErr.StatusCode) {
+		// TODO(milas): show a warning here (won't work with logrus)
+		return createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_0)
+	}
+	return err
+}
+
+func createAndPushManifest(
+	ctx context.Context,
+	resolver *imagetools.Resolver,
+	named reference.Named,
+	layers []v1.Descriptor,
+	ociVersion api.OCIVersion,
+) error {
+	toPush, err := generateManifest(layers, ociVersion)
+	if err != nil {
+		return err
+	}
+	for _, p := range toPush {
+		err = resolver.Push(ctx, named, p.Descriptor, p.Data)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func isNonAuthClientError(statusCode int) bool {
+	if statusCode < 400 || statusCode >= 500 {
+		// not a client error
+		return false
+	}
+	for _, v := range clientAuthStatusCodes {
+		if statusCode == v {
+			// client auth error
+			return false
+		}
+	}
+	// any other 4xx client error
+	return true
+}
+
+func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]Pushable, error) {
+	var toPush []Pushable
+	var config v1.Descriptor
+	var artifactType string
+	switch ociCompat {
+	case api.OCIVersion1_0:
+		configData, err := json.Marshal(v1.ImageConfig{})
+		if err != nil {
+			return nil, err
+		}
+		config = v1.Descriptor{
+			MediaType: v1.MediaTypeImageConfig,
+			Digest:    digest.FromBytes(configData),
+			Size:      int64(len(configData)),
+		}
+		// N.B. OCI 1.0 does NOT support specifying the artifact type, so it's
+		//		left as an empty string to omit it from the marshaled JSON
+		artifactType = ""
+		toPush = append(toPush, Pushable{Descriptor: config, Data: configData})
+	case api.OCIVersion1_1:
+		config = v1.DescriptorEmptyJSON
+		artifactType = "application/vnd.docker.compose.project"
+		// N.B. the descriptor has the data embedded in it
+		toPush = append(toPush, Pushable{Descriptor: config, Data: nil})
+	default:
+		return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat)
+	}
+
+	manifest, err := json.Marshal(v1.Manifest{
+		Versioned:    specs.Versioned{SchemaVersion: 2},
+		MediaType:    v1.MediaTypeImageManifest,
+		ArtifactType: artifactType,
+		Config:       config,
+		Layers:       layers,
+		Annotations: map[string]string{
+			"org.opencontainers.image.created": time.Now().Format(time.RFC3339),
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	manifestDescriptor := v1.Descriptor{
+		MediaType: v1.MediaTypeImageManifest,
+		Digest:    digest.FromString(string(manifest)),
+		Size:      int64(len(manifest)),
+		Annotations: map[string]string{
+			"com.docker.compose.version": api.ComposeVersion,
+		},
+		ArtifactType: artifactType,
+	}
+	toPush = append(toPush, Pushable{Descriptor: manifestDescriptor, Data: manifest})
+	return toPush, nil
+}
--- a/pkg/api/api.go
+++ b/pkg/api/api.go
@ -361,9 +361,28 @@ type PortOptions struct {
 	Index    int
 }

+// OCIVersion controls manifest generation to ensure compatibility
+// with different registries.
+//
+// Currently, this is not exposed as an option to the user – Compose uses
+// OCI 1.0 mode automatically for ECR registries based on domain and OCI 1.1
+// for all other registries.
+//
+// There are likely other popular registries that do not support the OCI 1.1
+// format, so it might make sense to expose this as a CLI flag or see if
+// there's a way to generically probe the registry for support level.
+type OCIVersion string
+
+const (
+	OCIVersion1_0 OCIVersion = "1.0"
+	OCIVersion1_1 OCIVersion = "1.1"
+)
+
 // PublishOptions group options of the Publish API
 type PublishOptions struct {
 	ResolveImageDigests bool
+
+	OCIVersion OCIVersion
 }

 func (e Event) String() string {
--- a/pkg/compose/publish.go
+++ b/pkg/compose/publish.go
@ -18,38 +18,15 @@ package compose

 import (
 	"context"
-	"encoding/json"
-	"fmt"
 	"os"
-	"path/filepath"
-	"strings"
-	"time"

 	"github.com/compose-spec/compose-go/types"
 	"github.com/distribution/reference"
 	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/compose/v2/internal/ocipush"
 	"github.com/docker/compose/v2/pkg/api"
 	"github.com/docker/compose/v2/pkg/progress"
 	"github.com/opencontainers/go-digest"
-	"github.com/opencontainers/image-spec/specs-go"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
-)
-
-// ociCompatibilityMode controls manifest generation to ensure compatibility
-// with different registries.
-//
-// Currently, this is not exposed as an option to the user – Compose uses
-// OCI 1.0 mode automatically for ECR registries based on domain and OCI 1.1
-// for all other registries.
-//
-// There are likely other popular registries that do not support the OCI 1.1
-// format, so it might make sense to expose this as a CLI flag or see if
-// there's a way to generically probe the registry for support level.
-type ociCompatibilityMode string
-
-const (
-	ociCompatibility1_0 ociCompatibilityMode = "1.0"
-	ociCompatibility1_1 ociCompatibilityMode = "1.1"
 )

 func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
@ -73,18 +50,18 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 		Auth: s.configFile(),
 	})

-	var layers []v1.Descriptor
+	var layers []ocipush.Pushable
 	for _, file := range project.ComposeFiles {
 		f, err := os.ReadFile(file)
 		if err != nil {
 			return err
 		}

-		layer, err := s.pushComposeFile(ctx, file, f, resolver, named)
-		if err != nil {
-			return err
-		}
-		layers = append(layers, layer)
+		layerDescriptor := ocipush.DescriptorForComposeFile(file, f)
+		layers = append(layers, ocipush.Pushable{
+			Descriptor: layerDescriptor,
+			Data:       f,
+		})
 	}

 	if options.ResolveImageDigests {
@ -93,17 +70,11 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 			return err
 		}

-		layer, err := s.pushComposeFile(ctx, "image-digests.yaml", yaml, resolver, named)
-		if err != nil {
-			return err
-		}
-		layers = append(layers, layer)
-	}
-
-	ociCompat := inferOCIVersion(named)
-	toPush, err := s.generateManifest(layers, ociCompat)
-	if err != nil {
-		return err
+		layerDescriptor := ocipush.DescriptorForComposeFile("image-diegests.yaml", yaml)
+		layers = append(layers, ocipush.Pushable{
+			Descriptor: layerDescriptor,
+			Data:       yaml,
+		})
 	}

 	w := progress.ContextWriter(ctx)
@ -113,12 +84,11 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 		Status: progress.Working,
 	})
 	if !s.dryRun {
-		for _, p := range toPush {
-			err = resolver.Push(ctx, named, p.Descriptor, p.Data)
-			if err != nil {
-				return err
-			}
+		err = ocipush.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
+		if err != nil {
+			return err
 		}
+
 		if err != nil {
 			w.Event(progress.Event{
 				ID:     repository,
@ -136,66 +106,6 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 	return nil
 }

-type push struct {
-	Descriptor v1.Descriptor
-	Data       []byte
-}
-
-func (s *composeService) generateManifest(layers []v1.Descriptor, ociCompat ociCompatibilityMode) ([]push, error) {
-	var toPush []push
-	var config v1.Descriptor
-	var artifactType string
-	switch ociCompat {
-	case ociCompatibility1_0:
-		configData, err := json.Marshal(v1.ImageConfig{})
-		if err != nil {
-			return nil, err
-		}
-		config = v1.Descriptor{
-			MediaType: v1.MediaTypeImageConfig,
-			Digest:    digest.FromBytes(configData),
-			Size:      int64(len(configData)),
-		}
-		// N.B. OCI 1.0 does NOT support specifying the artifact type, so it's
-		//		left as an empty string to omit it from the marshaled JSON
-		artifactType = ""
-		toPush = append(toPush, push{Descriptor: config, Data: configData})
-	case ociCompatibility1_1:
-		config = v1.DescriptorEmptyJSON
-		artifactType = "application/vnd.docker.compose.project"
-		// N.B. the descriptor has the data embedded in it
-		toPush = append(toPush, push{Descriptor: config, Data: nil})
-	default:
-		return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat)
-	}
-
-	manifest, err := json.Marshal(v1.Manifest{
-		Versioned:    specs.Versioned{SchemaVersion: 2},
-		MediaType:    v1.MediaTypeImageManifest,
-		ArtifactType: artifactType,
-		Config:       config,
-		Layers:       layers,
-		Annotations: map[string]string{
-			"org.opencontainers.image.created": time.Now().Format(time.RFC3339),
-		},
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	manifestDescriptor := v1.Descriptor{
-		MediaType: v1.MediaTypeImageManifest,
-		Digest:    digest.FromString(string(manifest)),
-		Size:      int64(len(manifest)),
-		Annotations: map[string]string{
-			"com.docker.compose.version": api.ComposeVersion,
-		},
-		ArtifactType: artifactType,
-	}
-	toPush = append(toPush, push{Descriptor: manifestDescriptor, Data: manifest})
-	return toPush, nil
-}
-
 func (s *composeService) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) {
 	project.ApplyProfiles([]string{"*"})
 	err := project.ResolveImages(func(named reference.Named) (digest.Digest, error) {
@ -221,50 +131,3 @@ func (s *composeService) generateImageDigestsOverride(ctx context.Context, proje
 	}
 	return override.MarshalYAML()
 }
-
-func (s *composeService) pushComposeFile(ctx context.Context, file string, content []byte, resolver *imagetools.Resolver, named reference.Named) (v1.Descriptor, error) {
-	w := progress.ContextWriter(ctx)
-	w.Event(progress.Event{
-		ID:     file,
-		Text:   "publishing",
-		Status: progress.Working,
-	})
-	layer := v1.Descriptor{
-		MediaType: "application/vnd.docker.compose.file+yaml",
-		Digest:    digest.FromString(string(content)),
-		Size:      int64(len(content)),
-		Annotations: map[string]string{
-			"com.docker.compose.version": api.ComposeVersion,
-			"com.docker.compose.file":    filepath.Base(file),
-		},
-	}
-	err := resolver.Push(ctx, named, layer, content)
-	w.Event(progress.Event{
-		ID:     file,
-		Text:   "published",
-		Status: statusFor(err),
-	})
-	return layer, err
-}
-
-func statusFor(err error) progress.EventStatus {
-	if err != nil {
-		return progress.Error
-	}
-	return progress.Done
-}
-
-// inferOCIVersion uses OCI 1.1 by default but falls back to OCI 1.0 if the
-// registry domain is known to require it.
-//
-// This is not ideal - with private registries, there isn't a bounded set of
-// domains. As it stands, it's primarily intended for compatibility with AWS
-// Elastic Container Registry (ECR) due to its ubiquity.
-func inferOCIVersion(named reference.Named) ociCompatibilityMode {
-	domain := reference.Domain(named)
-	if strings.HasSuffix(domain, "amazonaws.com") {
-		return ociCompatibility1_0
-	} else {
-		return ociCompatibility1_1
-	}
-}
--- a/pkg/compose/publish_test.go
+++ b/pkg/compose/publish_test.go
@ -1,56 +0,0 @@
-/*
-   Copyright 2023 Docker Compose CLI authors
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package compose
-
-import (
-	"testing"
-
-	"github.com/distribution/reference"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestInferOCIVersion(t *testing.T) {
-	tests := []struct {
-		ref  string
-		want ociCompatibilityMode
-	}{
-		{
-			ref:  "175142243308.dkr.ecr.us-east-1.amazonaws.com/compose:test",
-			want: ociCompatibility1_0,
-		},
-		{
-			ref:  "my-image:latest",
-			want: ociCompatibility1_1,
-		},
-		{
-			ref:  "docker.io/docker/compose:test",
-			want: ociCompatibility1_1,
-		},
-		{
-			ref:  "ghcr.io/docker/compose:test",
-			want: ociCompatibility1_1,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.ref, func(t *testing.T) {
-			named, err := reference.ParseDockerRef(tt.ref)
-			require.NoErrorf(t, err, "Test issue - invalid ref: %s", tt.ref)
-			assert.Equalf(t, tt.want, inferOCIVersion(named), "inferOCIVersion(%s)", tt.ref)
-		})
-	}
-}