diff --git a/pkg/api/labels.go b/pkg/api/labels.go
index 8e0a8604a..4352d9c03 100644
--- a/pkg/api/labels.go
+++ b/pkg/api/labels.go
@@ -47,14 +47,14 @@ const (
 	OneoffLabel = "com.docker.compose.oneoff"
 	// SlugLabel stores unique slug used for one-off container identity
 	SlugLabel = "com.docker.compose.slug"
-	// ImageNameLabel stores the content of the image section in the compose file
-	ImageNameLabel = "com.docker.compose.image_name"
 	// ImageDigestLabel stores digest of the container image used to run service
 	ImageDigestLabel = "com.docker.compose.image"
 	// DependenciesLabel stores service dependencies
 	DependenciesLabel = "com.docker.compose.depends_on"
-	// VersionLabel stores the compose tool version used to run application
+	// VersionLabel stores the compose tool version used to build/run application
 	VersionLabel = "com.docker.compose.version"
+	// ImageBuilderLabel stores the builder (classic or BuildKit) used to produce the image.
+	ImageBuilderLabel = "com.docker.compose.image.builder"
 )
 
 // ComposeVersion is the compose tool version as declared by label VersionLabel
diff --git a/pkg/compose/build.go b/pkg/compose/build.go
index dcb4c8e7a..6830c359a 100644
--- a/pkg/compose/build.go
+++ b/pkg/compose/build.go
@@ -145,7 +145,6 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types.
 				project.Services[i].Labels = types.Labels{}
 			}
 			project.Services[i].CustomLabels.Add(api.ImageDigestLabel, digest)
-			project.Services[i].CustomLabels.Add(api.ImageNameLabel, service.Image)
 		}
 	}
 	return nil
@@ -207,7 +206,6 @@ func (s *composeService) getLocalImagesDigests(ctx context.Context, project *typ
 		digest, ok := images[imgName]
 		if ok {
 			project.Services[i].CustomLabels.Add(api.ImageDigestLabel, digest)
-			project.Services[i].CustomLabels.Add(api.ImageNameLabel, project.Services[i].Image)
 		}
 	}
 
@@ -267,6 +265,8 @@ func (s *composeService) toBuildOptions(project *types.Project, service types.Se
 		tags = append(tags, service.Build.Tags...)
 	}
 
+	imageLabels := getImageBuildLabels(project, service)
+
 	return build.Options{
 		Inputs: build.Inputs{
 			ContextPath:    service.Build.Context,
@@ -281,7 +281,7 @@ func (s *composeService) toBuildOptions(project *types.Project, service types.Se
 		Target:      service.Build.Target,
 		Exports:     []bclient.ExportEntry{{Type: "image", Attrs: map[string]string{}}},
 		Platforms:   plats,
-		Labels:      service.Build.Labels,
+		Labels:      imageLabels,
 		NetworkMode: service.Build.Network,
 		ExtraHosts:  service.Build.ExtraHosts.AsList(),
 		Session:     sessionConfig,
@@ -331,7 +331,6 @@ func sshAgentProvider(sshKeys types.SSHConfig) (session.Attachable, error) {
 }
 
 func addSecretsConfig(project *types.Project, service types.ServiceConfig) (session.Attachable, error) {
-
 	var sources []secretsprovider.Source
 	for _, secret := range service.Build.Secrets {
 		config := project.Secrets[secret.Source]
@@ -379,6 +378,20 @@ func addPlatforms(project *types.Project, service types.ServiceConfig) ([]specs.
 	return plats, nil
 }
 
+func getImageBuildLabels(project *types.Project, service types.ServiceConfig) types.Labels {
+	ret := make(types.Labels)
+	if service.Build != nil {
+		for k, v := range service.Build.Labels {
+			ret.Add(k, v)
+		}
+	}
+
+	ret.Add(api.VersionLabel, api.ComposeVersion)
+	ret.Add(api.ProjectLabel, project.Name)
+	ret.Add(api.ServiceLabel, service.Name)
+	return ret
+}
+
 func useDockerDefaultPlatform(project *types.Project, platformList types.StringList) ([]specs.Platform, error) {
 	var plats []specs.Platform
 	if platform, ok := project.Environment["DOCKER_DEFAULT_PLATFORM"]; ok {
diff --git a/pkg/compose/build_classic.go b/pkg/compose/build_classic.go
index 3a41e618b..6d2d65c3d 100644
--- a/pkg/compose/build_classic.go
+++ b/pkg/compose/build_classic.go
@@ -93,6 +93,11 @@ func (s *composeService) doBuildClassicSimpleImage(ctx context.Context, options
 		return "", errors.Errorf("this builder doesn't support multi-arch build, set DOCKER_BUILDKIT=1 to use multi-arch builder")
 	}
 
+	if options.Labels == nil {
+		options.Labels = make(map[string]string)
+	}
+	options.Labels[api.ImageBuilderLabel] = "classic"
+
 	switch {
 	case isLocalDir(specifiedContext):
 		contextDir, relDockerfile, err = build.GetContextFromLocalDir(specifiedContext, dockerfileName)
diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go
index ef5d58780..743b25cbb 100644
--- a/pkg/compose/compose.go
+++ b/pkg/compose/compose.go
@@ -29,11 +29,12 @@ import (
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/cli/cli/streams"
-	"github.com/docker/compose/v2/pkg/api"
 	moby "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
+
+	"github.com/docker/compose/v2/pkg/api"
 )
 
 // NewComposeService create a local implementation of the compose.Service API
@@ -115,13 +116,9 @@ func (s *composeService) projectFromName(containers Containers, projectName stri
 		serviceLabel := c.Labels[api.ServiceLabel]
 		_, ok := set[serviceLabel]
 		if !ok {
-			serviceImage := c.Image
-			if serviceNameFromLabel, ok := c.Labels[api.ImageNameLabel]; ok {
-				serviceImage = serviceNameFromLabel
-			}
 			set[serviceLabel] = &types.ServiceConfig{
 				Name:   serviceLabel,
-				Image:  serviceImage,
+				Image:  c.Image,
 				Labels: c.Labels,
 			}
 		}
diff --git a/pkg/compose/down.go b/pkg/compose/down.go
index 091480402..bdcc4ca91 100644
--- a/pkg/compose/down.go
+++ b/pkg/compose/down.go
@@ -86,7 +86,11 @@ func (s *composeService) down(ctx context.Context, projectName string, options a
 	ops := s.ensureNetworksDown(ctx, project, w)
 
 	if options.Images != "" {
-		ops = append(ops, s.ensureImagesDown(ctx, project, options, w)...)
+		imgOps, err := s.ensureImagesDown(ctx, project, options, w)
+		if err != nil {
+			return err
+		}
+		ops = append(ops, imgOps...)
 	}
 
 	if options.Volumes {
@@ -118,15 +122,25 @@ func (s *composeService) ensureVolumesDown(ctx context.Context, project *types.P
 	return ops
 }
 
-func (s *composeService) ensureImagesDown(ctx context.Context, project *types.Project, options api.DownOptions, w progress.Writer) []downOp {
+func (s *composeService) ensureImagesDown(ctx context.Context, project *types.Project, options api.DownOptions, w progress.Writer) ([]downOp, error) {
+	imagePruner := NewImagePruner(s.apiClient(), project)
+	pruneOpts := ImagePruneOptions{
+		Mode:          ImagePruneMode(options.Images),
+		RemoveOrphans: options.RemoveOrphans,
+	}
+	images, err := imagePruner.ImagesToPrune(ctx, pruneOpts)
+	if err != nil {
+		return nil, err
+	}
+
 	var ops []downOp
-	for image := range s.getServiceImagesToRemove(options, project) {
-		image := image
+	for i := range images {
+		img := images[i]
 		ops = append(ops, func() error {
-			return s.removeImage(ctx, image, w)
+			return s.removeImage(ctx, img, w)
 		})
 	}
-	return ops
+	return ops, nil
 }
 
 func (s *composeService) ensureNetworksDown(ctx context.Context, project *types.Project, w progress.Writer) []downOp {
@@ -190,19 +204,6 @@ func (s *composeService) removeNetwork(ctx context.Context, name string, w progr
 	return nil
 }
 
-func (s *composeService) getServiceImagesToRemove(options api.DownOptions, project *types.Project) map[string]struct{} {
-	images := map[string]struct{}{}
-	for _, service := range project.Services {
-		image, ok := service.Labels[api.ImageNameLabel] // Information on the compose file at the creation of the container
-		if !ok || (options.Images == "local" && image != "") {
-			continue
-		}
-		image = api.GetImageNameOrDefault(service, project.Name)
-		images[image] = struct{}{}
-	}
-	return images
-}
-
 func (s *composeService) removeImage(ctx context.Context, image string, w progress.Writer) error {
 	id := fmt.Sprintf("Image %s", image)
 	w.Event(progress.NewEvent(id, progress.Working, "Removing"))
diff --git a/pkg/compose/down_test.go b/pkg/compose/down_test.go
index e0d88d59a..22b218e1d 100644
--- a/pkg/compose/down_test.go
+++ b/pkg/compose/down_test.go
@@ -18,12 +18,15 @@ package compose
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 
+	"github.com/compose-spec/compose-go/types"
 	moby "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/errdefs"
 	"github.com/golang/mock/gomock"
 	"gotest.tools/v3/assert"
 
@@ -149,10 +152,24 @@ func TestDownRemoveVolumes(t *testing.T) {
 	assert.NilError(t, err)
 }
 
-func TestDownRemoveImageLocal(t *testing.T) {
+func TestDownRemoveImages(t *testing.T) {
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 
+	opts := compose.DownOptions{
+		Project: &types.Project{
+			Name: strings.ToLower(testProject),
+			Services: types.Services{
+				{Name: "local-anonymous"},
+				{Name: "local-named", Image: "local-named-image"},
+				{Name: "remote", Image: "remote-image"},
+				{Name: "remote-tagged", Image: "registry.example.com/remote-image-tagged:v1.0"},
+				{Name: "no-images-anonymous"},
+				{Name: "no-images-named", Image: "missing-named-image"},
+			},
+		},
+	}
+
 	api := mocks.NewMockAPIClient(mockCtrl)
 	cli := mocks.NewMockCli(mockCtrl)
 	tested := composeService{
@@ -160,29 +177,87 @@ func TestDownRemoveImageLocal(t *testing.T) {
 	}
 	cli.EXPECT().Client().Return(api).AnyTimes()
 
-	container := testContainer("service1", "123", false)
-	container.Labels[compose.ImageNameLabel] = ""
+	api.EXPECT().ContainerList(gomock.Any(), projectFilterListOpt(false)).
+		Return([]moby.Container{
+			testContainer("service1", "123", false),
+		}, nil).
+		AnyTimes()
 
-	api.EXPECT().ContainerList(gomock.Any(), projectFilterListOpt(false)).Return(
-		[]moby.Container{container}, nil)
+	api.EXPECT().ImageList(gomock.Any(), moby.ImageListOptions{
+		Filters: filters.NewArgs(
+			projectFilter(strings.ToLower(testProject)),
+			filters.Arg("dangling", "false"),
+		),
+	}).Return([]moby.ImageSummary{
+		{
+			Labels:   types.Labels{compose.ServiceLabel: "local-anonymous"},
+			RepoTags: []string{"testproject-local-anonymous:latest"},
+		},
+		{
+			Labels:   types.Labels{compose.ServiceLabel: "local-named"},
+			RepoTags: []string{"local-named-image:latest"},
+		},
+	}, nil).AnyTimes()
 
-	api.EXPECT().VolumeList(gomock.Any(), filters.NewArgs(projectFilter(strings.ToLower(testProject)))).
-		Return(volume.VolumeListOKBody{
-			Volumes: []*moby.Volume{{Name: "myProject_volume"}},
-		}, nil)
-	api.EXPECT().NetworkList(gomock.Any(), moby.NetworkListOptions{Filters: filters.NewArgs(projectFilter(strings.ToLower(testProject)))}).
-		Return(nil, nil)
+	imagesToBeInspected := map[string]bool{
+		"testproject-local-anonymous":     true,
+		"local-named-image":               true,
+		"remote-image":                    true,
+		"testproject-no-images-anonymous": false,
+		"missing-named-image":             false,
+	}
+	for img, exists := range imagesToBeInspected {
+		var resp moby.ImageInspect
+		var err error
+		if exists {
+			resp.RepoTags = []string{img}
+		} else {
+			err = errdefs.NotFound(fmt.Errorf("test specified that image %q should not exist", img))
+		}
 
-	api.EXPECT().ContainerStop(gomock.Any(), "123", nil).Return(nil)
-	api.EXPECT().ContainerRemove(gomock.Any(), "123", moby.ContainerRemoveOptions{Force: true}).Return(nil)
+		api.EXPECT().ImageInspectWithRaw(gomock.Any(), img).
+			Return(resp, nil, err).
+			AnyTimes()
+	}
 
-	api.EXPECT().ImageRemove(gomock.Any(), "testproject-service1", moby.ImageRemoveOptions{}).Return(nil, nil)
+	api.EXPECT().ImageInspectWithRaw(gomock.Any(), "registry.example.com/remote-image-tagged:v1.0").
+		Return(moby.ImageInspect{RepoTags: []string{"registry.example.com/remote-image-tagged:v1.0"}}, nil, nil).
+		AnyTimes()
 
-	err := tested.Down(context.Background(), strings.ToLower(testProject), compose.DownOptions{Images: "local"})
+	localImagesToBeRemoved := []string{
+		"testproject-local-anonymous:latest",
+	}
+	for _, img := range localImagesToBeRemoved {
+		// test calls down --rmi=local then down --rmi=all, so local images
+		// get "removed" 2x, while other images are only 1x
+		api.EXPECT().ImageRemove(gomock.Any(), img, moby.ImageRemoveOptions{}).
+			Return(nil, nil).
+			Times(2)
+	}
+
+	t.Log("-> docker compose down --rmi=local")
+	opts.Images = "local"
+	err := tested.Down(context.Background(), strings.ToLower(testProject), opts)
+	assert.NilError(t, err)
+
+	otherImagesToBeRemoved := []string{
+		"local-named-image:latest",
+		"remote-image:latest",
+		"registry.example.com/remote-image-tagged:v1.0",
+	}
+	for _, img := range otherImagesToBeRemoved {
+		api.EXPECT().ImageRemove(gomock.Any(), img, moby.ImageRemoveOptions{}).
+			Return(nil, nil).
+			Times(1)
+	}
+
+	t.Log("-> docker compose down --rmi=all")
+	opts.Images = "all"
+	err = tested.Down(context.Background(), strings.ToLower(testProject), opts)
 	assert.NilError(t, err)
 }
 
-func TestDownRemoveImageLocalNoLabel(t *testing.T) {
+func TestDownRemoveImages_NoLabel(t *testing.T) {
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 
@@ -205,39 +280,23 @@ func TestDownRemoveImageLocalNoLabel(t *testing.T) {
 	api.EXPECT().NetworkList(gomock.Any(), moby.NetworkListOptions{Filters: filters.NewArgs(projectFilter(strings.ToLower(testProject)))}).
 		Return(nil, nil)
 
+	// ImageList returns no images for the project since they were unlabeled
+	// (created by an older version of Compose)
+	api.EXPECT().ImageList(gomock.Any(), moby.ImageListOptions{
+		Filters: filters.NewArgs(
+			projectFilter(strings.ToLower(testProject)),
+			filters.Arg("dangling", "false"),
+		),
+	}).Return(nil, nil)
+
+	api.EXPECT().ImageInspectWithRaw(gomock.Any(), "testproject-service1").
+		Return(moby.ImageInspect{}, nil, nil)
+
 	api.EXPECT().ContainerStop(gomock.Any(), "123", nil).Return(nil)
 	api.EXPECT().ContainerRemove(gomock.Any(), "123", moby.ContainerRemoveOptions{Force: true}).Return(nil)
 
+	api.EXPECT().ImageRemove(gomock.Any(), "testproject-service1:latest", moby.ImageRemoveOptions{}).Return(nil, nil)
+
 	err := tested.Down(context.Background(), strings.ToLower(testProject), compose.DownOptions{Images: "local"})
 	assert.NilError(t, err)
 }
-
-func TestDownRemoveImageAll(t *testing.T) {
-	mockCtrl := gomock.NewController(t)
-	defer mockCtrl.Finish()
-
-	api := mocks.NewMockAPIClient(mockCtrl)
-	cli := mocks.NewMockCli(mockCtrl)
-	tested := composeService{
-		dockerCli: cli,
-	}
-	cli.EXPECT().Client().Return(api).AnyTimes()
-
-	api.EXPECT().ContainerList(gomock.Any(), projectFilterListOpt(false)).Return(
-		[]moby.Container{testContainer("service1", "123", false)}, nil)
-
-	api.EXPECT().VolumeList(gomock.Any(), filters.NewArgs(projectFilter(strings.ToLower(testProject)))).
-		Return(volume.VolumeListOKBody{
-			Volumes: []*moby.Volume{{Name: "myProject_volume"}},
-		}, nil)
-	api.EXPECT().NetworkList(gomock.Any(), moby.NetworkListOptions{Filters: filters.NewArgs(projectFilter(strings.ToLower(testProject)))}).
-		Return(nil, nil)
-
-	api.EXPECT().ContainerStop(gomock.Any(), "123", nil).Return(nil)
-	api.EXPECT().ContainerRemove(gomock.Any(), "123", moby.ContainerRemoveOptions{Force: true}).Return(nil)
-
-	api.EXPECT().ImageRemove(gomock.Any(), "service1-img", moby.ImageRemoveOptions{}).Return(nil, nil)
-
-	err := tested.Down(context.Background(), strings.ToLower(testProject), compose.DownOptions{Images: "all"})
-	assert.NilError(t, err)
-}
diff --git a/pkg/compose/image_pruner.go b/pkg/compose/image_pruner.go
new file mode 100644
index 000000000..e828b46a0
--- /dev/null
+++ b/pkg/compose/image_pruner.go
@@ -0,0 +1,254 @@
+/*
+   Copyright 2022 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package compose
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/compose-spec/compose-go/types"
+	"github.com/distribution/distribution/v3/reference"
+	moby "github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/docker/docker/errdefs"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/docker/compose/v2/pkg/api"
+)
+
+// ImagePruneMode controls how aggressively images associated with the project
+// are removed from the engine.
+type ImagePruneMode string
+
+const (
+	// ImagePruneNone indicates that no project images should be removed.
+	ImagePruneNone ImagePruneMode = ""
+	// ImagePruneLocal indicates that only images built locally by Compose
+	// should be removed.
+	ImagePruneLocal ImagePruneMode = "local"
+	// ImagePruneAll indicates that all project-associated images, including
+	// remote images should be removed.
+	ImagePruneAll ImagePruneMode = "all"
+)
+
+// ImagePruneOptions controls the behavior of image pruning.
+type ImagePruneOptions struct {
+	Mode ImagePruneMode
+
+	// RemoveOrphans will result in the removal of images that were built for
+	// the project regardless of whether they are for a known service if true.
+	RemoveOrphans bool
+}
+
+// ImagePruner handles image removal during Compose `down` operations.
+type ImagePruner struct {
+	client  client.ImageAPIClient
+	project *types.Project
+}
+
+// NewImagePruner creates an ImagePruner object for a project.
+func NewImagePruner(imageClient client.ImageAPIClient, project *types.Project) *ImagePruner {
+	return &ImagePruner{
+		client:  imageClient,
+		project: project,
+	}
+}
+
+// ImagesToPrune returns the set of images that should be removed.
+func (p *ImagePruner) ImagesToPrune(ctx context.Context, opts ImagePruneOptions) ([]string, error) {
+	if opts.Mode == ImagePruneNone {
+		return nil, nil
+	} else if opts.Mode != ImagePruneLocal && opts.Mode != ImagePruneAll {
+		return nil, fmt.Errorf("unsupported image prune mode: %s", opts.Mode)
+	}
+	var images []string
+
+	if opts.Mode == ImagePruneAll {
+		namedImages, err := p.namedImages(ctx)
+		if err != nil {
+			return nil, err
+		}
+		images = append(images, namedImages...)
+	}
+
+	projectImages, err := p.labeledLocalImages(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, img := range projectImages {
+		if len(img.RepoTags) == 0 {
+			// currently, we're only pruning the tagged references, but
+			// if we start removing the dangling images and grouping by
+			// service, we can remove this (and should rely on `Image::ID`)
+			continue
+		}
+
+		var shouldPrune bool
+		if opts.RemoveOrphans {
+			// indiscriminately prune all project images even if they're not
+			// referenced by the current Compose state (e.g. the service was
+			// removed from YAML)
+			shouldPrune = true
+		} else {
+			// only prune the image if it belongs to a known service for the
+			// project AND is either an implicitly-named, locally-built image
+			// or `--rmi=all` has been specified.
+			// TODO(milas): now that Compose labels the images it builds, this
+			// makes less sense; arguably, locally-built but explicitly-named
+			// images should be removed with `--rmi=local` as well.
+			service, err := p.project.GetService(img.Labels[api.ServiceLabel])
+			if err == nil && (opts.Mode == ImagePruneAll || service.Image == "") {
+				shouldPrune = true
+			}
+		}
+
+		if shouldPrune {
+			images = append(images, img.RepoTags[0])
+		}
+	}
+
+	fallbackImages, err := p.unlabeledLocalImages(ctx)
+	if err != nil {
+		return nil, err
+	}
+	images = append(images, fallbackImages...)
+
+	images = normalizeAndDedupeImages(images)
+	return images, nil
+}
+
+// namedImages are those that are explicitly named in the service config.
+//
+// These could be registry-only images (no local build), hybrid (support build
+// as a fallback if cannot pull), or local-only (image does not exist in a
+// registry).
+func (p *ImagePruner) namedImages(ctx context.Context) ([]string, error) {
+	var images []string
+	for _, service := range p.project.Services {
+		if service.Image == "" {
+			continue
+		}
+		images = append(images, service.Image)
+	}
+	return p.filterImagesByExistence(ctx, images)
+}
+
+// labeledLocalImages are images that were locally-built by a current version of
+// Compose (it did not always label built images).
+//
+// The image name could either have been defined by the user or implicitly
+// created from the project + service name.
+func (p *ImagePruner) labeledLocalImages(ctx context.Context) ([]moby.ImageSummary, error) {
+	imageListOpts := moby.ImageListOptions{
+		Filters: filters.NewArgs(
+			projectFilter(p.project.Name),
+			// TODO(milas): we should really clean up the dangling images as
+			// well (historically we have NOT); need to refactor this to handle
+			// it gracefully without producing confusing CLI output, i.e. we
+			// do not want to print out a bunch of untagged/dangling image IDs,
+			// they should be grouped into a logical operation for the relevant
+			// service
+			filters.Arg("dangling", "false"),
+		),
+	}
+	projectImages, err := p.client.ImageList(ctx, imageListOpts)
+	if err != nil {
+		return nil, err
+	}
+	return projectImages, nil
+}
+
+// unlabeledLocalImages are images that match the implicit naming convention
+// for locally-built images but did not get labeled, presumably because they
+// were produced by an older version of Compose.
+//
+// This is transitional to ensure `down` continues to work as expected on
+// projects built/launched by previous versions of Compose. It can safely
+// be removed after some time.
+func (p *ImagePruner) unlabeledLocalImages(ctx context.Context) ([]string, error) {
+	var images []string
+	for _, service := range p.project.Services {
+		if service.Image != "" {
+			continue
+		}
+		img := api.GetImageNameOrDefault(service, p.project.Name)
+		images = append(images, img)
+	}
+	return p.filterImagesByExistence(ctx, images)
+}
+
+// filterImagesByExistence returns the subset of images that exist in the
+// engine store.
+//
+// NOTE: Any transient errors communicating with the API will result in an
+// image being returned as "existing", as this method is exclusively used to
+// find images to remove, so the worst case of being conservative here is an
+// attempt to remove an image that doesn't exist, which will cause a warning
+// but is otherwise harmless.
+func (p *ImagePruner) filterImagesByExistence(ctx context.Context, imageNames []string) ([]string, error) {
+	var mu sync.Mutex
+	var ret []string
+
+	eg, ctx := errgroup.WithContext(ctx)
+	for _, img := range imageNames {
+		img := img
+		eg.Go(func() error {
+			_, _, err := p.client.ImageInspectWithRaw(ctx, img)
+			if errdefs.IsNotFound(err) {
+				// err on the side of caution: only skip if we successfully
+				// queried the API and got back a definitive "not exists"
+				return nil
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			ret = append(ret, img)
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// normalizeAndDedupeImages returns the unique set of images after normalization.
+func normalizeAndDedupeImages(images []string) []string {
+	seen := make(map[string]struct{}, len(images))
+	for _, img := range images {
+		// since some references come from user input (service.image) and some
+		// come from the engine API, we standardize them, opting for the
+		// familiar name format since they'll also be displayed in the CLI
+		ref, err := reference.ParseNormalizedNamed(img)
+		if err == nil {
+			ref = reference.TagNameOnly(ref)
+			img = reference.FamiliarString(ref)
+		}
+		seen[img] = struct{}{}
+	}
+	ret := make([]string, 0, len(seen))
+	for v := range seen {
+		ret = append(ret, v)
+	}
+	// ensure a deterministic return result - the actual ordering is not useful
+	sort.Strings(ret)
+	return ret
+}
diff --git a/pkg/compose/kill_test.go b/pkg/compose/kill_test.go
index a50308a4d..b91a34a23 100644
--- a/pkg/compose/kill_test.go
+++ b/pkg/compose/kill_test.go
@@ -115,7 +115,6 @@ func containerLabels(service string, oneOff bool) map[string]string {
 	composefile := filepath.Join(workingdir, "compose.yaml")
 	labels := map[string]string{
 		compose.ServiceLabel:     service,
-		compose.ImageNameLabel:   service + "-img",
 		compose.ConfigFilesLabel: composefile,
 		compose.WorkingDirLabel:  workingdir,
 		compose.ProjectLabel:     strings.ToLower(testProject)}
diff --git a/pkg/e2e/build_test.go b/pkg/e2e/build_test.go
index 7eeebf948..0620f6e58 100644
--- a/pkg/e2e/build_test.go
+++ b/pkg/e2e/build_test.go
@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/icmd"
 )
@@ -211,6 +212,10 @@ func TestBuildImageDependencies(t *testing.T) {
 	doTest := func(t *testing.T, cli *CLI) {
 		resetState := func() {
 			cli.RunDockerComposeCmd(t, "down", "--rmi=all", "-t=0")
+			res := cli.RunDockerOrExitError(t, "image", "rm", "build-dependencies-service")
+			if res.Error != nil {
+				require.Contains(t, res.Stderr(), `Error: No such image: build-dependencies-service`)
+			}
 		}
 		resetState()
 		t.Cleanup(resetState)
@@ -229,6 +234,15 @@ func TestBuildImageDependencies(t *testing.T) {
 			"image", "inspect", "--format={{ index .RepoTags 0 }}",
 			"build-dependencies-service")
 		res.Assert(t, icmd.Expected{Out: "build-dependencies-service:latest"})
+
+		res = cli.RunDockerComposeCmd(t, "down", "-t0", "--rmi=all", "--remove-orphans")
+		t.Log(res.Combined())
+
+		res = cli.RunDockerOrExitError(t, "image", "inspect", "build-dependencies-service")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "Error: No such image: build-dependencies-service",
+		})
 	}
 
 	t.Run("ClassicBuilder", func(t *testing.T) {