From 31de84f547be058ec9415fd6bd0c208fe2b55992 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 13 Apr 2021 11:31:39 +0200 Subject: [PATCH 1/2] Set image digest as service image to trigger recreation after build Signed-off-by: aiordache --- cli/cmd/compose/images.go | 11 ++++-- local/compose/build.go | 35 +++++++++++++++--- local/compose/images.go | 74 ++++++++++++++++++++------------------- 3 files changed, 78 insertions(+), 42 deletions(-) diff --git a/cli/cmd/compose/images.go b/cli/cmd/compose/images.go index edcf8b07e..bb53f8716 100644 --- a/cli/cmd/compose/images.go +++ b/cli/cmd/compose/images.go @@ -99,8 +99,15 @@ func runImages(ctx context.Context, opts imageOptions, services []string) error for _, img := range images { id := stringid.TruncateID(img.ID) size := units.HumanSizeWithPrecision(float64(img.Size), 3) - - _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", img.ContainerName, img.Repository, img.Tag, id, size) + repo := img.Repository + if repo == "" { + repo = "" + } + tag := img.Tag + if tag == "" { + tag = "" + } + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", img.ContainerName, repo, tag, id, size) } }, "Container", "Repository", "Tag", "Image Id", "Size") diff --git a/local/compose/build.go b/local/compose/build.go index 35a113f48..545f4697d 100644 --- a/local/compose/build.go +++ b/local/compose/build.go @@ -134,12 +134,39 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. } err := s.build(ctx, project, opts, observedState, mode) - if err == nil { - if len(imagesToBuild) > 0 { - utils.DisplayScanSuggestMsg() + if err != nil { + return err + } + if len(imagesToBuild) > 0 { + utils.DisplayScanSuggestMsg() + } + + return s.updateServiceImages(ctx, project) +} + +func (s *composeService) updateServiceImages(ctx context.Context, project *types.Project) error { + imageNames := []string{} + for _, s := range project.Services { + imgName := s.Image + if imgName == "" { + imgName = getImageName(s, project.Name) + } + if !utils.StringContains(imageNames, imgName) { + imageNames = append(imageNames, imgName) } } - return err + images, err := s.getImages(ctx, imageNames) + if err != nil { + return err + } + for i, s := range project.Services { + img, ok := images[getImageName(s, project.Name)] + if !ok { + return fmt.Errorf("failed to retrieve image for service %s", s.Name) + } + project.Services[i].Image = img.ID + } + return nil } func (s *composeService) localImagePresent(ctx context.Context, imageName string) (bool, error) { diff --git a/local/compose/images.go b/local/compose/images.go index c6f33a9d9..aff1252b6 100644 --- a/local/compose/images.go +++ b/local/compose/images.go @@ -57,25 +57,7 @@ func (s *composeService) Images(ctx context.Context, projectName string, options imageIDs = append(imageIDs, c.ImageID) } } - - images := map[string]moby.ImageInspect{} - l := sync.Mutex{} - eg, ctx := errgroup.WithContext(ctx) - for _, img := range imageIDs { - img := img - eg.Go(func() error { - inspect, _, err := s.apiClient.ImageInspectWithRaw(ctx, img) - if err != nil { - return err - } - l.Lock() - images[img] = inspect - l.Unlock() - return nil - }) - } - err = eg.Wait() - + images, err := s.getImages(ctx, imageIDs) if err != nil { return nil, err } @@ -85,24 +67,44 @@ func (s *composeService) Images(ctx context.Context, projectName string, options if !ok { return nil, fmt.Errorf("failed to retrieve image for container %s", getCanonicalContainerName(container)) } - if len(img.RepoTags) == 0 { - return nil, fmt.Errorf("no image tag found for %s", img.ID) - } - tag := "" - repository := "" - repotag := strings.Split(img.RepoTags[0], ":") - repository = repotag[0] - if len(repotag) > 1 { - tag = repotag[1] - } - summary[i] = compose.ImageSummary{ - ID: img.ID, - ContainerName: getCanonicalContainerName(container), - Repository: repository, - Tag: tag, - Size: img.Size, - } + summary[i] = img + summary[i].ContainerName = getCanonicalContainerName(container) } return summary, nil } + +func (s *composeService) getImages(ctx context.Context, images []string) (map[string]compose.ImageSummary, error) { + summary := map[string]compose.ImageSummary{} + l := sync.Mutex{} + eg, ctx := errgroup.WithContext(ctx) + for _, img := range images { + img := img + eg.Go(func() error { + inspect, _, err := s.apiClient.ImageInspectWithRaw(ctx, img) + if err != nil { + return err + } + tag := "" + repository := "" + if len(inspect.RepoTags) > 0 { + + repotag := strings.Split(inspect.RepoTags[0], ":") + repository = repotag[0] + if len(repotag) > 1 { + tag = repotag[1] + } + } + l.Lock() + summary[img] = compose.ImageSummary{ + ID: inspect.ID, + Repository: repository, + Tag: tag, + Size: inspect.Size, + } + l.Unlock() + return nil + }) + } + return summary, eg.Wait() +} From c227fd5c5f2baf0923fe897c8d0178f62bd042f6 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 19 Apr 2021 11:04:29 +0200 Subject: [PATCH 2/2] refactor to reduce calls to InspectRawImage Signed-off-by: aiordache --- local/compose/build.go | 132 ++++++++++++++++++++++------------------ local/compose/images.go | 4 ++ 2 files changed, 76 insertions(+), 60 deletions(-) diff --git a/local/compose/build.go b/local/compose/build.go index 545f4697d..168bafa20 100644 --- a/local/compose/build.go +++ b/local/compose/build.go @@ -32,7 +32,6 @@ import ( "github.com/docker/buildx/util/progress" cliconfig "github.com/docker/cli/cli/config" moby "github.com/docker/docker/api/types" - "github.com/docker/docker/errdefs" bclient "github.com/moby/buildkit/client" specs "github.com/opencontainers/image-spec/specs-go/v1" @@ -73,7 +72,7 @@ func (s *composeService) Build(ctx context.Context, project *types.Project, opti } } - err := s.build(ctx, project, opts, Containers{}, options.Progress) + _, err := s.build(ctx, project, opts, Containers{}, options.Progress) if err == nil { if len(imagesToBuild) > 0 && !options.Quiet { utils.DisplayScanSuggestMsg() @@ -84,28 +83,60 @@ func (s *composeService) Build(ctx context.Context, project *types.Project, opti } func (s *composeService) ensureImagesExists(ctx context.Context, project *types.Project, observedState Containers, quietPull bool) error { + images, err := s.getImageDigests(ctx, project) + if err != nil { + return err + } + + mode := progress.PrinterModeAuto + if quietPull { + mode = progress.PrinterModeQuiet + } + opts, imagesToBuild, err := s.getBuildOptions(project, images) + if err != nil { + return err + } + builtImages, err := s.build(ctx, project, opts, observedState, mode) + if err != nil { + return err + } + + if len(imagesToBuild) > 0 { + utils.DisplayScanSuggestMsg() + } + for name, digest := range builtImages { + images[name] = digest + } + // set digest as service.Image + for i, service := range project.Services { + digest, ok := images[getImageName(service, project.Name)] + if ok { + project.Services[i].Image = digest + } + } + return nil +} + +func (s *composeService) getBuildOptions(project *types.Project, images map[string]string) (map[string]build.Options, []string, error) { opts := map[string]build.Options{} imagesToBuild := []string{} for _, service := range project.Services { if service.Image == "" && service.Build == nil { - return fmt.Errorf("invalid service %q. Must specify either image or build", service.Name) + return nil, nil, fmt.Errorf("invalid service %q. Must specify either image or build", service.Name) } - imageName := getImageName(service, project.Name) - localImagePresent, err := s.localImagePresent(ctx, imageName) - if err != nil { - return err - } + _, localImagePresent := images[imageName] if service.Build != nil { if localImagePresent && service.PullPolicy != types.PullPolicyBuild { continue } imagesToBuild = append(imagesToBuild, imageName) - opts[imageName], err = s.toBuildOptions(service, project.WorkingDir, imageName) + opt, err := s.toBuildOptions(service, project.WorkingDir, imageName) if err != nil { - return err + return nil, nil, err } + opts[imageName] = opt continue } if service.Image != "" { @@ -113,7 +144,6 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. continue } } - // Buildx has no command to "just pull", see // so we bake a temporary dockerfile that will just pull and export pulled image opts[service.Name] = build.Options{ @@ -127,84 +157,54 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. } } + return opts, imagesToBuild, nil - mode := progress.PrinterModeAuto - if quietPull { - mode = progress.PrinterModeQuiet - } - - err := s.build(ctx, project, opts, observedState, mode) - if err != nil { - return err - } - if len(imagesToBuild) > 0 { - utils.DisplayScanSuggestMsg() - } - - return s.updateServiceImages(ctx, project) } -func (s *composeService) updateServiceImages(ctx context.Context, project *types.Project) error { +func (s *composeService) getImageDigests(ctx context.Context, project *types.Project) (map[string]string, error) { imageNames := []string{} for _, s := range project.Services { - imgName := s.Image - if imgName == "" { - imgName = getImageName(s, project.Name) - } + imgName := getImageName(s, project.Name) if !utils.StringContains(imageNames, imgName) { imageNames = append(imageNames, imgName) } } - images, err := s.getImages(ctx, imageNames) + imgs, err := s.getImages(ctx, imageNames) if err != nil { - return err + return nil, err } - for i, s := range project.Services { - img, ok := images[getImageName(s, project.Name)] - if !ok { - return fmt.Errorf("failed to retrieve image for service %s", s.Name) - } - project.Services[i].Image = img.ID + images := map[string]string{} + for name, info := range imgs { + images[name] = info.ID } - return nil + return images, nil } -func (s *composeService) localImagePresent(ctx context.Context, imageName string) (bool, error) { - _, _, err := s.apiClient.ImageInspectWithRaw(ctx, imageName) - if err != nil { - if errdefs.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - -func (s *composeService) build(ctx context.Context, project *types.Project, opts map[string]build.Options, observedState Containers, mode string) error { +func (s *composeService) build(ctx context.Context, project *types.Project, opts map[string]build.Options, observedState Containers, mode string) (map[string]string, error) { info, err := s.apiClient.Info(ctx) if err != nil { - return err + return nil, err } if info.OSType == "windows" { // no support yet for Windows container builds in Buildkit // https://docs.docker.com/develop/develop-images/build_enhancements/#limitations err := s.windowsBuild(opts, mode) - return metrics.WrapCategorisedComposeError(err, metrics.BuildFailure) + return nil, metrics.WrapCategorisedComposeError(err, metrics.BuildFailure) } if len(opts) == 0 { - return nil + return nil, nil } const drivername = "default" configFile, err := cliconfig.Load(config.Dir()) if err != nil { - return err + return nil, err } d, err := driver.GetDriver(ctx, drivername, nil, s.apiClient, configFile, nil, nil, "", nil, nil, project.WorkingDir) if err != nil { - return err + return nil, err } driverInfo := []build.DriverInfo{ { @@ -221,13 +221,13 @@ func (s *composeService) build(ctx context.Context, project *types.Project, opts w := progress.NewPrinter(progressCtx, os.Stdout, mode) // We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here - _, err = build.Build(ctx, driverInfo, opts, nil, nil, w) + response, err := build.Build(ctx, driverInfo, opts, nil, nil, w) errW := w.Wait() if err == nil { err = errW } if err != nil { - return metrics.WrapCategorisedComposeError(err, metrics.BuildFailure) + return nil, metrics.WrapCategorisedComposeError(err, metrics.BuildFailure) } cw := composeprogress.ContextWriter(ctx) @@ -236,13 +236,25 @@ func (s *composeService) build(ctx context.Context, project *types.Project, opts if c.Image == imageName { err = s.removeContainers(ctx, cw, []moby.Container{c}, nil) if err != nil { - return err + return nil, err } } } } - return err + imagesBuilt := map[string]string{} + for name, img := range response { + if img == nil || len(img.ExporterResponse) == 0 { + continue + } + digest, ok := img.ExporterResponse["containerimage.digest"] + if !ok { + continue + } + imagesBuilt[name] = digest + } + + return imagesBuilt, err } func (s *composeService) toBuildOptions(service types.ServiceConfig, contextPath string, imageTag string) (build.Options, error) { diff --git a/local/compose/images.go b/local/compose/images.go index aff1252b6..dd27cafcb 100644 --- a/local/compose/images.go +++ b/local/compose/images.go @@ -24,6 +24,7 @@ import ( moby "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/errdefs" "golang.org/x/sync/errgroup" "github.com/docker/compose-cli/api/compose" @@ -83,6 +84,9 @@ func (s *composeService) getImages(ctx context.Context, images []string) (map[st eg.Go(func() error { inspect, _, err := s.apiClient.ImageInspectWithRaw(ctx, img) if err != nil { + if errdefs.IsNotFound(err) { + return nil + } return err } tag := ""