support refresh pull policy

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
This commit is contained in:
Nicolas De Loof 2025-02-21 11:05:12 +01:00 committed by Nicolas De loof
parent e38b729a30
commit 6c1ee1069b
6 changed files with 68 additions and 41 deletions

3
go.mod
View File

@ -7,7 +7,7 @@ require (
github.com/Microsoft/go-winio v0.6.2 github.com/Microsoft/go-winio v0.6.2
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/buger/goterm v1.0.4 github.com/buger/goterm v1.0.4
github.com/compose-spec/compose-go/v2 v2.4.8 github.com/compose-spec/compose-go/v2 v2.4.9-0.20250225151507-331db8fefcb7
github.com/containerd/containerd/v2 v2.0.2 github.com/containerd/containerd/v2 v2.0.2
github.com/containerd/platforms v1.0.0-rc.1 github.com/containerd/platforms v1.0.0-rc.1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
@ -168,6 +168,7 @@ require (
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
github.com/zclconf/go-cty v1.16.0 // indirect github.com/zclconf/go-cty v1.16.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect

6
go.sum
View File

@ -81,8 +81,8 @@ github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e
github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
github.com/compose-spec/compose-go/v2 v2.4.8 h1:7Myl8wDRl/4mRz77S+eyDJymGGEHu0diQdGSSeyq90A= github.com/compose-spec/compose-go/v2 v2.4.9-0.20250225151507-331db8fefcb7 h1:7NlxAsQcWvLpFlEHsBo80sJ1UMMs84kkf0yXGs6de2k=
github.com/compose-spec/compose-go/v2 v2.4.8/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/compose-spec/compose-go/v2 v2.4.9-0.20250225151507-331db8fefcb7/go.mod h1:6k5l/0TxCg0/2uLEhRVEsoBWBprS2uvZi32J7xub3lo=
github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo=
github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins=
github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=
@ -494,6 +494,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

View File

@ -532,6 +532,7 @@ type ImageSummary struct {
Repository string Repository string
Tag string Tag string
Size int64 Size int64
LastTagTime time.Time
} }
// ServiceStatus hold status about a service // ServiceStatus hold status about a service

View File

@ -23,6 +23,7 @@ import (
"os" "os"
"strings" "strings"
"sync" "sync"
"time"
"github.com/compose-spec/compose-go/v2/types" "github.com/compose-spec/compose-go/v2/types"
"github.com/containerd/platforms" "github.com/containerd/platforms"
@ -70,7 +71,7 @@ const bakeSuggest = "Compose now can delegate build to bake for better performan
var suggest sync.Once var suggest sync.Once
//nolint:gocyclo //nolint:gocyclo
func (s *composeService) build(ctx context.Context, project *types.Project, options api.BuildOptions, localImages map[string]string) (map[string]string, error) { func (s *composeService) build(ctx context.Context, project *types.Project, options api.BuildOptions, localImages map[string]api.ImageSummary) (map[string]string, error) {
imageIDs := map[string]string{} imageIDs := map[string]string{}
serviceToBuild := types.Services{} serviceToBuild := types.Services{}
@ -282,7 +283,11 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types.
} }
for name, digest := range builtImages { for name, digest := range builtImages {
images[name] = digest images[name] = api.ImageSummary{
Repository: name,
ID: digest,
LastTagTime: time.Now(),
}
} }
return nil return nil
}, },
@ -295,19 +300,16 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types.
// set digest as com.docker.compose.image label so we can detect outdated containers // set digest as com.docker.compose.image label so we can detect outdated containers
for name, service := range project.Services { for name, service := range project.Services {
image := api.GetImageNameOrDefault(service, project.Name) image := api.GetImageNameOrDefault(service, project.Name)
digest, ok := images[image] img, ok := images[image]
if ok { if ok {
if service.Labels == nil { service.CustomLabels.Add(api.ImageDigestLabel, img.ID)
service.Labels = types.Labels{}
}
service.CustomLabels.Add(api.ImageDigestLabel, digest)
} }
project.Services[name] = service project.Services[name] = service
} }
return nil return nil
} }
func (s *composeService) getLocalImagesDigests(ctx context.Context, project *types.Project) (map[string]string, error) { func (s *composeService) getLocalImagesDigests(ctx context.Context, project *types.Project) (map[string]api.ImageSummary, error) {
var imageNames []string var imageNames []string
for _, s := range project.Services { for _, s := range project.Services {
imgName := api.GetImageNameOrDefault(s, project.Name) imgName := api.GetImageNameOrDefault(s, project.Name)
@ -319,14 +321,10 @@ func (s *composeService) getLocalImagesDigests(ctx context.Context, project *typ
if err != nil { if err != nil {
return nil, err return nil, err
} }
images := map[string]string{}
for name, info := range imgs {
images[name] = info.ID
}
for i, service := range project.Services { for i, service := range project.Services {
imgName := api.GetImageNameOrDefault(service, project.Name) imgName := api.GetImageNameOrDefault(service, project.Name)
digest, ok := images[imgName] img, ok := imgs[imgName]
if !ok { if !ok {
continue continue
} }
@ -335,7 +333,7 @@ func (s *composeService) getLocalImagesDigests(ctx context.Context, project *typ
if err != nil { if err != nil {
return nil, err return nil, err
} }
inspect, err := s.apiClient().ImageInspect(ctx, digest) inspect, err := s.apiClient().ImageInspect(ctx, img.ID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -348,15 +346,15 @@ func (s *composeService) getLocalImagesDigests(ctx context.Context, project *typ
// there is a local image, but it's for the wrong platform, so // there is a local image, but it's for the wrong platform, so
// pretend it doesn't exist so that we can pull/build an image // pretend it doesn't exist so that we can pull/build an image
// for the correct platform instead // for the correct platform instead
delete(images, imgName) delete(imgs, imgName)
} }
} }
project.Services[i].CustomLabels.Add(api.ImageDigestLabel, digest) project.Services[i].CustomLabels.Add(api.ImageDigestLabel, img.ID)
} }
return images, nil return imgs, nil
} }
// resolveAndMergeBuildArgs returns the final set of build arguments to use for the service image build. // resolveAndMergeBuildArgs returns the final set of build arguments to use for the service image build.

View File

@ -101,10 +101,11 @@ func (s *composeService) getImageSummaries(ctx context.Context, repoTags []strin
} }
l.Lock() l.Lock()
summary[repoTag] = api.ImageSummary{ summary[repoTag] = api.ImageSummary{
ID: inspect.ID, ID: inspect.ID,
Repository: repository, Repository: repository,
Tag: tag, Tag: tag,
Size: inspect.Size, Size: inspect.Size,
LastTagTime: inspect.Metadata.LastTagTime,
} }
l.Unlock() l.Unlock()
return nil return nil

View File

@ -24,6 +24,7 @@ import (
"fmt" "fmt"
"io" "io"
"strings" "strings"
"time"
"github.com/compose-spec/compose-go/v2/types" "github.com/compose-spec/compose-go/v2/types"
"github.com/distribution/reference" "github.com/distribution/reference"
@ -153,7 +154,7 @@ func (s *composeService) pull(ctx context.Context, project *types.Project, opts
return multierror.Append(nil, pullErrors...).ErrorOrNil() return multierror.Append(nil, pullErrors...).ErrorOrNil()
} }
func imageAlreadyPresent(serviceImage string, localImages map[string]string) bool { func imageAlreadyPresent(serviceImage string, localImages map[string]api.ImageSummary) bool {
normalizedImage, err := reference.ParseDockerRef(serviceImage) normalizedImage, err := reference.ParseDockerRef(serviceImage)
if err != nil { if err != nil {
return false return false
@ -288,23 +289,16 @@ func encodedAuth(ref reference.Named, configFile driver.Auth) (string, error) {
return base64.URLEncoding.EncodeToString(buf), nil return base64.URLEncoding.EncodeToString(buf), nil
} }
func (s *composeService) pullRequiredImages(ctx context.Context, project *types.Project, images map[string]string, quietPull bool) error { func (s *composeService) pullRequiredImages(ctx context.Context, project *types.Project, images map[string]api.ImageSummary, quietPull bool) error {
var needPull []types.ServiceConfig var needPull []types.ServiceConfig
for _, service := range project.Services { for _, service := range project.Services {
if service.Image == "" { pull, err := mustPull(service, images)
continue if err != nil {
return err
} }
switch service.PullPolicy { if pull {
case "", types.PullPolicyMissing, types.PullPolicyIfNotPresent: needPull = append(needPull, service)
if _, ok := images[service.Image]; ok {
continue
}
case types.PullPolicyNever, types.PullPolicyBuild:
continue
case types.PullPolicyAlways:
// force pull
} }
needPull = append(needPull, service)
} }
if len(needPull) == 0 { if len(needPull) == 0 {
return nil return nil
@ -314,11 +308,15 @@ func (s *composeService) pullRequiredImages(ctx context.Context, project *types.
w := progress.ContextWriter(ctx) w := progress.ContextWriter(ctx)
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
eg.SetLimit(s.maxConcurrency) eg.SetLimit(s.maxConcurrency)
pulledImages := make([]string, len(needPull)) pulledImages := make([]api.ImageSummary, len(needPull))
for i, service := range needPull { for i, service := range needPull {
eg.Go(func() error { eg.Go(func() error {
id, err := s.pullServiceImage(ctx, service, s.configFile(), w, quietPull, project.Environment["DOCKER_DEFAULT_PLATFORM"]) id, err := s.pullServiceImage(ctx, service, s.configFile(), w, quietPull, project.Environment["DOCKER_DEFAULT_PLATFORM"])
pulledImages[i] = id pulledImages[i] = api.ImageSummary{
ID: id,
Repository: service.Image,
LastTagTime: time.Now(),
}
if err != nil && isServiceImageToBuild(service, project.Services) { if err != nil && isServiceImageToBuild(service, project.Services) {
// image can be built, so we can ignore pull failure // image can be built, so we can ignore pull failure
return nil return nil
@ -328,7 +326,7 @@ func (s *composeService) pullRequiredImages(ctx context.Context, project *types.
} }
err := eg.Wait() err := eg.Wait()
for i, service := range needPull { for i, service := range needPull {
if pulledImages[i] != "" { if pulledImages[i].ID != "" {
images[service.Image] = pulledImages[i] images[service.Image] = pulledImages[i]
} }
} }
@ -336,6 +334,32 @@ func (s *composeService) pullRequiredImages(ctx context.Context, project *types.
}, s.stdinfo()) }, s.stdinfo())
} }
func mustPull(service types.ServiceConfig, images map[string]api.ImageSummary) (bool, error) {
if service.Image == "" {
return false, nil
}
policy, duration, err := service.GetPullPolicy()
if err != nil {
return false, err
}
switch policy {
case types.PullPolicyAlways:
// force pull
return true, nil
case types.PullPolicyNever, types.PullPolicyBuild:
return false, nil
case types.PullPolicyRefresh:
img, ok := images[service.Image]
if !ok {
return true, nil
}
return time.Now().After(img.LastTagTime.Add(duration)), nil
default: // Pull if missing
_, ok := images[service.Image]
return !ok, nil
}
}
func isServiceImageToBuild(service types.ServiceConfig, services types.Services) bool { func isServiceImageToBuild(service types.ServiceConfig, services types.Services) bool {
if service.Build != nil { if service.Build != nil {
return true return true