From 0f4bcb9f17eb871529e7f20f5a3106551215cee7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 25 Nov 2020 16:51:54 +0100 Subject: [PATCH] Build and pull in parallel with BuildKit Signed-off-by: Nicolas De Loof --- go.sum | 37 ++++++++++++---- local/build.go | 64 ++++++++++++++++++++++++++++ local/compose.go | 108 +++++------------------------------------------ 3 files changed, 103 insertions(+), 106 deletions(-) diff --git a/go.sum b/go.sum index 82697e6a6..d5f7a2df0 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AkihiroSuda/containerd-fuse-overlayfs v0.0.0-20200220082720-bb896865146c/go.mod h1:K4kx7xAA5JimeQCnN+dbeLlfaBxzZLaLiDD8lusFI8w= -github.com/AlecAivazis/survey/v2 v2.1.1 h1:LEMbHE0pLj75faaVEKClEX1TM4AJmmnOh9eimREzLWI= -github.com/AlecAivazis/survey/v2 v2.1.1/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk= +github.com/AlecAivazis/survey/v2 v2.2.3 h1:utJR2X4Ibp2fBxdjalQUiMFf3zfQNjA15YE8+ftlKEs= +github.com/AlecAivazis/survey/v2 v2.2.3/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -86,7 +86,10 @@ github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jB github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab h1:9pygWVFqbY9lPxM0peffumuVDyMuIMzNLyO9uFjJuQo= github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= - +github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc= +github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.10 h1:k5wTrpnVU2/xv8ZuzGkbXVd3js5zJ8RnumPo5RxiIxU= github.com/Microsoft/hcsshim v0.8.10/go.mod h1:g5uw8EV2mAlzqe94tfNBNdr89fnbD/n3HV0OhsddkmM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -145,10 +148,21 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= - +github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/compose-spec/compose-go v0.0.0-20201122180821-0f87549cda1e h1:EO6seAKT1UtMjk1RGmuXS1tFkSkWr6FPjQru/mC+r/U= +github.com/compose-spec/compose-go v0.0.0-20201122180821-0f87549cda1e/go.mod h1:Pm8gjIoib7x9av1XGkijXddjyV4aTHEq9EwVOvWH624= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d/go.mod h1:CStdkl05lBnJej94BPFoJ7vB8cELKXwViS+dgfW0/M8= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= - +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v0.0.0-20191219165238-8375c3424e4d/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1 h1:u7SFAJyRqWcG6ogaMAx3KjSTy1e3hT9QxqX7Jco7dRc= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55 h1:FGO0nwSBESgoGCakj+w3OQXyrMLsz2omdo9b2UfG/BQ= +github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20181001140422-bd77b46c8352/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= github.com/containerd/continuity v0.0.0-20200928162600-f2cc35102c2a h1:jEIoR0aA5GogXZ8pP3DUzE+zrhaF6/1rYZy+7KkYEWM= @@ -416,7 +430,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U= github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= - +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -506,7 +521,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= - +github.com/moby/buildkit v0.7.0 h1:5T41l8i0f+OqHvRThh2NKVXiLpclXYmgO8QePoyIJrw= +github.com/moby/buildkit v0.7.0/go.mod h1:zOhLO1TiQepuSfzoNDQ542IGJQy0CHr79T4MOJvaewY= github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf h1:Un6PNx5oMK6CCwO3QTUyPiK2mtZnPrpDl5UnZ64eCkw= github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -527,7 +543,7 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= - +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -537,7 +553,10 @@ github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= - +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= diff --git a/local/build.go b/local/build.go index fef0cf0db..b27fe9c20 100644 --- a/local/build.go +++ b/local/build.go @@ -1,3 +1,5 @@ +// +build local + /* Copyright 2020 Docker Compose CLI authors @@ -18,13 +20,75 @@ package local import ( "context" + "fmt" + "os" "path" + "strings" "github.com/compose-spec/compose-go/types" "github.com/docker/buildx/build" + "github.com/docker/buildx/driver" _ "github.com/docker/buildx/driver/docker" // required to get default driver registered + "github.com/docker/buildx/util/progress" ) +func (s *local) ensureImagesExists(ctx context.Context, project *types.Project) error { + opts := map[string]build.Options{} + for _, service := range project.Services { + if service.Image == "" && service.Build == nil { + return fmt.Errorf("invalid service %q. Must specify either image or build", service.Name) + } + + // TODO build vs pull should be controlled by pull policy, see https://github.com/compose-spec/compose-spec/issues/26 + if service.Image != "" { + needPull, err := s.needPull(ctx, service) + if err != nil { + return err + } + if !needPull { + continue + } + } + if service.Build != nil { + opts[service.Name] = s.buildImage(ctx, service, project.WorkingDir) + continue + } + + // Buildx has no command to "just pull", see + // so we bake a temporary dockerfile that will just pull and export pulled image + opts[service.Name] = build.Options{ + Inputs: build.Inputs{ + ContextPath: ".", + DockerfilePath: "-", + InStream: strings.NewReader("FROM " + service.Image), + }, + Tags: []string{service.Image}, + Pull: true, + } + + } + + return s.build(ctx, project, opts) +} + +func (s *local) build(ctx context.Context, project *types.Project, opts map[string]build.Options) error { + const drivername = "default" + d, err := driver.GetDriver(ctx, drivername, nil, s.containerService.apiClient, nil, nil, "", nil, project.WorkingDir) + if err != nil { + return err + } + driverInfo := []build.DriverInfo{ + { + Name: "default", + Driver: d, + }, + } + // We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here + w := progress.NewPrinter(ctx, os.Stdout, "auto") + _, err = build.Build(ctx, driverInfo, opts, nil, nil, w) + return err +} + func (s *local) buildImage(ctx context.Context, service types.ServiceConfig, contextPath string) build.Options { var tags []string if service.Image != "" { diff --git a/local/compose.go b/local/compose.go index e59296731..d32cc2012 100644 --- a/local/compose.go +++ b/local/compose.go @@ -23,7 +23,6 @@ import ( "encoding/json" "fmt" "io" - "os" "path/filepath" "sort" "strconv" @@ -31,9 +30,6 @@ import ( "sync" "github.com/compose-spec/compose-go/types" - "github.com/docker/buildx/build" - "github.com/docker/buildx/driver" - px "github.com/docker/buildx/util/progress" moby "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" @@ -41,7 +37,6 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/go-connections/nat" "github.com/pkg/errors" "github.com/sanathkr/go-yaml" @@ -76,14 +71,12 @@ func (s *local) Up(ctx context.Context, project *types.Project, detach bool) err } } - for _, service := range project.Services { - err := s.applyPullPolicy(ctx, project, service) - if err != nil { - return err - } + err := s.ensureImagesExists(ctx, project) + if err != nil { + return err } - err := inDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { + err = inDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { return s.ensureService(c, project, service) }) return err @@ -99,94 +92,15 @@ func getContainerName(c moby.Container) string { return c.Names[0][1:] } -func (s *local) applyPullPolicy(ctx context.Context, project *types.Project, service types.ServiceConfig) error { - w := progress.ContextWriter(ctx) - // TODO build vs pull should be controlled by pull policy - opts := map[string]build.Options{} - if service.Build != nil { - opts[service.Name] = s.buildImage(ctx, service, project.WorkingDir) - } - - if len(opts) > 0 { - w := px.NewPrinter(ctx, os.Stdout, "auto") - const drivername = "buildx_buildkit_default" - d, err := driver.GetDriver(ctx, drivername, nil, s.containerService.apiClient, nil, nil, "", nil, project.WorkingDir) - if err != nil { - return err - } - driverInfo := []build.DriverInfo{ - { - Name: "default", - Driver: d, - }, - } - // We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here - // FIXME pass auth file from - _, err = build.Build(ctx, driverInfo, opts, nil, nil, w) - return err - } - - if service.Image != "" { - _, _, err := s.containerService.apiClient.ImageInspectWithRaw(ctx, service.Image) - if err != nil { - if errdefs.IsNotFound(err) { - stream, err := s.containerService.apiClient.ImagePull(ctx, service.Image, moby.ImagePullOptions{}) - if err != nil { - return err - } - dec := json.NewDecoder(stream) - for { - var jm jsonmessage.JSONMessage - if err := dec.Decode(&jm); err != nil { - if err == io.EOF { - break - } - return err - } - toProgressEvent(jm, w) - } - } - } - } - return nil -} - -func toProgressEvent(jm jsonmessage.JSONMessage, w progress.Writer) { - if jm.Progress != nil { - if jm.Progress.Total != 0 { - status := progress.Working - if jm.Status == "Pull complete" { - status = progress.Done - } - w.Event(progress.Event{ - ID: jm.ID, - Text: jm.Status, - Status: status, - StatusText: jm.Progress.String(), - }) - } else { - if jm.Error != nil { - w.Event(progress.Event{ - ID: jm.ID, - Text: jm.Status, - Status: progress.Error, - StatusText: jm.Error.Message, - }) - } else if jm.Status == "Pull complete" || jm.Status == "Already exists" { - w.Event(progress.Event{ - ID: jm.ID, - Text: jm.Status, - Status: progress.Done, - }) - } else { - w.Event(progress.Event{ - ID: jm.ID, - Text: jm.Status, - Status: progress.Working, - }) - } +func (s *local) needPull(ctx context.Context, service types.ServiceConfig) (bool, error) { + _, _, err := s.containerService.apiClient.ImageInspectWithRaw(ctx, service.Image) + if err != nil { + if errdefs.IsNotFound(err) { + return true, nil } + return false, err } + return false, nil } func (s *local) Down(ctx context.Context, projectName string) error {