From 2f09b634ccb919f33d972c3d54943d91f9f6d416 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 3 Dec 2020 12:22:01 +0100 Subject: [PATCH] Implement `docker compose pull` Signed-off-by: Djordje Lukic --- aci/compose.go | 4 ++ api/client/compose.go | 10 ++- api/compose/api.go | 2 + cli/cmd/compose/compose.go | 4 +- cli/cmd/compose/pull.go | 71 ++++++++++++++++++++ ecs/local/compose.go | 4 ++ ecs/up.go | 4 ++ example/backend.go | 4 ++ local/compose.go | 128 ++++++++++++++++++++++++++++++++++++- progress/event.go | 1 + progress/tty.go | 27 ++++++-- progress/tty_test.go | 8 +-- 12 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 cli/cmd/compose/pull.go diff --git a/aci/compose.go b/aci/compose.go index 874daf8c7..e77d3a6b7 100644 --- a/aci/compose.go +++ b/aci/compose.go @@ -52,6 +52,10 @@ func (cs *aciComposeService) Push(ctx context.Context, project *types.Project) e return errdefs.ErrNotImplemented } +func (cs *aciComposeService) Pull(ctx context.Context, project *types.Project) error { + return errdefs.ErrNotImplemented +} + func (cs *aciComposeService) Up(ctx context.Context, project *types.Project, detach bool) error { logrus.Debugf("Up on project with name %q", project.Name) diff --git a/api/client/compose.go b/api/client/compose.go index 486faa536..bc22be484 100644 --- a/api/client/compose.go +++ b/api/client/compose.go @@ -37,32 +37,30 @@ func (c *composeService) Push(ctx context.Context, project *types.Project) error return errdefs.ErrNotImplemented } -// Up executes the equivalent to a `compose up` +func (c *composeService) Pull(ctx context.Context, project *types.Project) error { + return errdefs.ErrNotImplemented +} + func (c *composeService) Up(context.Context, *types.Project, bool) error { return errdefs.ErrNotImplemented } -// Down executes the equivalent to a `compose down` func (c *composeService) Down(context.Context, string) error { return errdefs.ErrNotImplemented } -// Logs executes the equivalent to a `compose logs` func (c *composeService) Logs(context.Context, string, io.Writer) error { return errdefs.ErrNotImplemented } -// Ps executes the equivalent to a `compose ps` func (c *composeService) Ps(context.Context, string) ([]compose.ServiceStatus, error) { return nil, errdefs.ErrNotImplemented } -// List executes the equivalent to a `docker stack ls` func (c *composeService) List(context.Context, string) ([]compose.Stack, error) { return nil, errdefs.ErrNotImplemented } -// Convert translate compose model into backend's native format func (c *composeService) Convert(context.Context, *types.Project, string) ([]byte, error) { return nil, errdefs.ErrNotImplemented } diff --git a/api/compose/api.go b/api/compose/api.go index 050272911..6ac4ac454 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -29,6 +29,8 @@ type Service interface { Build(ctx context.Context, project *types.Project) error // Push executes the equivalent ot a `compose push` Push(ctx context.Context, project *types.Project) error + // Pull executes the equivalent of a `compose pull` + Pull(ctx context.Context, project *types.Project) error // Up executes the equivalent to a `compose up` Up(ctx context.Context, project *types.Project, detach bool) error // Down executes the equivalent to a `compose down` diff --git a/cli/cmd/compose/compose.go b/cli/cmd/compose/compose.go index d7a45b44d..f8012cc4f 100644 --- a/cli/cmd/compose/compose.go +++ b/cli/cmd/compose/compose.go @@ -93,7 +93,9 @@ func Command(contextType string) *cobra.Command { if contextType == store.LocalContextType { command.AddCommand( buildCommand(), - pushCommand()) + pushCommand(), + pullCommand(), + ) } return command diff --git a/cli/cmd/compose/pull.go b/cli/cmd/compose/pull.go new file mode 100644 index 000000000..f757ebb42 --- /dev/null +++ b/cli/cmd/compose/pull.go @@ -0,0 +1,71 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + + "github.com/compose-spec/compose-go/cli" + "github.com/spf13/cobra" + + "github.com/docker/compose-cli/api/client" + "github.com/docker/compose-cli/progress" +) + +type pullOptions struct { + composeOptions +} + +func pullCommand() *cobra.Command { + opts := pullOptions{} + pullCmd := &cobra.Command{ + Use: "pull [SERVICE...]", + RunE: func(cmd *cobra.Command, args []string) error { + return runPull(cmd.Context(), opts, args) + }, + } + + pullCmd.Flags().StringVar(&opts.WorkingDir, "workdir", "", "Work dir") + pullCmd.Flags().StringArrayVarP(&opts.ConfigPaths, "file", "f", []string{}, "Compose configuration files") + + return pullCmd +} + +func runPull(ctx context.Context, opts pullOptions, services []string) error { + c, err := client.New(ctx) + if err != nil { + return err + } + + _, err = progress.Run(ctx, func(ctx context.Context) (string, error) { + options, err := opts.toProjectOptions() + if err != nil { + return "", err + } + project, err := cli.ProjectFromOptions(options) + if err != nil { + return "", err + } + + err = filter(project, services) + if err != nil { + return "", err + } + return "", c.ComposeService().Pull(ctx, project) + }) + return err +} diff --git a/ecs/local/compose.go b/ecs/local/compose.go index 96ffb8663..c1d8b8cdf 100644 --- a/ecs/local/compose.go +++ b/ecs/local/compose.go @@ -49,6 +49,10 @@ func (e ecsLocalSimulation) Push(ctx context.Context, project *types.Project) er return errdefs.ErrNotImplemented } +func (e ecsLocalSimulation) Pull(ctx context.Context, project *types.Project) error { + return errdefs.ErrNotImplemented +} + func (e ecsLocalSimulation) Up(ctx context.Context, project *types.Project, detach bool) error { cmd := exec.Command("docker-compose", "version", "--short") b := bytes.Buffer{} diff --git a/ecs/up.go b/ecs/up.go index eabb80d2f..a5c1ce3af 100644 --- a/ecs/up.go +++ b/ecs/up.go @@ -35,6 +35,10 @@ func (b *ecsAPIService) Push(ctx context.Context, project *types.Project) error return errdefs.ErrNotImplemented } +func (b *ecsAPIService) Pull(ctx context.Context, project *types.Project) error { + return errdefs.ErrNotImplemented +} + func (b *ecsAPIService) Up(ctx context.Context, project *types.Project, detach bool) error { err := b.aws.CheckRequirements(ctx, b.Region) if err != nil { diff --git a/example/backend.go b/example/backend.go index 4238a9b46..480783177 100644 --- a/example/backend.go +++ b/example/backend.go @@ -147,6 +147,10 @@ func (cs *composeService) Push(ctx context.Context, project *types.Project) erro return errdefs.ErrNotImplemented } +func (cs *composeService) Pull(ctx context.Context, project *types.Project) error { + return errdefs.ErrNotImplemented +} + func (cs *composeService) Up(ctx context.Context, project *types.Project, detach bool) error { fmt.Printf("Up command on project %q", project.Name) return nil diff --git a/local/compose.go b/local/compose.go index 2ba976c5b..790fb5dba 100644 --- a/local/compose.go +++ b/local/compose.go @@ -32,7 +32,7 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" "github.com/docker/buildx/build" - "github.com/docker/cli/cli/config" + cliconfig "github.com/docker/cli/cli/config" "github.com/docker/distribution/reference" moby "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -52,6 +52,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/docker/compose-cli/api/compose" + "github.com/docker/compose-cli/config" "github.com/docker/compose-cli/formatter" "github.com/docker/compose-cli/progress" ) @@ -72,7 +73,7 @@ func (s *composeService) Build(ctx context.Context, project *types.Project) erro } func (s *composeService) Push(ctx context.Context, project *types.Project) error { - configFile, err := config.Load(config.Dir()) + configFile, err := cliconfig.Load(config.Dir(ctx)) if err != nil { return err } @@ -136,7 +137,7 @@ func (s *composeService) Push(ctx context.Context, project *types.Project) error if jm.Error != nil { return errors.New(jm.Error.Message) } - toProgressEvent(service.Name, jm, w) + toProgressEvent("Pushing "+service.Name, jm, w) } return nil }) @@ -144,6 +145,127 @@ func (s *composeService) Push(ctx context.Context, project *types.Project) error return eg.Wait() } +func (s *composeService) Pull(ctx context.Context, project *types.Project) error { + configFile, err := cliconfig.Load(config.Dir(ctx)) + if err != nil { + return err + } + info, err := s.apiClient.Info(ctx) + if err != nil { + return err + } + + if info.IndexServerAddress == "" { + info.IndexServerAddress = registry.IndexServer + } + + w := progress.ContextWriter(ctx) + eg, ctx := errgroup.WithContext(ctx) + + for _, srv := range project.Services { + service := srv + eg.Go(func() error { + w.Event(progress.Event{ + ID: service.Name, + Status: progress.Working, + Text: "Pulling", + }) + ref, err := reference.ParseNormalizedNamed(service.Image) + if err != nil { + return err + } + + repoInfo, err := registry.ParseRepositoryInfo(ref) + if err != nil { + return err + } + + key := repoInfo.Index.Name + if repoInfo.Index.Official { + key = info.IndexServerAddress + } + + authConfig, err := configFile.GetAuthConfig(key) + if err != nil { + return err + } + + buf, err := json.Marshal(authConfig) + if err != nil { + return err + } + + stream, err := s.apiClient.ImagePull(ctx, service.Image, moby.ImagePullOptions{ + RegistryAuth: base64.URLEncoding.EncodeToString(buf), + }) + if err != nil { + w.Event(progress.Event{ + ID: service.Name, + Status: progress.Error, + Text: "Error", + }) + return err + } + + dec := json.NewDecoder(stream) + for { + var jm jsonmessage.JSONMessage + if err := dec.Decode(&jm); err != nil { + if err == io.EOF { + break + } + return err + } + if jm.Error != nil { + return errors.New(jm.Error.Message) + } + toPullProgressEvent(service.Name, jm, w) + } + w.Event(progress.Event{ + ID: service.Name, + Status: progress.Done, + Text: "Pulled", + }) + return nil + }) + } + + return eg.Wait() +} + +func toPullProgressEvent(parent string, jm jsonmessage.JSONMessage, w progress.Writer) { + if jm.ID == "" || jm.Progress == nil { + return + } + + var ( + text string + status = progress.Working + ) + + text = jm.Progress.String() + + if jm.Status == "Pull complete" || + jm.Status == "Already exists" || + strings.Contains(jm.Status, "Image is up to date") || + strings.Contains(jm.Status, "Downloaded newer image") { + status = progress.Done + } + + if jm.Error != nil { + status = progress.Error + text = jm.Error.Message + } + + w.Event(progress.Event{ + ID: jm.ID, + ParentID: parent, + Text: jm.Status, + Status: status, + StatusText: text, + }) +} + func toProgressEvent(prefix string, jm jsonmessage.JSONMessage, w progress.Writer) { if jm.ID == "" { // skipped diff --git a/progress/event.go b/progress/event.go index 132671e79..05ea7ff12 100644 --- a/progress/event.go +++ b/progress/event.go @@ -33,6 +33,7 @@ const ( // Event represents a progress event. type Event struct { ID string + ParentID string Text string Status EventStatus StatusText string diff --git a/progress/tty.go b/progress/tty.go index 79e252478..1b7fc74df 100644 --- a/progress/tty.go +++ b/progress/tty.go @@ -79,6 +79,7 @@ func (w *ttyWriter) Event(e Event) { last.Status = e.Status last.Text = e.Text last.StatusText = e.StatusText + last.ParentID = e.ParentID w.events[e.ID] = last } else { e.startTime = time.Now() @@ -116,24 +117,41 @@ func (w *ttyWriter) print() { var statusPadding int for _, v := range w.eventIDs { - l := len(fmt.Sprintf("%s %s", w.events[v].ID, w.events[v].Text)) + event := w.events[v] + l := len(fmt.Sprintf("%s %s", event.ID, event.Text)) if statusPadding < l { statusPadding = l } + if event.ParentID != "" { + statusPadding -= 2 + } } numLines := 0 for _, v := range w.eventIDs { - line := lineText(w.events[v], terminalWidth, statusPadding, runtime.GOOS != "windows") + event := w.events[v] + if event.ParentID != "" { + continue + } + line := lineText(event, "", terminalWidth, statusPadding, runtime.GOOS != "windows") // nolint: errcheck fmt.Fprint(w.out, line) numLines++ + for _, v := range w.eventIDs { + ev := w.events[v] + if ev.ParentID == event.ID { + line := lineText(ev, " ", terminalWidth, statusPadding, runtime.GOOS != "windows") + // nolint: errcheck + fmt.Fprint(w.out, line) + numLines++ + } + } } w.numLines = numLines } -func lineText(event Event, terminalWidth, statusPadding int, color bool) string { +func lineText(event Event, pad string, terminalWidth, statusPadding int, color bool) string { endTime := time.Now() if event.Status != Working { endTime = event.endTime @@ -154,7 +172,8 @@ func lineText(event Event, terminalWidth, statusPadding int, color bool) string if maxStatusLen > 0 && len(status) > maxStatusLen { status = status[:maxStatusLen] + "..." } - text := fmt.Sprintf(" %s %s %s%s %s", + text := fmt.Sprintf("%s %s %s %s%s %s", + pad, event.spinner.String(), event.ID, event.Text, diff --git a/progress/tty_test.go b/progress/tty_test.go index 51dc6f612..5907ebd64 100644 --- a/progress/tty_test.go +++ b/progress/tty_test.go @@ -41,18 +41,18 @@ func TestLineText(t *testing.T) { lineWidth := len(fmt.Sprintf("%s %s", ev.ID, ev.Text)) - out := lineText(ev, 50, lineWidth, true) + out := lineText(ev, "", 50, lineWidth, true) assert.Equal(t, out, "\x1b[37m . id Text Status 0.0s\n\x1b[0m") - out = lineText(ev, 50, lineWidth, false) + out = lineText(ev, "", 50, lineWidth, false) assert.Equal(t, out, " . id Text Status 0.0s\n") ev.Status = Done - out = lineText(ev, 50, lineWidth, true) + out = lineText(ev, "", 50, lineWidth, true) assert.Equal(t, out, "\x1b[34m . id Text Status 0.0s\n\x1b[0m") ev.Status = Error - out = lineText(ev, 50, lineWidth, true) + out = lineText(ev, "", 50, lineWidth, true) assert.Equal(t, out, "\x1b[31m . id Text Status 0.0s\n\x1b[0m") }