Merge pull request #432 from docker/local-e2e

Refactor e2e tests

Makefile

@@ -23,7 +23,12 @@ ifeq ($(UNAME_S),Darwin)
 endif
 
 GIT_TAG?=$(shell git describe --tags --match "v[0-9]*")
-TESTIFY_OPTS=$(if $(TESTIFY),-testify.m $(TESTIFY),)
+TEST_FLAGS?=
+E2E_TEST?=
+ifeq ($(E2E_TEST),)
+else
+	TEST_FLAGS=-run $(E2E_TEST)
+endif
 
 all: cli
 
@@ -38,14 +43,14 @@ cli: ## Compile the cli
 	--build-arg GIT_TAG=$(GIT_TAG) \
 	--output ./bin
 
-e2e-local: ## Run End to end local tests. set env TESTIFY=Test1 for running single test
+e2e-local: ## Run End to end local tests. Set E2E_TEST=TestName to run a single test
-	go test -v ./tests/e2e ./tests/skip-win-ci-e2e ./local/e2e $(TESTIFY_OPTS)
+	go test -count=1 -v $(TEST_FLAGS) ./tests/e2e ./tests/skip-win-ci-e2e ./local/e2e
 
-e2e-win-ci: ## Run End to end local tests on windows CI, no docker for linux containers available ATM. set env TESTIFY=Test1 for running single test
+e2e-win-ci: ## Run end to end local tests on Windows CI, no Docker for Linux containers available ATM. Set E2E_TEST=TestName to run a single test
-	go test -v ./tests/e2e $(TESTIFY_OPTS)
+	go test -count=1 -v $(TEST_FLAGS) ./tests/e2e
 
-e2e-aci: ## Run End to end ACI tests. set env TESTIFY=Test1 for running single test
+e2e-aci: ## Run End to end ACI tests. Set E2E_TEST=TestName to run a single test
-	go test -v ./tests/aci-e2e $(TESTIFY_OPTS)
+	go test -count=1 -v $(TEST_FLAGS) ./tests/aci-e2e
 
 cross: ## Compile the CLI for linux, darwin and windows
 	@docker build . --target cross \

README.md

@@ -62,7 +62,7 @@ You might need to run again `docker login azure` to properly use the command lin
 
 You can also run a single ACI test from the test suite:
 ```
-TESTIFY=TestACIRunSingleContainer AZURE_TENANT_ID="xxx" AZURE_CLIENT_ID="yyy" AZURE_CLIENT_SECRET="yyy" make e2e-aci
+AZURE_TENANT_ID="xxx" AZURE_CLIENT_ID="yyy" AZURE_CLIENT_SECRET="yyy" make E2E_TEST=TestContainerRun e2e-aci
 ```
 
 ## Release

go.mod

@@ -41,11 +41,10 @@ require (
 	github.com/hashicorp/go-version v1.2.1 // indirect
 	github.com/moby/term v0.0.0-20200611042045-63b9a826fb74
 	github.com/morikuni/aec v1.0.0
-	github.com/onsi/gomega v1.10.1
+	github.com/onsi/gomega v1.10.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/pkg/errors v0.9.1
-	github.com/robpike/filter v0.0.0-20150108201509-2984852a2183
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
@@ -58,6 +57,5 @@ require (
 	google.golang.org/protobuf v1.25.0
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/ini.v1 v1.57.0
-	gotest.tools v2.2.0+incompatible
 	gotest.tools/v3 v3.0.2
 )

go.sum

@@ -19,8 +19,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest v0.11.0 h1:tnO41Uo+/0sxTMFY/U7aKg2abek3JOnnXcuSuba74jI=
 github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest v0.11.2 h1:BR5GoSGobeiMwGOOIxXuvNKNPy+HMGdteKB8kJUDnBE=
-github.com/Azure/go-autorest/autorest v0.11.2/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest v0.11.3 h1:fyYnmYujkIXUgv88D9/Wo2ybE4Zwd/TmQd5sSI5u2Ws=
 github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
@@ -73,12 +71,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.30.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.33.18 h1:Ccy1SV2SsgJU3rfrD+SOhQ0jvuzfrFuja/oKI86ruPw=
-github.com/aws/aws-sdk-go v1.33.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.33.19 h1:SMna0QLInNqm+nNL9tb7OVWTqSfNYSxrCa2adnyVth4=
-github.com/aws/aws-sdk-go v1.33.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.33.20 h1:mtXKHmMQO6o0i2GTjyiVNZGlXqJDCUbiik0OQeMds/o=
-github.com/aws/aws-sdk-go v1.33.20/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.33.21 h1:ziUemjajvLABlnJFe+8sM3fpqlg/DNA4944rUZ05PhY=
 github.com/aws/aws-sdk-go v1.33.21/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc=
@@ -452,8 +444,6 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
 github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/robpike/filter v0.0.0-20150108201509-2984852a2183 h1:qDhD/wJDGyWrXKLIKmEKpKK/ejaZlguyeEaLZzmrtzo=
-github.com/robpike/filter v0.0.0-20150108201509-2984852a2183/go.mod h1:3dvYi47BCPInRb2ILlNnrXfl++XpwTWLbIxPyJsUvCw=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

local/e2e/backend_test.go

@@ -17,61 +17,67 @@
 package e2e
 
 import (
+	"fmt"
+	"os"
+	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"gotest.tools/v3/icmd"
-	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
 
-	"github.com/docker/api/tests/framework"
+	. "github.com/docker/api/tests/framework"
 )
 
-type LocalBackendTestSuite struct {
+var binDir string
-	framework.Suite
+
+func TestMain(m *testing.M) {
+	p, cleanup, err := SetupExistingCLI()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	binDir = p
+	exitCode := m.Run()
+	cleanup()
+	os.Exit(exitCode)
 }
 
-func (m *LocalBackendTestSuite) BeforeTest(suiteName string, testName string) {
+func TestLocalBackend(t *testing.T) {
-	m.NewDockerCommand("context", "create", "local", "test-context").ExecOrDie()
+	c := NewParallelE2eCLI(t, binDir)
-	m.NewDockerCommand("context", "use", "test-context").ExecOrDie()
+	c.RunDockerCmd("context", "create", "local", "test-context").Assert(t, icmd.Success)
-}
+	c.RunDockerCmd("context", "use", "test-context").Assert(t, icmd.Success)
 
-func (m *LocalBackendTestSuite) AfterTest(suiteName string, testName string) {
+	t.Run("run", func(t *testing.T) {
-	m.NewDockerCommand("context", "rm", "-f", "test-context").ExecOrDie()
+		t.Parallel()
-}
+		res := c.RunDockerCmd("run", "-d", "nginx")
+		res.Assert(t, icmd.Success)
+		containerName := strings.TrimSpace(res.Combined())
+		t.Cleanup(func() {
+			_ = c.RunDockerCmd("rm", "-f", containerName)
+		})
+		res = c.RunDockerCmd("inspect", containerName)
+		res.Assert(t, icmd.Expected{Out: `"Status": "running"`})
+	})
 
-func (m *LocalBackendTestSuite) TestPs() {
+	t.Run("run with ports", func(t *testing.T) {
-	out := m.NewDockerCommand("ps").ExecOrDie()
+		t.Parallel()
-	require.Equal(m.T(), "CONTAINER ID        IMAGE               COMMAND             STATUS              PORTS\n", out)
+		res := c.RunDockerCmd("run", "-d", "-p", "8080:80", "nginx")
-}
+		res.Assert(t, icmd.Success)
+		containerName := strings.TrimSpace(res.Combined())
+		t.Cleanup(func() {
+			_ = c.RunDockerCmd("rm", "-f", containerName)
+		})
+		res = c.RunDockerCmd("inspect", containerName)
+		res.Assert(t, icmd.Expected{Out: `"Status": "running"`})
+		res = c.RunDockerCmd("ps")
+		res.Assert(t, icmd.Expected{Out: "0.0.0.0:8080->80/tcp"})
+	})
 
-func (m *LocalBackendTestSuite) TestRun() {
+	t.Run("inspect not found", func(t *testing.T) {
-	_, err := m.NewDockerCommand("run", "-d", "--name", "nginx", "nginx").Exec()
+		t.Parallel()
-	require.Nil(m.T(), err)
+		res := c.RunDockerCmd("inspect", "nonexistentcontainer")
-	out := m.NewDockerCommand("ps").ExecOrDie()
+		res.Assert(t, icmd.Expected{
-	defer func() {
+			ExitCode: 1,
-		m.NewDockerCommand("rm", "-f", "nginx").ExecOrDie()
+			Err:      "Error: No such container: nonexistentcontainer",
-	}()
+		})
-	assert.Contains(m.T(), out, "nginx")
+	})
-}
-
-func (m *LocalBackendTestSuite) TestRunWithPorts() {
-	_, err := m.NewDockerCommand("run", "-d", "--name", "nginx", "-p", "8080:80", "nginx").Exec()
-	require.Nil(m.T(), err)
-	out := m.NewDockerCommand("ps").ExecOrDie()
-	defer func() {
-		m.NewDockerCommand("rm", "-f", "nginx").ExecOrDie()
-	}()
-	assert.Contains(m.T(), out, "8080")
-
-	out = m.NewDockerCommand("inspect", "nginx").ExecOrDie()
-	assert.Contains(m.T(), out, "\"Status\": \"running\"")
-}
-
-func (m *LocalBackendTestSuite) TestInspectNotFound() {
-	out, _ := m.NewDockerCommand("inspect", "nonexistentcontainer").Exec()
-	assert.Contains(m.T(), out, "Error: No such container: nonexistentcontainer")
-}
-
-func TestLocalBackendTestSuite(t *testing.T) {
-	suite.Run(t, new(LocalBackendTestSuite))
 }

tests/aci-e2e/e2e-aci_test.go

@@ -18,460 +18,595 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"math/rand"
+	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
-	"os/exec"
+	"runtime"
+	"strconv"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
-	"github.com/docker/api/errdefs"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/icmd"
+	"gotest.tools/v3/poll"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
 	azure_storage "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/storage/mgmt/storage"
 	"github.com/Azure/azure-storage-file-go/azfile"
 	"github.com/Azure/go-autorest/autorest/to"
-	. "github.com/onsi/gomega"
-	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/suite"
 
-	azure "github.com/docker/api/aci"
+	"github.com/docker/api/aci"
 	"github.com/docker/api/aci/login"
+	"github.com/docker/api/containers"
 	"github.com/docker/api/context/store"
+	"github.com/docker/api/errdefs"
 	"github.com/docker/api/tests/aci-e2e/storage"
 	. "github.com/docker/api/tests/framework"
 )
 
 const (
-	location          = "westeurope"
+	contextName = "aci-test"
-	contextName       = "acitest"
+	location    = "westeurope"
-	testContainerName = "testcontainername"
-	testShareName     = "dockertestshare"
-	testFileContent   = "Volume mounted with success!"
-	testFileName      = "index.html"
 )
 
-var (
+var binDir string
-	subscriptionID string
-)
 
-type E2eACISuite struct {
+func TestMain(m *testing.M) {
-	Suite
+	p, cleanup, err := SetupExistingCLI()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	binDir = p
+	exitCode := m.Run()
+	cleanup()
+	os.Exit(exitCode)
 }
 
-func (s *E2eACISuite) TestLoginLogoutCreateContextError() {
+// Cannot be parallelized as login/logout is global.
-	s.Step("Logs in azure using service principal credentials", azureLogin)
+func TestLoginLogout(t *testing.T) {
+	startTime := strconv.Itoa(int(time.Now().UnixNano()))
+	c := NewE2eCLI(t, binDir)
+	rg := "E2E-" + startTime
 
-	s.Step("logout from azure", func() {
+	t.Run("login", func(t *testing.T) {
-		output := s.NewDockerCommand("logout", "azure").ExecOrDie()
+		azureLogin(t)
-		Expect(output).To(ContainSubstring(""))
+	})
+
+	t.Run("create context", func(t *testing.T) {
+		sID := getSubscriptionID(t)
+		err := createResourceGroup(sID, rg)
+		assert.Check(t, is.Nil(err))
+		t.Cleanup(func() {
+			_ = deleteResourceGroup(rg)
+		})
+
+		res := c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rg, "--location", location)
+		res.Assert(t, icmd.Success)
+		res = c.RunDockerCmd("context", "use", contextName)
+		res.Assert(t, icmd.Expected{Out: contextName})
+		res = c.RunDockerCmd("context", "ls")
+		res.Assert(t, icmd.Expected{Out: contextName + " *"})
+	})
+
+	t.Run("delete context", func(t *testing.T) {
+		res := c.RunDockerCmd("context", "use", "default")
+		res.Assert(t, icmd.Expected{Out: "default"})
+
+		res = c.RunDockerCmd("context", "rm", contextName)
+		res.Assert(t, icmd.Expected{Out: contextName})
+	})
+
+	t.Run("logout", func(t *testing.T) {
 		_, err := os.Stat(login.GetTokenStorePath())
-		Expect(os.IsNotExist(err)).To(BeTrue())
+		assert.NilError(t, err)
+		res := c.RunDockerCmd("logout", "azure")
+		res.Assert(t, icmd.Expected{Out: "Removing login credentials for Azure"})
+		_, err = os.Stat(login.GetTokenStorePath())
+		assert.ErrorContains(t, err, "no such file or directory")
 	})
 
-	s.Step("check context create fails with an explicit error and returns a specific error code", func() {
+	t.Run("create context fail", func(t *testing.T) {
-		cmd := exec.Command("docker", "context", "create", "aci", "someContext")
+		res := c.RunDockerCmd("context", "create", "aci", "fail-context")
-		bytes, err := cmd.CombinedOutput()
+		res.Assert(t, icmd.Expected{
-		Expect(err).NotTo(BeNil())
+			ExitCode: errdefs.ExitCodeLoginRequired,
-		Expect(string(bytes)).To(ContainSubstring("not logged in to azure, you need to run \"docker login azure\" first"))
+			Err:      `not logged in to azure, you need to run "docker login azure" first`,
-		Expect(cmd.ProcessState.ExitCode()).To(Equal(errdefs.ExitCodeLoginRequired))
+		})
 	})
 }
 
-func (s *E2eACISuite) TestACIRunSingleContainer() {
+func TestContainerRun(t *testing.T) {
-	var containerName string
+	c := NewParallelE2eCLI(t, binDir)
-	resourceGroupName := s.setupTestResourceGroup()
+	sID, rg := setupTestResourceGroup(t, c)
-	defer deleteResourceGroup(resourceGroupName)
 
-	var nginxExposedURL string
+	const (
-	var containerID string
+		testShareName   = "dockertestshare"
-	s.Step("runs nginx on port 80", func() {
+		testFileContent = "Volume mounted successfully!"
-		aciContext := store.AciContext{
+		testFileName    = "index.html"
-			SubscriptionID: subscriptionID,
+	)
-			Location:       location,
+
-			ResourceGroup:  resourceGroupName,
+	// Bootstrap volume
+	aciContext := store.AciContext{
+		SubscriptionID: sID,
+		Location:       location,
+		ResourceGroup:  rg,
+	}
+	saName := "e2e" + strconv.Itoa(int(time.Now().UnixNano()))
+	_, cleanupSa := createStorageAccount(t, aciContext, saName)
+	t.Cleanup(func() {
+		if err := cleanupSa(); err != nil {
+			t.Error(err)
 		}
+	})
+	keys := getStorageKeys(t, aciContext, saName)
+	assert.Assert(t, len(keys) > 0)
+	k := *keys[0].Value
+	cred, u := createFileShare(t, k, testShareName, saName)
+	uploadFile(t, *cred, u.String(), testFileName, testFileContent)
 
-		testStorageAccountName := "storageteste2e" + RandStringBytes(6) // "between 3 and 24 characters in length and use numbers and lower-case letters only"
+	// Used in subtests
-		createStorageAccount(aciContext, testStorageAccountName)
+	var (
-		defer deleteStorageAccount(aciContext, testStorageAccountName)
+		container string
-		keys := getStorageKeys(aciContext, testStorageAccountName)
+		hostIP    string
-		firstKey := *keys[0].Value
+		endpoint  string
-		credential, u := createFileShare(firstKey, testShareName, testStorageAccountName)
+	)
-		uploadFile(credential, u.String(), testFileName, testFileContent)
 
+	t.Run("run", func(t *testing.T) {
 		mountTarget := "/usr/share/nginx/html"
-		output := s.NewDockerCommand("run", "-d", "nginx",
+		res := c.RunDockerCmd(
-			"-v", fmt.Sprintf("%s:%s@%s:%s",
+			"run", "-d",
-				testStorageAccountName, firstKey, testShareName, mountTarget),
+			"-v", fmt.Sprintf("%s:%s@%s:%s", saName, k, testShareName, mountTarget),
 			"-p", "80:80",
-		).ExecOrDie()
+			"nginx",
-		runOutput := Lines(output)
+		)
-		containerName = runOutput[len(runOutput)-1]
+		res.Assert(t, icmd.Success)
-
+		container = getContainerName(res.Stdout())
-		output = s.NewDockerCommand("ps").ExecOrDie()
+		t.Logf("Container name: %s", container)
-		lines := Lines(output)
-		Expect(len(lines)).To(Equal(2))
-
-		containerFields := Columns(lines[1])
-		Expect(containerFields[1]).To(Equal("nginx"))
-		Expect(containerFields[2]).To(Equal("Running"))
-		exposedIP := containerFields[3]
-		containerID = containerFields[0]
-		Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
-
-		nginxExposedURL = strings.ReplaceAll(exposedIP, "->80/tcp", "")
-		output = s.NewCommand("curl", nginxExposedURL).ExecOrDie()
-		Expect(output).To(ContainSubstring(testFileContent))
-
-		output = s.NewDockerCommand("logs", containerID).ExecOrDie()
-		Expect(output).To(ContainSubstring("GET"))
 	})
 
-	s.Step("inspect command", func() {
+	t.Run("inspect", func(t *testing.T) {
-		inspect := s.NewDockerCommand("inspect", containerID).ExecOrDie()
+		res := c.RunDockerCmd("inspect", container)
-		Expect(inspect).To(ContainSubstring("\"Platform\": \"Linux\""))
+		res.Assert(t, icmd.Success)
-		Expect(inspect).To(ContainSubstring("\"CPULimit\": 1"))
+
-		Expect(inspect).To(ContainSubstring("\"RestartPolicyCondition\": \"none\""))
+		containerInspect, err := ParseContainerInspect(res.Stdout())
+		assert.NilError(t, err)
+		assert.Equal(t, containerInspect.Platform, "Linux")
+		assert.Equal(t, containerInspect.CPULimit, 1.0)
+		assert.Equal(t, containerInspect.RestartPolicyCondition, containers.RestartPolicyNone)
+
+		assert.Assert(t, is.Len(containerInspect.Ports, 1))
+		hostIP = containerInspect.Ports[0].HostIP
+		endpoint = fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
+		t.Logf("Endpoint: %s", endpoint)
 	})
 
-	s.Step("exec command", func() {
+	t.Run("ps", func(t *testing.T) {
-		output := s.NewDockerCommand("exec", containerName, "pwd").ExecOrDie()
+		res := c.RunDockerCmd("ps")
-		Expect(output).To(ContainSubstring("/"))
+		res.Assert(t, icmd.Success)
-
+		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
-		_, err := s.NewDockerCommand("exec", containerName, "echo", "fail_with_argument").Exec()
+		l := out[len(out)-1]
-		Expect(err.Error()).To(ContainSubstring("ACI exec command does not accept arguments to the command. " +
+		assert.Assert(t, strings.Contains(l, container))
-			"Only the binary should be specified"))
+		assert.Assert(t, strings.Contains(l, "nginx"))
+		assert.Assert(t, strings.Contains(l, "Running"))
+		assert.Assert(t, strings.Contains(l, hostIP+":80->80/tcp"))
 	})
 
-	s.Step("follow logs from nginx", func() {
+	t.Run("http get", func(t *testing.T) {
-		timeChan := make(chan time.Time)
+		r, err := http.Get(endpoint)
-
+		assert.NilError(t, err)
-		ctx := s.NewDockerCommand("logs", "--follow", containerName).WithTimeout(timeChan)
+		assert.Equal(t, r.StatusCode, http.StatusOK)
-		outChan := make(chan string)
+		b, err := ioutil.ReadAll(r.Body)
-
+		assert.NilError(t, err)
-		go func() {
+		assert.Assert(t, strings.Contains(string(b), testFileContent), "Actual content: "+string(b))
-			output, err := ctx.Exec()
-			// check the process is cancelled by the test, not another unexpected error
-			Expect(err.Error()).To(ContainSubstring("timed out"))
-			outChan <- output
-		}()
-		// Ensure logs -- follow is strated before we curl nginx
-		time.Sleep(5 * time.Second)
-
-		s.NewCommand("curl", nginxExposedURL+"/test").ExecOrDie()
-		// Give the `logs --follow` a little time to get logs of the curl call
-		time.Sleep(5 * time.Second)
-
-		// Trigger a timeout to make ctx.Exec exit
-		timeChan <- time.Now()
-
-		output := <-outChan
-
-		Expect(output).To(ContainSubstring("/test"))
 	})
 
-	s.Step("removes container nginx", func() {
+	t.Run("logs", func(t *testing.T) {
-		output := s.NewDockerCommand("rm", containerName).ExecOrDie()
+		res := c.RunDockerCmd("logs", container)
-		Expect(Lines(output)[0]).To(Equal(containerName))
+		res.Assert(t, icmd.Expected{Out: "GET"})
 	})
 
-	s.Step("re-run nginx with modified cpu/mem, and without --detach and follow logs", func() {
+	t.Run("exec", func(t *testing.T) {
-		shutdown := make(chan time.Time)
+		res := c.RunDockerCmd("exec", container, "pwd")
-		errs := make(chan error)
+		res.Assert(t, icmd.Expected{Out: "/"})
-		outChan := make(chan string)
+
-		cmd := s.NewDockerCommand("run", "nginx", "--restart", "on-failure", "--memory", "0.1G", "--cpus", "0.1", "-p", "80:80", "--name", testContainerName).WithTimeout(shutdown)
+		res = c.RunDockerCmd("exec", container, "echo", "fail_with_argument")
-		go func() {
+		res.Assert(t, icmd.Expected{
-			output, err := cmd.Exec()
+			ExitCode: 1,
-			outChan <- output
+			Err:      "ACI exec command does not accept arguments to the command. Only the binary should be specified",
-			errs <- err
-		}()
-		err := WaitFor(time.Second, 100*time.Second, errs, func() bool {
-			output := s.NewDockerCommand("ps").ExecOrDie()
-			lines := Lines(output)
-			if len(lines) != 2 {
-				return false
-			}
-			containerFields := Columns(lines[1])
-			if containerFields[2] != "Running" {
-				return false
-			}
-			containerID = containerFields[0]
-			nginxExposedURL = strings.ReplaceAll(containerFields[3], "->80/tcp", "")
-			return true
 		})
-		Expect(err).NotTo(HaveOccurred())
-
-		s.NewCommand("curl", nginxExposedURL+"/test").ExecOrDie()
-		inspect := s.NewDockerCommand("inspect", containerID).ExecOrDie()
-		Expect(inspect).To(ContainSubstring("\"CPULimit\": 0.1"))
-		Expect(inspect).To(ContainSubstring("\"MemoryLimit\": 107374182"))
-		Expect(inspect).To(ContainSubstring("\"RestartPolicyCondition\": \"on-failure\""))
-
-		// Give a little time to get logs of the curl call
-		time.Sleep(5 * time.Second)
-		// Kill
-		close(shutdown)
-
-		output := <-outChan
-		Expect(output).To(ContainSubstring("/test"))
 	})
 
-	s.Step("removes container nginx", func() {
+	t.Run("logs follow", func(t *testing.T) {
-		output := s.NewDockerCommand("rm", testContainerName).ExecOrDie()
+		cmd := c.NewDockerCmd("logs", "--follow", container)
-		Expect(Lines(output)[0]).To(Equal(testContainerName))
+		res := icmd.StartCmd(cmd)
-	})
-}
 
-func (s *E2eACISuite) TestACIComposeApplication() {
+		checkUp := func(t poll.LogT) poll.Result {
-	defer deleteResourceGroup(s.setupTestResourceGroup())
+			r, _ := http.Get(endpoint + "/is_up")
-
+			if r != nil && r.StatusCode == http.StatusNotFound {
-	var exposedURL string
+				return poll.Success()
-	const composeFile = "../composefiles/aci-demo/aci_demo_port.yaml"
-	const composeFileMultiplePorts = "../composefiles/aci-demo/aci_demo_multi_port.yaml"
-	const composeProjectName = "acie2e"
-	const serverContainer = composeProjectName + "_web"
-	const wordsContainer = composeProjectName + "_words"
-
-	s.Step("deploys a compose app", func() {
-		// specifically do not specify project name here, it will be derived from current folder "acie2e"
-		s.NewDockerCommand("compose", "up", "-f", composeFile).ExecOrDie()
-		output := s.NewDockerCommand("ps").ExecOrDie()
-		Lines := Lines(output)
-		Expect(len(Lines)).To(Equal(4))
-		webChecked := false
-
-		for _, line := range Lines[1:] {
-			Expect(line).To(ContainSubstring("Running"))
-			if strings.Contains(line, serverContainer) {
-				webChecked = true
-				containerFields := Columns(line)
-				exposedIP := containerFields[3]
-				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
-
-				exposedURL = strings.ReplaceAll(exposedIP, "->80/tcp", "")
-				output = s.NewCommand("curl", exposedURL).ExecOrDie()
-				Expect(output).To(ContainSubstring("Docker Compose demo"))
-				output = s.NewCommand("curl", exposedURL+"/words/noun").ExecOrDie()
-				Expect(output).To(ContainSubstring("\"word\":"))
 			}
+			return poll.Continue("waiting for container to serve request")
+		}
+		poll.WaitOn(t, checkUp, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))
+
+		assert.Assert(t, !strings.Contains(res.Stdout(), "/test"))
+
+		checkLogs := func(t poll.LogT) poll.Result {
+			if strings.Contains(res.Stdout(), "/test") {
+				return poll.Success()
+			}
+			return poll.Continue("waiting for logs to contain /test")
 		}
 
-		Expect(webChecked).To(BeTrue())
+		// Do request on /test
+		go func() {
+			time.Sleep(3 * time.Second)
+			_, _ = http.Get(endpoint + "/test")
+		}()
+
+		poll.WaitOn(t, checkLogs, poll.WithDelay(3*time.Second), poll.WithTimeout(20*time.Second))
+
+		if runtime.GOOS == "windows" {
+			err := res.Cmd.Process.Kill()
+			assert.NilError(t, err)
+		} else {
+			err := res.Cmd.Process.Signal(syscall.SIGTERM)
+			assert.NilError(t, err)
+		}
 	})
 
-	s.Step("get logs from web service", func() {
+	t.Run("rm", func(t *testing.T) {
-		output := s.NewDockerCommand("logs", serverContainer).ExecOrDie()
+		res := c.RunDockerCmd("rm", container)
-		Expect(output).To(ContainSubstring("Listening on port 80"))
+		res.Assert(t, icmd.Expected{Out: container})
-	})
+		checkStopped := func(t poll.LogT) poll.Result {
-
+			res := c.RunDockerCmd("inspect", container)
-	s.Step("updates a compose app", func() {
+			if res.ExitCode == 1 {
-		s.NewDockerCommand("compose", "up", "-f", composeFileMultiplePorts, "--project-name", composeProjectName).ExecOrDie()
+				return poll.Success()
-		// Expect(output).To(ContainSubstring("Successfully deployed"))
-		output := s.NewDockerCommand("ps").ExecOrDie()
-		Lines := Lines(output)
-		Expect(len(Lines)).To(Equal(4))
-		webChecked := false
-		wordsChecked := false
-
-		for _, line := range Lines[1:] {
-			Expect(line).To(ContainSubstring("Running"))
-			if strings.Contains(line, serverContainer) {
-				webChecked = true
-				containerFields := Columns(line)
-				exposedIP := containerFields[3]
-				Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
-
-				url := strings.ReplaceAll(exposedIP, "->80/tcp", "")
-				Expect(exposedURL).To(Equal(url))
 			}
-			if strings.Contains(line, wordsContainer) {
+			return poll.Continue("waiting for container to stop")
-				wordsChecked = true
+		}
-				containerFields := Columns(line)
+		poll.WaitOn(t, checkStopped, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))
-				exposedIP := containerFields[3]
+	})
-				Expect(exposedIP).To(ContainSubstring(":8080->8080/tcp"))
+}
 
-				url := strings.ReplaceAll(exposedIP, "->8080/tcp", "")
+func TestContainerRunAttached(t *testing.T) {
-				output = s.NewCommand("curl", url+"/noun").ExecOrDie()
+	c := NewParallelE2eCLI(t, binDir)
-				Expect(output).To(ContainSubstring("\"word\":"))
+	_, _ = setupTestResourceGroup(t, c)
+
+	// Used in subtests
+	var (
+		container string
+		endpoint  string
+	)
+
+	t.Run("run attached limits", func(t *testing.T) {
+		container = "test-container"
+		cmd := c.NewDockerCmd(
+			"run",
+			"--name", container,
+			"--restart", "on-failure",
+			"--memory", "0.1G", "--cpus", "0.1",
+			"-p", "80:80",
+			"nginx",
+		)
+		runRes := icmd.StartCmd(cmd)
+
+		checkRunning := func(t poll.LogT) poll.Result {
+			res := c.RunDockerCmd("inspect", container)
+			if res.ExitCode == 0 {
+				return poll.Success()
+			}
+			return poll.Continue("waiting for container to be running")
+		}
+		poll.WaitOn(t, checkRunning, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))
+
+		inspectRes := c.RunDockerCmd("inspect", container)
+		inspectRes.Assert(t, icmd.Success)
+
+		containerInspect, err := ParseContainerInspect(inspectRes.Stdout())
+		assert.NilError(t, err)
+		assert.Equal(t, containerInspect.Platform, "Linux")
+		assert.Equal(t, containerInspect.CPULimit, 0.1)
+		assert.Equal(t, containerInspect.MemoryLimit, uint64(107374182))
+		assert.Equal(t, containerInspect.RestartPolicyCondition, containers.RestartPolicyOnFailure)
+
+		assert.Assert(t, is.Len(containerInspect.Ports, 1))
+		endpoint = fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
+		t.Logf("Endpoint: %s", endpoint)
+
+		assert.Assert(t, !strings.Contains(runRes.Stdout(), "/test"))
+		checkRequest := func(t poll.LogT) poll.Result {
+			r, _ := http.Get(endpoint + "/test")
+			if r != nil && r.StatusCode == http.StatusNotFound {
+				return poll.Success()
+			}
+			return poll.Continue("waiting for container to serve request")
+		}
+		poll.WaitOn(t, checkRequest, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))
+
+		checkLog := func(t poll.LogT) poll.Result {
+			if strings.Contains(runRes.Stdout(), "/test") {
+				return poll.Success()
+			}
+			return poll.Continue("waiting for logs to contain /test")
+		}
+		poll.WaitOn(t, checkLog, poll.WithDelay(1*time.Second), poll.WithTimeout(20*time.Second))
+	})
+
+	t.Run("rm attached", func(t *testing.T) {
+		res := c.RunDockerCmd("rm", container)
+		res.Assert(t, icmd.Expected{Out: container})
+	})
+}
+
+func TestCompose(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+	_, _ = setupTestResourceGroup(t, c)
+
+	const (
+		composeFile              = "../composefiles/aci-demo/aci_demo_port.yaml"
+		composeFileMultiplePorts = "../composefiles/aci-demo/aci_demo_multi_port.yaml"
+		composeProjectName       = "acie2e"
+		serverContainer          = composeProjectName + "_web"
+		wordsContainer           = composeProjectName + "_words"
+	)
+
+	t.Run("compose up", func(t *testing.T) {
+		// Name of Compose project is taken from current folder "acie2e"
+		res := c.RunDockerCmd("compose", "up", "-f", composeFile)
+		res.Assert(t, icmd.Success)
+
+		res = c.RunDockerCmd("ps")
+		res.Assert(t, icmd.Success)
+		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
+		// Check three containers are running
+		assert.Assert(t, is.Len(out, 4))
+		webRunning := false
+		for _, l := range out {
+			if strings.Contains(l, serverContainer) {
+				webRunning = true
+				strings.Contains(l, ":80->80/tcp")
 			}
 		}
+		assert.Assert(t, webRunning, "web container not running")
 
-		Expect(webChecked).To(BeTrue())
+		res = c.RunDockerCmd("inspect", serverContainer)
-		Expect(wordsChecked).To(BeTrue())
+		res.Assert(t, icmd.Success)
+
+		containerInspect, err := ParseContainerInspect(res.Stdout())
+		assert.NilError(t, err)
+		assert.Assert(t, is.Len(containerInspect.Ports, 1))
+		endpoint := fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
+		t.Logf("Endpoint: %s", endpoint)
+
+		r, err := http.Get(endpoint + "/words/noun")
+		assert.NilError(t, err)
+		assert.Equal(t, r.StatusCode, http.StatusOK)
+		b, err := ioutil.ReadAll(r.Body)
+		assert.NilError(t, err)
+		assert.Assert(t, strings.Contains(string(b), `"word":`))
 	})
 
-	s.Step("shutdown compose app", func() {
+	t.Run("logs web", func(t *testing.T) {
-		s.NewDockerCommand("compose", "down", "--project-name", composeProjectName).ExecOrDie()
+		res := c.RunDockerCmd("logs", serverContainer)
+		res.Assert(t, icmd.Expected{Out: "Listening on port 80"})
+	})
+
+	t.Run("update", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "up", "-f", composeFileMultiplePorts, "--project-name", composeProjectName)
+		res.Assert(t, icmd.Success)
+
+		res = c.RunDockerCmd("ps")
+		res.Assert(t, icmd.Success)
+		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
+		// Check three containers are running
+		assert.Assert(t, is.Len(out, 4))
+
+		for _, cName := range []string{serverContainer, wordsContainer} {
+			res = c.RunDockerCmd("inspect", cName)
+			res.Assert(t, icmd.Success)
+
+			containerInspect, err := ParseContainerInspect(res.Stdout())
+			assert.NilError(t, err)
+			assert.Assert(t, is.Len(containerInspect.Ports, 1))
+			endpoint := fmt.Sprintf("http://%s:%d", containerInspect.Ports[0].HostIP, containerInspect.Ports[0].HostPort)
+			t.Logf("Endpoint: %s", endpoint)
+			var route string
+			switch cName {
+			case serverContainer:
+				route = "/words/noun"
+				assert.Equal(t, containerInspect.Ports[0].HostPort, uint32(80))
+				assert.Equal(t, containerInspect.Ports[0].ContainerPort, uint32(80))
+			case wordsContainer:
+				route = "/noun"
+				assert.Equal(t, containerInspect.Ports[0].HostPort, uint32(8080))
+				assert.Equal(t, containerInspect.Ports[0].ContainerPort, uint32(8080))
+			}
+			checkUp := func(t poll.LogT) poll.Result {
+				r, _ := http.Get(endpoint + route)
+				if r != nil && r.StatusCode == http.StatusOK {
+					return poll.Success()
+				}
+				return poll.Continue("Waiting for container to serve request")
+			}
+			poll.WaitOn(t, checkUp, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))
+
+			res = c.RunDockerCmd("ps")
+			p := containerInspect.Ports[0]
+			res.Assert(t, icmd.Expected{
+				Out: fmt.Sprintf("%s:%d->%d/tcp", p.HostIP, p.HostPort, p.ContainerPort),
+			})
+		}
+	})
+
+	t.Run("down", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "down", "--project-name", composeProjectName)
+		res.Assert(t, icmd.Success)
+
+		res = c.RunDockerCmd("ps")
+		res.Assert(t, icmd.Success)
+		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
+		assert.Equal(t, len(out), 1)
 	})
 }
 
-func (s *E2eACISuite) TestACIDeployMySQlwithEnvVars() {
+func TestRunEnvVars(t *testing.T) {
-	defer deleteResourceGroup(s.setupTestResourceGroup())
+	c := NewParallelE2eCLI(t, binDir)
+	_, _ = setupTestResourceGroup(t, c)
 
-	s.Step("runs mysql with env variables", func() {
+	t.Run("run", func(t *testing.T) {
-		err := os.Setenv("MYSQL_USER", "user1")
+		cmd := c.NewDockerCmd(
-		Expect(err).To(BeNil())
+			"run", "-d",
-		s.NewDockerCommand("run", "-d", "mysql:5.7", "-e", "MYSQL_ROOT_PASSWORD=rootpwd", "-e", "MYSQL_DATABASE=mytestdb", "-e", "MYSQL_USER", "-e", "MYSQL_PASSWORD=userpwd").ExecOrDie()
+			"-e", "MYSQL_ROOT_PASSWORD=rootpwd",
+			"-e", "MYSQL_DATABASE=mytestdb",
+			"-e", "MYSQL_USER",
+			"-e", "MYSQL_PASSWORD=userpwd",
+			"mysql:5.7",
+		)
+		cmd.Env = append(cmd.Env, "MYSQL_USER=user1")
+		res := icmd.RunCmd(cmd)
+		res.Assert(t, icmd.Success)
+		out := strings.Split(strings.TrimSpace(res.Stdout()), "\n")
+		container := strings.TrimSpace(out[len(out)-1])
+		t.Logf("Container name: %s", container)
 
-		output := s.NewDockerCommand("ps").ExecOrDie()
+		res = c.RunDockerCmd("inspect", container)
-		lines := Lines(output)
+		res.Assert(t, icmd.Success)
-		Expect(len(lines)).To(Equal(2))
 
-		containerFields := Columns(lines[1])
+		containerInspect, err := ParseContainerInspect(res.Stdout())
-		containerID := containerFields[0]
+		assert.NilError(t, err)
-		Expect(containerFields[1]).To(Equal("mysql:5.7"))
+		assert.Equal(t, containerInspect.Image, "mysql:5.7")
-		Expect(containerFields[2]).To(Equal("Running"))
 
-		errs := make(chan error)
+		check := func(t poll.LogT) poll.Result {
-		err = WaitFor(time.Second, 100*time.Second, errs, func() bool {
+			res := c.RunDockerCmd("logs", container)
-			output = s.NewDockerCommand("logs", containerID).ExecOrDie()
+			if strings.Contains(res.Stdout(), "Giving user user1 access to schema mytestdb") {
-			return strings.Contains(output, "Giving user user1 access to schema mytestdb")
+				return poll.Success()
-		})
+			}
-		Expect(err).To(BeNil())
+			return poll.Continue("waiting for DB container to be up")
-	})
+		}
-
+		poll.WaitOn(t, check, poll.WithDelay(5*time.Second), poll.WithTimeout(60*time.Second))
-	s.Step("switches back to default context", func() {
-		output := s.NewCommand("docker", "context", "use", "default").ExecOrDie()
-		Expect(output).To(ContainSubstring("default"))
-	})
-
-	s.Step("deletes test context", func() {
-		output := s.NewCommand("docker", "context", "rm", contextName).ExecOrDie()
-		Expect(output).To(ContainSubstring(contextName))
 	})
 }
 
-func (s *E2eACISuite) setupTestResourceGroup() string {
+func setupTestResourceGroup(t *testing.T, c *E2eCLI) (string, string) {
-	var resourceGroupName = randomResourceGroup()
+	startTime := strconv.Itoa(int(time.Now().UnixNano()))
-	s.Step("should be initialized with default context", s.checkDefaultContext)
+	name := "E2E-" + startTime
-	s.Step("Logs in azure using service principal credentials", azureLogin)
+	azureLogin(t)
-	s.Step("creates a new aci context for tests and use it", s.createAciContextAndUseIt(resourceGroupName))
+	sID := getSubscriptionID(t)
-	s.Step("ensures no container is running initially", s.checkNoContainnersRunning)
+	t.Logf("Create resource group %q", name)
-	return resourceGroupName
+	err := createResourceGroup(sID, name)
+	assert.Check(t, is.Nil(err))
+	t.Cleanup(func() {
+		if err := deleteResourceGroup(name); err != nil {
+			t.Error(err)
+		}
+	})
+	createAciContextAndUseIt(t, c, sID, name)
+	// Check nothing is running
+	res := c.RunDockerCmd("ps")
+	res.Assert(t, icmd.Success)
+	assert.Assert(t, is.Len(strings.Split(strings.TrimSpace(res.Stdout()), "\n"), 1))
+	return sID, name
 }
 
-func (s *E2eACISuite) checkDefaultContext() {
+func deleteResourceGroup(rgName string) error {
-	output := s.NewCommand("docker", "context", "ls").ExecOrDie()
+	ctx := context.TODO()
-	Expect(output).To(Not(ContainSubstring(contextName)))
+	helper := aci.NewACIResourceGroupHelper()
-	Expect(output).To(ContainSubstring("default *"))
+	models, err := helper.GetSubscriptionIDs(ctx)
+	if err != nil {
+		return err
+	}
+	if len(models) == 0 {
+		return errors.New("unable to delete resource group: no models")
+	}
+	return helper.DeleteAsync(ctx, *models[0].SubscriptionID, rgName)
 }
 
-func azureLogin() {
+func azureLogin(t *testing.T) {
+	t.Log("Log in to Azure")
 	login, err := login.NewAzureLoginService()
-	Expect(err).To(BeNil())
+	assert.NilError(t, err)
+
 	// in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth`
 	clientID := os.Getenv("AZURE_CLIENT_ID")
 	clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
 	tenantID := os.Getenv("AZURE_TENANT_ID")
 	err = login.TestLoginFromServicePrincipal(clientID, clientSecret, tenantID)
-	Expect(err).To(BeNil())
+	assert.NilError(t, err)
 }
 
-func (s *E2eACISuite) createAciContextAndUseIt(resourceGroupName string) func() {
+func getSubscriptionID(t *testing.T) string {
-	return func() {
+	ctx := context.TODO()
-		setupTestResourceGroup(resourceGroupName)
+	helper := aci.NewACIResourceGroupHelper()
-		helper := azure.NewACIResourceGroupHelper()
+	models, err := helper.GetSubscriptionIDs(ctx)
-		models, err := helper.GetSubscriptionIDs(context.TODO())
+	assert.Check(t, is.Nil(err))
-		Expect(err).To(BeNil())
+	assert.Check(t, len(models) == 1)
-		subscriptionID = *models[0].SubscriptionID
+	return *models[0].SubscriptionID
-
-		s.NewDockerCommand("context", "create", "aci", contextName, "--subscription-id", subscriptionID, "--resource-group", resourceGroupName, "--location", location).ExecOrDie()
-
-		currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie()
-		Expect(currentContext).To(ContainSubstring(contextName))
-		output := s.NewCommand("docker", "context", "ls").ExecOrDie()
-		Expect(output).To(ContainSubstring("acitest *"))
-	}
 }
 
-func (s *E2eACISuite) checkNoContainnersRunning() {
+func createResourceGroup(sID, rgName string) error {
-	output := s.NewDockerCommand("ps").ExecOrDie()
+	helper := aci.NewACIResourceGroupHelper()
-	Expect(len(Lines(output))).To(Equal(1))
+	_, err := helper.CreateOrUpdate(context.TODO(), sID, rgName, resources.Group{Location: to.StringPtr(location)})
+	return err
 }
 
-func randomResourceGroup() string {
+func createAciContextAndUseIt(t *testing.T, c *E2eCLI, sID, rgName string) {
-	return "resourceGroupTestE2E-" + RandStringBytes(10)
+	t.Log("Create ACI context")
+	res := c.RunDockerCmd("context", "create", "aci", contextName, "--subscription-id", sID, "--resource-group", rgName, "--location", location)
+	res.Assert(t, icmd.Success)
+	res = c.RunDockerCmd("context", "use", contextName)
+	res.Assert(t, icmd.Expected{Out: contextName})
+	res = c.RunDockerCmd("context", "ls")
+	res.Assert(t, icmd.Expected{Out: contextName + " *"})
 }
 
-func createStorageAccount(aciContext store.AciContext, accountName string) azure_storage.Account {
+func createStorageAccount(t *testing.T, aciContext store.AciContext, name string) (azure_storage.Account, func() error) {
-	log.Println("Creating storage account " + accountName)
+	t.Logf("Create storage account %q", name)
-	storageAccount, err := storage.CreateStorageAccount(context.TODO(), aciContext, accountName)
+	account, err := storage.CreateStorageAccount(context.TODO(), aciContext, name)
-	Expect(err).To(BeNil())
+	assert.Check(t, is.Nil(err))
-	Expect(*storageAccount.Name).To(Equal(accountName))
+	assert.Check(t, is.Equal(*(account.Name), name))
-	return storageAccount
+	return account, func() error { return deleteStorageAccount(aciContext, name) }
 }
 
-func getStorageKeys(aciContext store.AciContext, storageAccountName string) []azure_storage.AccountKey {
+func deleteStorageAccount(aciContext store.AciContext, name string) error {
-	list, err := storage.ListKeys(context.TODO(), aciContext, storageAccountName)
+	_, err := storage.DeleteStorageAccount(context.TODO(), aciContext, name)
-	Expect(err).To(BeNil())
+	return err
-	Expect(list.Keys).ToNot(BeNil())
-	Expect(len(*list.Keys)).To(BeNumerically(">", 0))
-
-	return *list.Keys
 }
 
-func deleteStorageAccount(aciContext store.AciContext, testStorageAccountName string) {
+func getStorageKeys(t *testing.T, aciContext store.AciContext, saName string) []azure_storage.AccountKey {
-	log.Println("Deleting storage account " + testStorageAccountName)
+	l, err := storage.ListKeys(context.TODO(), aciContext, saName)
-	_, err := storage.DeleteStorageAccount(context.TODO(), aciContext, testStorageAccountName)
+	assert.NilError(t, err)
-	Expect(err).To(BeNil())
+	assert.Assert(t, l.Keys != nil)
+	return *l.Keys
 }
 
-func createFileShare(key, shareName string, testStorageAccountName string) (azfile.SharedKeyCredential, url.URL) {
+func createFileShare(t *testing.T, key, share, storageAccount string) (*azfile.SharedKeyCredential, *url.URL) {
 	// Create a ShareURL object that wraps a soon-to-be-created share's URL and a default pipeline.
-	u, _ := url.Parse(fmt.Sprintf("https://%s.file.core.windows.net/%s", testStorageAccountName, shareName))
+	u, _ := url.Parse(fmt.Sprintf("https://%s.file.core.windows.net/%s", storageAccount, share))
-	credential, err := azfile.NewSharedKeyCredential(testStorageAccountName, key)
+	cred, err := azfile.NewSharedKeyCredential(storageAccount, key)
-	Expect(err).To(BeNil())
+	assert.NilError(t, err)
 
-	shareURL := azfile.NewShareURL(*u, azfile.NewPipeline(credential, azfile.PipelineOptions{}))
+	shareURL := azfile.NewShareURL(*u, azfile.NewPipeline(cred, azfile.PipelineOptions{}))
 	_, err = shareURL.Create(context.TODO(), azfile.Metadata{}, 0)
-	Expect(err).To(BeNil())
+	assert.NilError(t, err)
-
+	return cred, u
-	return *credential, *u
 }
 
-func uploadFile(credential azfile.SharedKeyCredential, baseURL, fileName, fileContent string) {
+func uploadFile(t *testing.T, cred azfile.SharedKeyCredential, baseURL, fileName, content string) {
 	fURL, err := url.Parse(baseURL + "/" + fileName)
-	Expect(err).To(BeNil())
+	assert.NilError(t, err)
-	fileURL := azfile.NewFileURL(*fURL, azfile.NewPipeline(&credential, azfile.PipelineOptions{}))
+	fileURL := azfile.NewFileURL(*fURL, azfile.NewPipeline(&cred, azfile.PipelineOptions{}))
-	err = azfile.UploadBufferToAzureFile(context.TODO(), []byte(fileContent), fileURL, azfile.UploadToAzureFileOptions{})
+	err = azfile.UploadBufferToAzureFile(context.TODO(), []byte(content), fileURL, azfile.UploadToAzureFileOptions{})
-	Expect(err).To(BeNil())
+	assert.NilError(t, err)
 }
 
-func TestE2eACI(t *testing.T) {
+func getContainerName(stdout string) string {
-	suite.Run(t, new(E2eACISuite))
+	out := strings.Split(strings.TrimSpace(stdout), "\n")
-}
+	return strings.TrimSpace(out[len(out)-1])
-
-func setupTestResourceGroup(resourceGroupName string) {
-	log.Println("Creating resource group " + resourceGroupName)
-	ctx := context.TODO()
-	helper := azure.NewACIResourceGroupHelper()
-	models, err := helper.GetSubscriptionIDs(ctx)
-	Expect(err).To(BeNil())
-	_, err = helper.CreateOrUpdate(ctx, *models[0].SubscriptionID, resourceGroupName, resources.Group{
-		Location: to.StringPtr(location),
-	})
-	Expect(err).To(BeNil())
-}
-
-func deleteResourceGroup(resourceGroupName string) {
-	log.Println("Deleting resource group " + resourceGroupName)
-	ctx := context.TODO()
-	helper := azure.NewACIResourceGroupHelper()
-	models, err := helper.GetSubscriptionIDs(ctx)
-	Expect(err).To(BeNil())
-	err = helper.DeleteAsync(ctx, *models[0].SubscriptionID, resourceGroupName)
-	Expect(err).To(BeNil())
-}
-
-func RandStringBytes(n int) string {
-	rand.Seed(time.Now().UnixNano())
-	const digits = "0123456789"
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = digits[rand.Intn(len(digits))]
-	}
-	return string(b)
 }

tests/e2e/e2e_test.go

@@ -17,293 +17,444 @@
 package main
 
 import (
+	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
-	. "github.com/onsi/gomega"
+	"gotest.tools/v3/assert"
-	"github.com/stretchr/testify/suite"
+	"gotest.tools/v3/golden"
-	"gotest.tools/golden"
+	"gotest.tools/v3/icmd"
 
 	. "github.com/docker/api/tests/framework"
 )
 
-type E2eSuite struct {
+var binDir string
-	Suite
-}
 
-func (s *E2eSuite) TestContextHelp() {
+func TestMain(m *testing.M) {
-	output := s.NewDockerCommand("context", "create", "aci", "--help").ExecOrDie()
+	p, cleanup, err := SetupExistingCLI()
-	Expect(output).To(ContainSubstring("docker context create aci CONTEXT [flags]"))
+	if err != nil {
-	Expect(output).To(ContainSubstring("--location"))
+		fmt.Println(err)
-	Expect(output).To(ContainSubstring("--subscription-id"))
+		os.Exit(1)
-	Expect(output).To(ContainSubstring("--resource-group"))
-}
-
-func (s *E2eSuite) TestListAndShowDefaultContext() {
-	output := s.NewDockerCommand("context", "show").ExecOrDie()
-	Expect(output).To(ContainSubstring("default"))
-	output = s.NewCommand("docker", "context", "ls").ExecOrDie()
-	golden.Assert(s.T(), output, GoldenFile("ls-out-default"))
-}
-
-func (s *E2eSuite) TestCreateDockerContextAndListIt() {
-	s.NewDockerCommand("context", "create", "test-docker", "--from", "default").ExecOrDie()
-	output := s.NewCommand("docker", "context", "ls").ExecOrDie()
-	golden.Assert(s.T(), output, GoldenFile("ls-out-test-docker"))
-}
-
-func (s *E2eSuite) TestContextListQuiet() {
-	s.NewDockerCommand("context", "create", "test-docker", "--from", "default").ExecOrDie()
-	output := s.NewCommand("docker", "context", "ls", "-q").ExecOrDie()
-	Expect(output).To(Equal(`default
-test-docker
-`))
-}
-
-func (s *E2eSuite) TestInspectDefaultContext() {
-	output := s.NewDockerCommand("context", "inspect", "default").ExecOrDie()
-	Expect(output).To(ContainSubstring(`"Name": "default"`))
-}
-
-func (s *E2eSuite) TestInspectContextNoArgs() {
-	output := s.NewDockerCommand("context", "inspect").ExecOrDie()
-	Expect(output).To(ContainSubstring(`"Name": "default"`))
-}
-
-func (s *E2eSuite) TestInspectContextRegardlessCurrentContext() {
-	s.NewDockerCommand("context", "create", "local", "localCtx").ExecOrDie()
-	s.NewDockerCommand("context", "use", "localCtx").ExecOrDie()
-	output := s.NewDockerCommand("context", "inspect").ExecOrDie()
-	Expect(output).To(ContainSubstring(`"Name": "localCtx"`))
-}
-
-func (s *E2eSuite) TestContextLsFormat() {
-	output, err := s.NewDockerCommand("context", "ls", "--format", "{{ json . }}").Exec()
-	Expect(err).To(BeNil())
-	Expect(output).To(ContainSubstring(`"Name":"default"`))
-}
-
-func (s *E2eSuite) TestComposeOnDefaultContext() {
-	s.NewDockerCommand("context", "use", "default").ExecOrDie()
-	output := s.NewDockerCommand("context", "inspect").ExecOrDie()
-	Expect(output).To(ContainSubstring(`"Name": "default"`))
-	output, err := s.NewDockerCommand("compose", "up").Exec()
-	Expect(err).NotTo(BeNil())
-	Expect(output).To(ContainSubstring(`compose command not supported on context type`))
-}
-
-func (s *E2eSuite) TestContextCreateParseErrorDoesNotDelegateToLegacy() {
-	s.Step("should dispay new cli error when parsing context create flags", func() {
-		_, err := s.NewDockerCommand("context", "create", "aci", "--subscription-id", "titi").Exec()
-		Expect(err.Error()).NotTo(ContainSubstring("unknown flag"))
-		Expect(err.Error()).To(ContainSubstring("accepts 1 arg(s), received 0"))
-	})
-}
-
-func (s *E2eSuite) TestCannotRemoveCurrentContext() {
-	s.NewDockerCommand("context", "create", "test-context-rm", "--from", "default").ExecOrDie()
-	s.NewDockerCommand("context", "use", "test-context-rm").ExecOrDie()
-	_, err := s.NewDockerCommand("context", "rm", "test-context-rm").Exec()
-	Expect(err.Error()).To(ContainSubstring("cannot delete current context"))
-}
-
-func (s *E2eSuite) TestCanForceRemoveCurrentContext() {
-	s.NewDockerCommand("context", "create", "test-context-rmf", "--from", "default").ExecOrDie()
-	s.NewDockerCommand("context", "use", "test-context-rmf").ExecOrDie()
-	s.NewDockerCommand("context", "rm", "-f", "test-context-rmf").ExecOrDie()
-	out := s.NewDockerCommand("context", "ls").ExecOrDie()
-	Expect(out).To(ContainSubstring("default *"))
-}
-
-func (s *E2eSuite) TestContextCreateAciChecksContextNameBeforeInteractivePart() {
-	s.NewDockerCommand("context", "create", "mycontext", "--from", "default").ExecOrDie()
-	_, err := s.NewDockerCommand("context", "create", "aci", "mycontext").Exec()
-	Expect(err.Error()).To(ContainSubstring("context mycontext: already exists"))
-}
-
-func (s *E2eSuite) TestClassicLoginWithparameters() {
-	output, err := s.NewDockerCommand("login", "-u", "nouser", "-p", "wrongpasword").Exec()
-	Expect(output).To(ContainSubstring("Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password"))
-	Expect(err).NotTo(BeNil())
-}
-
-func (s *E2eSuite) TestClassicLoginRegardlessCurrentContext() {
-	s.NewDockerCommand("context", "create", "local", "localCtx").ExecOrDie()
-	s.NewDockerCommand("context", "use", "localCtx").ExecOrDie()
-	output, err := s.NewDockerCommand("login", "-u", "nouser", "-p", "wrongpasword").Exec()
-	Expect(output).To(ContainSubstring("Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password"))
-	Expect(err).NotTo(BeNil())
-}
-
-func (s *E2eSuite) TestClassicLogin() {
-	output, err := s.NewDockerCommand("login", "someregistry.docker.io").Exec()
-	Expect(output).To(ContainSubstring("Cannot perform an interactive login from a non TTY device"))
-	Expect(err).NotTo(BeNil())
-	output, err = s.NewDockerCommand("logout", "someregistry.docker.io").Exec()
-	Expect(output).To(ContainSubstring("someregistry.docker.io"))
-	Expect(err).To(BeNil())
-}
-
-func (s *E2eSuite) TestCloudLogin() {
-	output, err := s.NewDockerCommand("login", "mycloudbackend").Exec()
-	Expect(output).To(ContainSubstring("unknown backend type for cloud login: mycloudbackend"))
-	Expect(err).NotTo(BeNil())
-}
-
-func (s *E2eSuite) TestSetupError() {
-	s.Step("should display an error if cannot shell out to com.docker.cli", func() {
-		err := os.Setenv("PATH", s.BinDir)
-		Expect(err).To(BeNil())
-		err = os.Remove(filepath.Join(s.BinDir, DockerClassicExecutable()))
-		Expect(err).To(BeNil())
-		output, err := s.NewDockerCommand("ps").Exec()
-		Expect(output).To(ContainSubstring("com.docker.cli"))
-		Expect(output).To(ContainSubstring("not found"))
-		Expect(err).NotTo(BeNil())
-	})
-}
-
-func (s *E2eSuite) TestLegacy() {
-	s.Step("should list all legacy commands", func() {
-		output := s.NewDockerCommand("--help").ExecOrDie()
-		Expect(output).To(ContainSubstring("swarm"))
-	})
-
-	s.Step("should execute legacy commands", func() {
-		output, _ := s.NewDockerCommand("swarm", "join").Exec()
-		Expect(output).To(ContainSubstring("\"docker swarm join\" requires exactly 1 argument."))
-	})
-
-	s.Step("should run local container in less than 10 secs", func() {
-		s.NewDockerCommand("pull", "hello-world").ExecOrDie()
-		output := s.NewDockerCommand("run", "--rm", "hello-world").WithTimeout(time.NewTimer(20 * time.Second).C).ExecOrDie()
-		Expect(output).To(ContainSubstring("Hello from Docker!"))
-	})
-
-	s.Step("should execute legacy commands in other moby contexts", func() {
-		s.NewDockerCommand("context", "create", "mobyCtx", "--from=default").ExecOrDie()
-		s.NewDockerCommand("context", "use", "mobyCtx").ExecOrDie()
-		output, _ := s.NewDockerCommand("swarm", "join").Exec()
-		Expect(output).To(ContainSubstring("\"docker swarm join\" requires exactly 1 argument."))
-	})
-}
-
-func (s *E2eSuite) TestLeaveLegacyErrorMessagesUnchanged() {
-	output, err := s.NewDockerCommand("foo").Exec()
-	golden.Assert(s.T(), output, "unknown-foo-command.golden")
-	Expect(err).NotTo(BeNil())
-}
-
-func (s *E2eSuite) TestPassThroughRootLegacyFlags() {
-	output, err := s.NewDockerCommand("-H", "tcp://localhost:123", "version").Exec()
-	Expect(err).NotTo(BeNil())
-	Expect(output).NotTo(ContainSubstring("unknown shorthand flag"))
-	Expect(output).To(ContainSubstring("localhost:123"))
-
-	output, _ = s.NewDockerCommand("-H", "tcp://localhost:123", "login", "-u", "nouser", "-p", "wrongpasword").Exec()
-	Expect(output).NotTo(ContainSubstring("unknown shorthand flag"))
-	Expect(output).To(ContainSubstring("WARNING! Using --password via the CLI is insecure"))
-
-	output, _ = s.NewDockerCommand("--log-level", "debug", "login", "-u", "nouser", "-p", "wrongpasword").Exec()
-	Expect(output).NotTo(ContainSubstring("unknown shorthand flag"))
-	Expect(output).To(ContainSubstring("WARNING! Using --password via the CLI is insecure"))
-
-	output, _ = s.NewDockerCommand("login", "--help").Exec()
-	Expect(output).NotTo(ContainSubstring("--log-level"))
-}
-
-func (s *E2eSuite) TestDisplayFriendlyErrorMessageForLegacyCommands() {
-	s.NewDockerCommand("context", "create", "example", "test-example").ExecOrDie()
-	output, err := s.NewDockerCommand("--context", "test-example", "images").Exec()
-	Expect(output).To(Equal("Command \"images\" not available in current context (test-example), you can use the \"default\" context to run this command\n"))
-	Expect(err).NotTo(BeNil())
-}
-
-func (s *E2eSuite) TestExecMobyIfUsingHostFlag() {
-	s.NewDockerCommand("context", "create", "example", "test-example").ExecOrDie()
-	s.NewDockerCommand("context", "use", "test-example").ExecOrDie()
-	output, err := s.NewDockerCommand("-H", defaultEndpoint(), "ps").Exec()
-	Expect(err).To(BeNil())
-	Expect(output).To(ContainSubstring("CONTAINER ID"))
-}
-
-func defaultEndpoint() string {
-	if runtime.GOOS == "windows" {
-		return "npipe:////./pipe/docker_engine"
 	}
-	return "unix:///var/run/docker.sock"
+	binDir = p
+	exitCode := m.Run()
+	cleanup()
+	os.Exit(exitCode)
 }
 
-func (s *E2eSuite) TestExecMobyIfUsingversionFlag() {
+func TestComposeNotImplemented(t *testing.T) {
-	s.NewDockerCommand("context", "create", "example", "test-example").ExecOrDie()
+	c := NewParallelE2eCLI(t, binDir)
-	s.NewDockerCommand("context", "use", "test-example").ExecOrDie()
+	res := c.RunDockerCmd("context", "show")
-	output, err := s.NewDockerCommand("-v").Exec()
+	res.Assert(t, icmd.Expected{Out: "default"})
-	Expect(err).To(BeNil())
+	res = c.RunDockerCmd("compose", "up")
-	Expect(output).To(ContainSubstring("Docker version"))
+	res.Assert(t, icmd.Expected{
-}
+		ExitCode: 1,
-
+		Err:      `compose command not supported on context type "moby": not implemented`,
-func (s *E2eSuite) TestDisplaysAdditionalLineInDockerVersion() {
-	output := s.NewDockerCommand("version").ExecOrDie()
-	Expect(output).To(ContainSubstring("Azure integration"))
-}
-
-func (s *E2eSuite) TestAllowsFormatFlagInVersion() {
-	s.NewDockerCommand("version", "-f", "{{ json . }}").ExecOrDie()
-	s.NewDockerCommand("version", "--format", "{{ json . }}").ExecOrDie()
-}
-
-func (s *E2eSuite) TestMockBackend() {
-	s.Step("creates a new test context to hardcoded example backend", func() {
-		s.NewDockerCommand("context", "create", "example", "test-example").ExecOrDie()
-		// Expect(output).To(ContainSubstring("test-example context acitest created"))
-	})
-
-	s.Step("uses the test context", func() {
-		currentContext := s.NewDockerCommand("context", "use", "test-example").ExecOrDie()
-		Expect(currentContext).To(ContainSubstring("test-example"))
-		output := s.NewDockerCommand("context", "ls").ExecOrDie()
-		golden.Assert(s.T(), output, GoldenFile("ls-out-test-example"))
-		output = s.NewDockerCommand("context", "show").ExecOrDie()
-		Expect(output).To(ContainSubstring("test-example"))
-	})
-
-	s.Step("can run ps command", func() {
-		output := s.NewDockerCommand("ps").ExecOrDie()
-		lines := Lines(output)
-		Expect(len(lines)).To(Equal(3))
-		Expect(lines[2]).To(ContainSubstring("1234                alpine"))
-	})
-
-	s.Step("can run quiet ps command", func() {
-		output := s.NewDockerCommand("ps", "-q").ExecOrDie()
-		lines := Lines(output)
-		Expect(len(lines)).To(Equal(2))
-		Expect(lines[0]).To(Equal("id"))
-		Expect(lines[1]).To(Equal("1234"))
-	})
-
-	s.Step("can run ps command with all ", func() {
-		output := s.NewDockerCommand("ps", "-q", "--all").ExecOrDie()
-		lines := Lines(output)
-		Expect(len(lines)).To(Equal(3))
-		Expect(lines[0]).To(Equal("id"))
-		Expect(lines[1]).To(Equal("1234"))
-		Expect(lines[2]).To(Equal("stopped"))
-	})
-
-	s.Step("can run inspect command on container", func() {
-		golden.Assert(s.T(), s.NewDockerCommand("inspect", "id").ExecOrDie(), "inspect-id.golden")
-	})
-
-	s.Step("can run 'run' command", func() {
-		output := s.NewDockerCommand("run", "-d", "nginx", "-p", "80:80").ExecOrDie()
-		Expect(output).To(ContainSubstring("Running container \"nginx\" with name"))
 	})
 }
 
-func TestE2e(t *testing.T) {
+func TestContextDefault(t *testing.T) {
-	suite.Run(t, new(E2eSuite))
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("show", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "show")
+		res.Assert(t, icmd.Expected{Out: "default"})
+	})
+
+	t.Run("ls", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "ls")
+		res.Assert(t, icmd.Success)
+		golden.Assert(t, res.Stdout(), GoldenFile("ls-out-default"))
+	})
+
+	t.Run("inspect", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "inspect", "default")
+		res.Assert(t, icmd.Expected{Out: `"Name": "default"`})
+	})
+
+	t.Run("inspect current", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "inspect")
+		res.Assert(t, icmd.Expected{Out: `"Name": "default"`})
+	})
+}
+
+func TestContextCreateDocker(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+	res := c.RunDockerCmd("context", "create", "test-docker", "--from", "default")
+	res.Assert(t, icmd.Expected{Out: "test-docker"})
+
+	t.Run("ls", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "ls")
+		res.Assert(t, icmd.Success)
+		golden.Assert(t, res.Stdout(), GoldenFile("ls-out-test-docker"))
+	})
+
+	t.Run("ls quiet", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "ls", "-q")
+		golden.Assert(t, res.Stdout(), "ls-out-test-docker-quiet.golden")
+	})
+
+	t.Run("ls format", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "ls", "--format", "{{ json . }}")
+		res.Assert(t, icmd.Expected{Out: `"Name":"default"`})
+	})
+}
+
+func TestContextInspect(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+	res := c.RunDockerCmd("context", "create", "test-docker", "--from", "default")
+	res.Assert(t, icmd.Expected{Out: "test-docker"})
+
+	t.Run("inspect current", func(t *testing.T) {
+		// Cannot be run in parallel because of "context use"
+		res := c.RunDockerCmd("context", "use", "test-docker")
+		res.Assert(t, icmd.Expected{Out: "test-docker"})
+
+		res = c.RunDockerCmd("context", "inspect")
+		res.Assert(t, icmd.Expected{Out: `"Name": "test-docker"`})
+	})
+}
+
+func TestContextHelpACI(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("help", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "create", "aci", "--help")
+		// Can't use golden here as the help prints the config directory which changes
+		res.Assert(t, icmd.Expected{Out: "docker context create aci CONTEXT [flags]"})
+		res.Assert(t, icmd.Expected{Out: "--location"})
+		res.Assert(t, icmd.Expected{Out: "--subscription-id"})
+		res.Assert(t, icmd.Expected{Out: "--resource-group"})
+	})
+
+	t.Run("check exec", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "create", "aci", "--subscription-id", "invalid-id")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "accepts 1 arg(s), received 0",
+		})
+		assert.Assert(t, !strings.Contains(res.Combined(), "unknown flag"))
+	})
+}
+
+func TestContextDuplicateACI(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	c.RunDockerCmd("context", "create", "mycontext", "--from", "default").Assert(t, icmd.Success)
+	res := c.RunDockerCmd("context", "create", "aci", "mycontext")
+	res.Assert(t, icmd.Expected{
+		ExitCode: 1,
+		Err:      "context mycontext: already exists",
+	})
+}
+
+func TestContextRemove(t *testing.T) {
+
+	t.Run("remove current", func(t *testing.T) {
+		c := NewParallelE2eCLI(t, binDir)
+
+		c.RunDockerCmd("context", "create", "test-context-rm", "--from", "default").Assert(t, icmd.Success)
+		res := c.RunDockerCmd("context", "use", "test-context-rm")
+		res.Assert(t, icmd.Expected{Out: "test-context-rm"})
+		res = c.RunDockerCmd("context", "rm", "test-context-rm")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "cannot delete current context",
+		})
+	})
+
+	t.Run("force remove current", func(t *testing.T) {
+		c := NewParallelE2eCLI(t, binDir)
+
+		c.RunDockerCmd("context", "create", "test-context-rmf").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "use", "test-context-rmf").Assert(t, icmd.Success)
+		res := c.RunDockerCmd("context", "rm", "-f", "test-context-rmf")
+		res.Assert(t, icmd.Expected{Out: "test-context-rmf"})
+		res = c.RunDockerCmd("context", "ls")
+		res.Assert(t, icmd.Expected{Out: "default *"})
+	})
+}
+
+func TestLoginCommandDelegation(t *testing.T) {
+	// These tests just check that the existing CLI is called in various cases.
+	// They do not test actual login functionality.
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("default context", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("login", "-u", "nouser", "-p", "wrongpasword")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password",
+		})
+	})
+
+	t.Run("interactive", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("login", "someregistry.docker.io")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "Cannot perform an interactive login from a non TTY device",
+		})
+	})
+
+	t.Run("logout", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("logout", "someregistry.docker.io")
+		res.Assert(t, icmd.Expected{Out: "someregistry.docker.io"})
+	})
+
+	t.Run("existing context", func(t *testing.T) {
+		c := NewParallelE2eCLI(t, binDir)
+		c.RunDockerCmd("context", "create", "local", "local").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "use", "local").Assert(t, icmd.Success)
+		res := c.RunDockerCmd("login", "-u", "nouser", "-p", "wrongpasword")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password",
+		})
+	})
+}
+
+func TestCloudLogin(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("unknown backend", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("login", "mycloudbackend")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "unknown backend type for cloud login: mycloudbackend",
+		})
+	})
+}
+
+func TestMissingExistingCLI(t *testing.T) {
+	t.Parallel()
+	home, err := ioutil.TempDir("", "")
+	assert.NilError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(home)
+	})
+
+	bin, err := ioutil.TempDir("", "")
+	assert.NilError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(bin)
+	})
+	err = CopyFile(filepath.Join(binDir, DockerExecutableName), filepath.Join(bin, DockerExecutableName))
+	assert.NilError(t, err)
+
+	c := icmd.Cmd{
+		Env:     []string{"HOME=" + home, "PATH=" + bin},
+		Command: []string{filepath.Join(bin, "docker")},
+	}
+	res := icmd.RunCmd(c)
+	res.Assert(t, icmd.Expected{
+		ExitCode: 1,
+		Err:      `"com.docker.cli": executable file not found`,
+	})
+}
+
+func TestLegacy(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("help", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("--help")
+		res.Assert(t, icmd.Expected{Out: "swarm"})
+	})
+
+	t.Run("swarm", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("swarm", "join")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      `"docker swarm join" requires exactly 1 argument.`,
+		})
+	})
+
+	t.Run("local run", func(t *testing.T) {
+		t.Parallel()
+		cmd := c.NewDockerCmd("run", "--rm", "hello-world")
+		cmd.Timeout = 20 * time.Second
+		res := icmd.RunCmd(cmd)
+		res.Assert(t, icmd.Expected{Out: "Hello from Docker!"})
+	})
+
+	t.Run("error messages", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("foo")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "docker: 'foo' is not a docker command.",
+		})
+	})
+
+	t.Run("host flag", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("-H", "tcp://localhost:123", "version")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "Cannot connect to the Docker daemon at tcp://localhost:123",
+		})
+	})
+
+	t.Run("existing contexts delegate", func(t *testing.T) {
+		c := NewParallelE2eCLI(t, binDir)
+		c.RunDockerCmd("context", "create", "moby-ctx", "--from=default").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "use", "moby-ctx").Assert(t, icmd.Success)
+		res := c.RunDockerCmd("swarm", "join")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      `"docker swarm join" requires exactly 1 argument.`,
+		})
+	})
+
+	t.Run("host flag overrides context", func(t *testing.T) {
+		c := NewParallelE2eCLI(t, binDir)
+		c.RunDockerCmd("context", "create", "example", "test-example").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "use", "test-example").Assert(t, icmd.Success)
+		endpoint := "unix:///var/run/docker.sock"
+		if runtime.GOOS == "windows" {
+			endpoint = "npipe:////./pipe/docker_engine"
+		}
+		res := c.RunDockerCmd("-H", endpoint, "ps")
+		res.Assert(t, icmd.Success)
+		// Example backend's ps output includes these strings
+		assert.Assert(t, !strings.Contains(res.Stdout(), "id"))
+		assert.Assert(t, !strings.Contains(res.Stdout(), "1234"))
+	})
+}
+
+func TestLegacyLogin(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("host flag login", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("-H", "tcp://localhost:123", "login", "-u", "nouser", "-p", "wrongpasword")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "WARNING! Using --password via the CLI is insecure. Use --password-stdin.",
+		})
+	})
+
+	t.Run("log level flag login", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("--log-level", "debug", "login", "-u", "nouser", "-p", "wrongpasword")
+		res.Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      "WARNING! Using --password via the CLI is insecure",
+		})
+	})
+
+	t.Run("login help global flags", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("login", "--help")
+		res.Assert(t, icmd.Success)
+		assert.Assert(t, !strings.Contains(res.Combined(), "--log-level"))
+	})
+}
+
+func TestUnsupportedCommand(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	res := c.RunDockerCmd("context", "create", "example", "test-example")
+	res.Assert(t, icmd.Success)
+	res = c.RunDockerCmd("--context", "test-example", "images")
+	res.Assert(t, icmd.Expected{
+		ExitCode: 1,
+		Err:      `Command "images" not available in current context (test-example), you can use the "default" context to run this command`,
+	})
+}
+
+func TestVersion(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+
+	t.Run("azure version", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("version")
+		res.Assert(t, icmd.Expected{Out: "Azure integration"})
+	})
+
+	t.Run("format", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("version", "-f", "{{ json . }}")
+		res.Assert(t, icmd.Expected{Out: `"Client":`})
+		res = c.RunDockerCmd("version", "--format", "{{ json . }}")
+		res.Assert(t, icmd.Expected{Out: `"Client":`})
+	})
+
+	t.Run("delegate version flag", func(t *testing.T) {
+		c := NewParallelE2eCLI(t, binDir)
+		c.RunDockerCmd("context", "create", "example", "test-example").Assert(t, icmd.Success)
+		c.RunDockerCmd("context", "use", "test-example").Assert(t, icmd.Success)
+		res := c.RunDockerCmd("-v")
+		res.Assert(t, icmd.Expected{Out: "Docker version"})
+	})
+}
+
+func TestMockBackend(t *testing.T) {
+	c := NewParallelE2eCLI(t, binDir)
+	c.RunDockerCmd("context", "create", "example", "test-example").Assert(t, icmd.Success)
+	res := c.RunDockerCmd("context", "use", "test-example")
+	res.Assert(t, icmd.Expected{Out: "test-example"})
+
+	t.Run("use", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("context", "show")
+		res.Assert(t, icmd.Expected{Out: "test-example"})
+		res = c.RunDockerCmd("context", "ls")
+		golden.Assert(t, res.Stdout(), GoldenFile("ls-out-test-example"))
+	})
+
+	t.Run("ps", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("ps")
+		res.Assert(t, icmd.Success)
+		golden.Assert(t, res.Stdout(), "ps-out-example.golden")
+	})
+
+	t.Run("ps quiet", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("ps", "-q")
+		res.Assert(t, icmd.Success)
+		golden.Assert(t, res.Stdout(), "ps-quiet-out-example.golden")
+	})
+
+	t.Run("ps quiet all", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("ps", "-q", "--all")
+		res.Assert(t, icmd.Success)
+		golden.Assert(t, res.Stdout(), "ps-quiet-all-out-example.golden")
+	})
+
+	t.Run("inspect", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("inspect", "id")
+		res.Assert(t, icmd.Success)
+		golden.Assert(t, res.Stdout(), "inspect-id.golden")
+	})
+
+	t.Run("run", func(t *testing.T) {
+		t.Parallel()
+		res := c.RunDockerCmd("run", "-d", "nginx", "-p", "80:80")
+		res.Assert(t, icmd.Expected{
+			Out: `Running container "nginx" with name`,
+		})
+	})
 }

tests/e2e/testdata/ls-out-test-docker-quiet.golden

@@ -0,0 +1,2 @@
+default
+test-docker

tests/e2e/testdata/ps-out-example.golden

@@ -0,0 +1,3 @@
+CONTAINER ID        IMAGE               COMMAND             STATUS              PORTS
+id                  nginx                                                       
+1234                alpine                                                      

tests/e2e/testdata/ps-quiet-all-out-example.golden

@@ -0,0 +1,3 @@
+id
+1234
+stopped

tests/e2e/testdata/ps-quiet-out-example.golden

@@ -0,0 +1,2 @@
+id
+1234

tests/e2e/testdata/unknown-foo-command.golden

@@ -1,2 +0,0 @@
-docker: 'foo' is not a docker command.
-See 'docker --help'

tests/framework/e2e.go

@@ -0,0 +1,183 @@
+/*
+   Copyright 2020 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/icmd"
+
+	"github.com/docker/api/containers"
+)
+
+var (
+	// DockerExecutableName is the OS dependent Docker CLI binary name
+	DockerExecutableName   = "docker"
+	existingExectuableName = "com.docker.cli"
+)
+
+func init() {
+	if runtime.GOOS == "windows" {
+		DockerExecutableName = DockerExecutableName + ".exe"
+		existingExectuableName = existingExectuableName + ".exe"
+	}
+}
+
+// E2eCLI is used to wrap the CLI for end to end testing
+type E2eCLI struct {
+	BinDir    string
+	ConfigDir string
+}
+
+// NewParallelE2eCLI returns a configured TestE2eCLI with t.Parallel() set
+func NewParallelE2eCLI(t *testing.T, binDir string) *E2eCLI {
+	t.Parallel()
+	return newE2eCLI(t, binDir)
+}
+
+// NewE2eCLI returns a configured TestE2eCLI
+func NewE2eCLI(t *testing.T, binDir string) *E2eCLI {
+	return newE2eCLI(t, binDir)
+}
+
+func newE2eCLI(t *testing.T, binDir string) *E2eCLI {
+	d, err := ioutil.TempDir("", "")
+	assert.Check(t, is.Nil(err))
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			conf, _ := ioutil.ReadFile(filepath.Join(d, "config.json"))
+			t.Errorf("Config: %s\n", string(conf))
+			t.Error("Contents of config dir:")
+			for _, p := range dirContents(d) {
+				t.Errorf(p)
+			}
+		}
+		_ = os.RemoveAll(d)
+	})
+
+	return &E2eCLI{binDir, d}
+}
+
+func dirContents(dir string) []string {
+	res := []string{}
+	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		res = append(res, filepath.Join(dir, path))
+		return nil
+	})
+	return res
+}
+
+// SetupExistingCLI copies the existing CLI in a temporary directory so that the
+// new CLI can be configured to use it
+func SetupExistingCLI() (string, func(), error) {
+	p, err := exec.LookPath(existingExectuableName)
+	if err != nil {
+		p, err = exec.LookPath(DockerExecutableName)
+		if err != nil {
+			return "", nil, errors.New("existing CLI not found in PATH")
+		}
+	}
+	d, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", nil, err
+	}
+	if err := CopyFile(p, filepath.Join(d, existingExectuableName)); err != nil {
+		return "", nil, err
+	}
+	bin, err := filepath.Abs("../../bin/" + DockerExecutableName)
+	if err != nil {
+		return "", nil, err
+	}
+	if err := CopyFile(bin, filepath.Join(d, DockerExecutableName)); err != nil {
+		return "", nil, err
+	}
+	cleanup := func() {
+		_ = os.RemoveAll(d)
+	}
+	return d, cleanup, nil
+}
+
+// CopyFile copies a file from a path to a path setting permissions to 0777
+func CopyFile(sourceFile string, destinationFile string) error {
+	input, err := ioutil.ReadFile(sourceFile)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(destinationFile, input, 0777)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// NewCmd creates a cmd object configured with the test environment set
+func (c *E2eCLI) NewCmd(command string, args ...string) icmd.Cmd {
+	path := c.BinDir + ":" + os.Getenv("PATH")
+	if runtime.GOOS == "windows" {
+		path = c.BinDir + ";" + os.Getenv("PATH")
+	}
+	env := append(os.Environ(),
+		"DOCKER_CONFIG="+c.ConfigDir,
+		"KUBECONFIG=invalid",
+		"PATH="+path,
+	)
+	return icmd.Cmd{
+		Command: append([]string{command}, args...),
+		Env:     env,
+	}
+}
+
+// NewDockerCmd creates a docker cmd without running it
+func (c *E2eCLI) NewDockerCmd(args ...string) icmd.Cmd {
+	return c.NewCmd(filepath.Join(c.BinDir, DockerExecutableName), args...)
+}
+
+// RunDockerCmd runs a docker command and returns a result
+func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
+	return icmd.RunCmd(c.NewDockerCmd(args...))
+}
+
+// GoldenFile golden file specific to platform
+func GoldenFile(name string) string {
+	if runtime.GOOS == "windows" {
+		return name + "-windows.golden"
+	}
+	return name + ".golden"
+}
+
+// ParseContainerInspect parses the output of a `docker inspect` command for a
+// container
+func ParseContainerInspect(stdout string) (*containers.Container, error) {
+	var res containers.Container
+	rdr := bytes.NewReader([]byte(stdout))
+	if err := json.NewDecoder(rdr).Decode(&res); err != nil {
+		return nil, err
+	}
+	return &res, nil
+}

tests/framework/exec.go

@@ -1,203 +0,0 @@
-/*
-   Copyright 2020 Docker, Inc.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package framework
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os/exec"
-	"runtime"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func (b CmdContext) makeCmd() *exec.Cmd {
-	return exec.Command(b.command, b.args...)
-}
-
-// CmdContext is used to build, customize and execute a command.
-// Add more functions to customize the context as needed.
-type CmdContext struct {
-	command string
-	args    []string
-	envs    []string
-	dir     string
-	stdin   io.Reader
-	timeout <-chan time.Time
-	retries RetriesContext
-}
-
-// RetriesContext is used to tweak retry loop.
-type RetriesContext struct {
-	count    int
-	interval time.Duration
-}
-
-// WithinDirectory tells Docker the cwd.
-func (b *CmdContext) WithinDirectory(path string) *CmdContext {
-	b.dir = path
-	return b
-}
-
-// WithEnvs set envs in context.
-func (b *CmdContext) WithEnvs(envs []string) *CmdContext {
-	b.envs = envs
-	return b
-}
-
-// WithTimeout controls maximum duration.
-func (b *CmdContext) WithTimeout(t <-chan time.Time) *CmdContext {
-	b.timeout = t
-	return b
-}
-
-// WithRetries sets how many times to retry the command before issuing an error
-func (b *CmdContext) WithRetries(count int) *CmdContext {
-	b.retries.count = count
-	return b
-}
-
-// Every interval between 2 retries
-func (b *CmdContext) Every(interval time.Duration) *CmdContext {
-	b.retries.interval = interval
-	return b
-}
-
-// WithStdinData feeds via stdin.
-func (b CmdContext) WithStdinData(data string) *CmdContext {
-	b.stdin = strings.NewReader(data)
-	return &b
-}
-
-// WithStdinReader feeds via stdin.
-func (b CmdContext) WithStdinReader(reader io.Reader) *CmdContext {
-	b.stdin = reader
-	return &b
-}
-
-// ExecOrDie runs a docker command.
-func (b CmdContext) ExecOrDie() string {
-	str, err := b.Exec()
-	logrus.Debugf("stdout: %s", str)
-	gomega.Expect(err).NotTo(gomega.HaveOccurred())
-	return str
-}
-
-// Exec runs a docker command.
-func (b CmdContext) Exec() (string, error) {
-	retry := b.retries.count
-	for ; ; retry-- {
-		cmd := b.makeCmd()
-		cmd.Dir = b.dir
-		cmd.Stdin = b.stdin
-		if b.envs != nil {
-			cmd.Env = b.envs
-		}
-		stdout, err := Execute(cmd, b.timeout)
-		if err == nil || retry < 1 {
-			return stdout, err
-		}
-		time.Sleep(b.retries.interval)
-	}
-}
-
-//WaitFor waits for a condition to be true
-func WaitFor(interval, duration time.Duration, abort <-chan error, condition func() bool) error {
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
-	timeout := make(chan int)
-	go func() {
-		time.Sleep(duration)
-		close(timeout)
-	}()
-	for {
-		select {
-		case err := <-abort:
-			return err
-		case <-timeout:
-			return fmt.Errorf("timeout after %v", duration)
-		case <-ticker.C:
-			if condition() {
-				return nil
-			}
-		}
-	}
-}
-
-// Execute executes a command.
-// The command cannot be re-used afterwards.
-func Execute(cmd *exec.Cmd, timeout <-chan time.Time) (string, error) {
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = mergeWriter(cmd.Stdout, &stdout)
-	cmd.Stderr = mergeWriter(cmd.Stderr, &stderr)
-
-	logrus.Infof("Execute '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately
-	if err := cmd.Start(); err != nil {
-		return "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, stdout.String(), stderr.String(), err)
-	}
-	errCh := make(chan error, 1)
-	go func() {
-		errCh <- cmd.Wait()
-	}()
-	select {
-	case err := <-errCh:
-		if err != nil {
-			logrus.Debugf("%s %s failed: %v", cmd.Path, strings.Join(cmd.Args[1:], " "), err)
-			return stderr.String(), fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, stdout.String(), stderr.String(), err)
-		}
-	case <-timeout:
-		logrus.Debugf("%s %s timed-out", cmd.Path, strings.Join(cmd.Args[1:], " "))
-		if err := terminateProcess(cmd); err != nil {
-			return "", err
-		}
-		return stdout.String(), fmt.Errorf(
-			"timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v",
-			cmd.Args, stdout.String(), stderr.String())
-	}
-	if stderr.String() != "" {
-		logrus.Debugf("stderr: %s", stderr.String())
-	}
-	return stdout.String(), nil
-}
-
-func terminateProcess(cmd *exec.Cmd) error {
-	if runtime.GOOS == "windows" {
-		return cmd.Process.Kill()
-	}
-	return cmd.Process.Signal(syscall.SIGTERM)
-}
-
-func mergeWriter(other io.Writer, buf io.Writer) io.Writer {
-	if other != nil {
-		return io.MultiWriter(other, buf)
-	}
-	return buf
-}
-
-// Powershell runs a powershell command.
-func Powershell(input string) (string, error) {
-	output, err := Execute(exec.Command("powershell", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Unrestricted", "-Command", input), nil)
-	if err != nil {
-		return "", fmt.Errorf("fail to execute %s: %s", input, err)
-	}
-	return strings.TrimSpace(output), nil
-}

tests/framework/helper.go

@@ -1,51 +0,0 @@
-/*
-   Copyright 2020 Docker, Inc.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package framework
-
-import (
-	"runtime"
-	"strings"
-
-	"github.com/robpike/filter"
-)
-
-func nonEmptyString(s string) bool {
-	return strings.TrimSpace(s) != ""
-}
-
-// Lines get lines from a raw string
-func Lines(output string) []string {
-	return filter.Choose(strings.Split(output, "\n"), nonEmptyString).([]string)
-}
-
-// Columns get columns from a line
-func Columns(line string) []string {
-	return filter.Choose(strings.Split(line, " "), nonEmptyString).([]string)
-}
-
-// GoldenFile golden file specific to platform
-func GoldenFile(name string) string {
-	if IsWindows() {
-		return name + "-windows.golden"
-	}
-	return name + ".golden"
-}
-
-// IsWindows windows or other GOOS
-func IsWindows() bool {
-	return runtime.GOOS == "windows"
-}

tests/framework/suite.go

@@ -1,163 +0,0 @@
-/*
-   Copyright 2020 Docker, Inc.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package framework
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/onsi/gomega"
-	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/suite"
-)
-
-// Suite is used to store context information for e2e tests
-type Suite struct {
-	suite.Suite
-	ConfigDir string
-	BinDir    string
-}
-
-// SetupSuite is run before running any tests
-func (s *Suite) SetupSuite() {
-	d, _ := ioutil.TempDir("", "")
-	s.BinDir = d
-	gomega.RegisterFailHandler(func(message string, callerSkip ...int) {
-		log.Error(message)
-		cp := filepath.Join(s.ConfigDir, "config.json")
-		d, _ := ioutil.ReadFile(cp)
-		fmt.Printf("Bin dir:%s\n", s.BinDir)
-		fmt.Printf("Contents of %s:\n%s\n\nContents of config dir:\n", cp, string(d))
-		for _, p := range dirContents(s.ConfigDir) {
-			fmt.Println(p)
-		}
-		s.T().Fail()
-	})
-	s.copyExecutablesInBinDir()
-}
-
-// TearDownSuite is run after all tests
-func (s *Suite) TearDownSuite() {
-	_ = os.RemoveAll(s.BinDir)
-}
-
-func dirContents(dir string) []string {
-	res := []string{}
-	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		res = append(res, filepath.Join(dir, path))
-		return nil
-	})
-	return res
-}
-
-func (s *Suite) copyExecutablesInBinDir() {
-	p, err := exec.LookPath(DockerClassicExecutable())
-	if err != nil {
-		p, err = exec.LookPath(dockerExecutable())
-	}
-	gomega.Expect(err).To(gomega.BeNil())
-	err = copyFile(p, filepath.Join(s.BinDir, DockerClassicExecutable()))
-	gomega.Expect(err).To(gomega.BeNil())
-	dockerPath, err := filepath.Abs("../../bin/" + dockerExecutable())
-	gomega.Expect(err).To(gomega.BeNil())
-	err = copyFile(dockerPath, filepath.Join(s.BinDir, dockerExecutable()))
-	gomega.Expect(err).To(gomega.BeNil())
-	err = os.Setenv("PATH", concatenatePath(s.BinDir))
-	gomega.Expect(err).To(gomega.BeNil())
-}
-
-func concatenatePath(path string) string {
-	if IsWindows() {
-		return fmt.Sprintf("%s;%s", path, os.Getenv("PATH"))
-	}
-	return fmt.Sprintf("%s:%s", path, os.Getenv("PATH"))
-}
-
-func copyFile(sourceFile string, destinationFile string) error {
-	input, err := ioutil.ReadFile(sourceFile)
-	if err != nil {
-		return err
-	}
-
-	err = ioutil.WriteFile(destinationFile, input, 0777)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// BeforeTest is run before each test
-func (s *Suite) BeforeTest(suite, test string) {
-	d, _ := ioutil.TempDir("", "")
-	s.ConfigDir = d
-	_ = os.Setenv("DOCKER_CONFIG", s.ConfigDir)
-}
-
-// AfterTest is run after each test
-func (s *Suite) AfterTest(suite, test string) {
-	_ = os.RemoveAll(s.ConfigDir)
-}
-
-// ListProcessesCommand creates a command to list processes, "tasklist" on windows, "ps" otherwise.
-func (s *Suite) ListProcessesCommand() *CmdContext {
-	if IsWindows() {
-		return s.NewCommand("tasklist")
-	}
-	return s.NewCommand("ps", "-x")
-}
-
-// NewCommand creates a command context.
-func (s *Suite) NewCommand(command string, args ...string) *CmdContext {
-	return &CmdContext{
-		command: command,
-		args:    args,
-		retries: RetriesContext{interval: time.Second},
-	}
-}
-
-// Step runs a step in a test, with an identified name and output in test results
-func (s *Suite) Step(name string, test func()) {
-	s.T().Run(name, func(t *testing.T) {
-		test()
-	})
-}
-
-func dockerExecutable() string {
-	if IsWindows() {
-		return "docker.exe"
-	}
-	return "docker"
-}
-
-// DockerClassicExecutable binary name based on platform
-func DockerClassicExecutable() string {
-	const comDockerCli = "com.docker.cli"
-	if IsWindows() {
-		return comDockerCli + ".exe"
-	}
-	return comDockerCli
-}
-
-// NewDockerCommand creates a docker builder.
-func (s *Suite) NewDockerCommand(args ...string) *CmdContext {
-	return s.NewCommand(dockerExecutable(), args...)
-}

tests/skip-win-ci-e2e/skip_win_ci_test.go

@@ -17,58 +17,89 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
-	"log"
+	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
-	"github.com/docker/api/cli/mobycli"
+	"gotest.tools/v3/assert"
-
+	"gotest.tools/v3/icmd"
-	. "github.com/onsi/gomega"
+	"gotest.tools/v3/poll"
-	"github.com/stretchr/testify/suite"
 
 	. "github.com/docker/api/tests/framework"
 )
 
-type NonWinCIE2eSuite struct {
+var binDir string
-	Suite
+
+func TestMain(m *testing.M) {
+	p, cleanup, err := SetupExistingCLI()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	binDir = p
+	exitCode := m.Run()
+	cleanup()
+	os.Exit(exitCode)
 }
 
-func (s *NonWinCIE2eSuite) TestKillChildOnCancel() {
+func TestKillChildProcess(t *testing.T) {
-	s.Step("should kill com.docker.cli if parent command is cancelled", func() {
+	c := NewParallelE2eCLI(t, binDir)
-		imageName := "test-sleep-image"
-		out := s.ListProcessesCommand().ExecOrDie()
-		Expect(out).NotTo(ContainSubstring(imageName))
 
-		dir := s.ConfigDir
+	image := "test-sleep-image"
-		Expect(ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(`FROM alpine:3.10
+	pCmd := icmd.Command("ps", "-x")
-RUN sleep 100`), 0644)).To(Succeed())
+	if runtime.GOOS == "windows" {
-		shutdown := make(chan time.Time)
+		pCmd = icmd.Command("tasklist")
-		errs := make(chan error)
+	}
-		ctx := s.NewDockerCommand("build", "--no-cache", "-t", imageName, ".").WithinDirectory(dir).WithTimeout(shutdown)
+	pRes := icmd.RunCmd(pCmd)
-		go func() {
+	pRes.Assert(t, icmd.Success)
-			_, err := ctx.Exec()
+	assert.Assert(t, !strings.Contains(pRes.Combined(), image))
-			errs <- err
-		}()
-		mobyBuild := mobycli.ComDockerCli + " build --no-cache -t " + imageName
-		err := WaitFor(time.Second, 10*time.Second, errs, func() bool {
-			out := s.ListProcessesCommand().ExecOrDie()
-			return strings.Contains(out, mobyBuild)
-		})
-		Expect(err).NotTo(HaveOccurred())
-		log.Println("Killing docker process")
 
-		close(shutdown)
+	d := writeDockerfile(t)
-		err = WaitFor(time.Second, 12*time.Second, nil, func() bool {
+	buildArgs := []string{"build", "--no-cache", "-t", image, "."}
-			out := s.ListProcessesCommand().ExecOrDie()
+	cmd := c.NewDockerCmd(buildArgs...)
-			return !strings.Contains(out, mobyBuild)
+	cmd.Dir = d
-		})
+	res := icmd.StartCmd(cmd)
-		Expect(err).NotTo(HaveOccurred())
+
+	buildRunning := func(t poll.LogT) poll.Result {
+		res := icmd.RunCmd(pCmd)
+		if strings.Contains(res.Combined(), strings.Join(buildArgs, " ")) {
+			return poll.Success()
+		}
+		return poll.Continue("waiting for child process to be running")
+	}
+	poll.WaitOn(t, buildRunning, poll.WithDelay(1*time.Second))
+
+	if runtime.GOOS == "windows" {
+		err := res.Cmd.Process.Kill()
+		assert.NilError(t, err)
+	} else {
+		err := res.Cmd.Process.Signal(syscall.SIGTERM)
+		assert.NilError(t, err)
+	}
+	buildStopped := func(t poll.LogT) poll.Result {
+		res := icmd.RunCmd(pCmd)
+		if !strings.Contains(res.Combined(), strings.Join(buildArgs, " ")) {
+			return poll.Success()
+		}
+		return poll.Continue("waiting for child process to be killed")
+	}
+	poll.WaitOn(t, buildStopped, poll.WithDelay(1*time.Second), poll.WithTimeout(60*time.Second))
+}
+
+func writeDockerfile(t *testing.T) string {
+	d, err := ioutil.TempDir("", "")
+	assert.NilError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(d)
 	})
-}
+	err = ioutil.WriteFile(filepath.Join(d, "Dockerfile"), []byte(`FROM alpine:3.10
-
+RUN sleep 100`), 0644)
-func TestNonWinCIE2(t *testing.T) {
+	assert.NilError(t, err)
-	suite.Run(t, new(NonWinCIE2eSuite))
+	return d
 }