Rework e2e tests into suite

Signed-off-by: Christopher Crone <christopher.crone@docker.com>
This commit is contained in:
Christopher Crone 2020-05-20 15:57:10 +02:00
parent 384cb89193
commit a8146ad59c
7 changed files with 160 additions and 95 deletions

View File

@ -44,10 +44,10 @@ cli: ## Compile the cli
--target cli
e2e-local: ## Run End to end local tests
go run ./tests/e2e/e2e.go
go test -v ./tests/e2e
e2e-aci: ## Run End to end ACI tests (requires azure login)
go run ./tests/aci-e2e/e2e-aci.go
go test -v ./tests/aci-e2e
cross: ## Compile the CLI for linux, darwin and windows
@docker build . \

View File

@ -55,7 +55,7 @@ cross:
@GOOS=windows GOARCH=amd64 $(GO_BUILD) -o $(BINARY)-windows-amd64.exe ./cli
test:
@go test ./...
@go test -cover $(shell go list ./... | grep -vE 'e2e')
lint:
golangci-lint run --timeout 10m0s ./...

View File

@ -6,6 +6,7 @@ import (
"log"
"net/url"
"strings"
"testing"
"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
"github.com/Azure/go-autorest/autorest/to"
@ -14,6 +15,7 @@ import (
"github.com/Azure/azure-storage-file-go/azfile"
. "github.com/onsi/gomega"
"github.com/stretchr/testify/suite"
"github.com/docker/api/azure"
"github.com/docker/api/context/store"
@ -29,51 +31,60 @@ const (
testContainerName = "testcontainername"
)
func main() {
SetupTest()
var (
subscriptionID string
)
type E2eACISuite struct {
Suite
}
func (s *E2eACISuite) TestContextHelp() {
It("ensures context command includes azure-login and aci-create", func() {
output := NewDockerCommand("context", "create", "--help").ExecOrDie()
output := s.NewDockerCommand("context", "create", "--help").ExecOrDie()
Expect(output).To(ContainSubstring("docker context create CONTEXT BACKEND [OPTIONS] [flags]"))
Expect(output).To(ContainSubstring("--aci-location"))
Expect(output).To(ContainSubstring("--aci-subscription-id"))
Expect(output).To(ContainSubstring("--aci-resource-group"))
})
}
func (s *E2eACISuite) TestContextDefault() {
It("should be initialized with default context", func() {
_, err := NewCommand("docker", "context", "rm", "-f", contextName).Exec()
_, err := s.NewCommand("docker", "context", "rm", "-f", contextName).Exec()
if err == nil {
log.Println("Cleaning existing test context")
}
NewCommand("docker", "context", "use", "default").ExecOrDie()
output := NewCommand("docker", "context", "ls").ExecOrDie()
s.NewCommand("docker", "context", "use", "default").ExecOrDie()
output := s.NewCommand("docker", "context", "ls").ExecOrDie()
Expect(output).To(Not(ContainSubstring(contextName)))
Expect(output).To(ContainSubstring("default *"))
})
}
var subscriptionID string
func (s *E2eACISuite) TestACIBackend() {
It("creates a new aci context for tests", func() {
setupTestResourceGroup(resourceGroupName)
var err error
subscriptionID, err = azure.GetSubscriptionID(context.TODO())
Expect(err).To(BeNil())
NewDockerCommand("context", "create", contextName, "aci", "--aci-subscription-id", subscriptionID, "--aci-resource-group", resourceGroupName, "--aci-location", location).ExecOrDie()
s.NewDockerCommand("context", "create", contextName, "aci", "--aci-subscription-id", subscriptionID, "--aci-resource-group", resourceGroupName, "--aci-location", location).ExecOrDie()
// Expect(output).To(ContainSubstring("ACI context acitest created"))
})
defer deleteResourceGroup(resourceGroupName)
It("uses the aci context", func() {
currentContext := NewCommand("docker", "context", "use", contextName).ExecOrDie()
currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie()
Expect(currentContext).To(ContainSubstring(contextName))
output := NewCommand("docker", "context", "ls").ExecOrDie()
output := s.NewCommand("docker", "context", "ls").ExecOrDie()
Expect(output).To(ContainSubstring("acitest *"))
})
It("ensures no container is running initially", func() {
output := NewDockerCommand("ps").ExecOrDie()
output := s.NewDockerCommand("ps").ExecOrDie()
Expect(len(Lines(output))).To(Equal(1))
})
@ -91,13 +102,13 @@ func main() {
uploadFile(credential, u.String(), testFileName, testFileContent)
mountTarget := "/usr/share/nginx/html"
output := NewDockerCommand("run", "nginx",
output := s.NewDockerCommand("run", "nginx",
"-v", fmt.Sprintf("%s:%s@%s:%s",
testStorageAccountName, firstKey, testShareName, mountTarget),
"-p", "80:80",
"--name", testContainerName).ExecOrDie()
Expect(output).To(Equal(testContainerName + "\n"))
output = NewDockerCommand("ps").ExecOrDie()
output = s.NewDockerCommand("ps").ExecOrDie()
lines := Lines(output)
Expect(len(lines)).To(Equal(2))
@ -108,19 +119,19 @@ func main() {
Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
publishedURL := strings.ReplaceAll(exposedIP, "->80/tcp", "")
output = NewCommand("curl", publishedURL).ExecOrDie()
output = s.NewCommand("curl", publishedURL).ExecOrDie()
Expect(output).To(ContainSubstring(testFileContent))
})
It("removes container nginx", func() {
output := NewDockerCommand("rm", testContainerName).ExecOrDie()
output := s.NewDockerCommand("rm", testContainerName).ExecOrDie()
Expect(Lines(output)[0]).To(Equal(testContainerName))
})
It("deploys a compose app", func() {
NewDockerCommand("compose", "up", "-f", "./tests/composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
s.NewDockerCommand("compose", "up", "-f", "../composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
// Expect(output).To(ContainSubstring("Successfully deployed"))
output := NewDockerCommand("ps").ExecOrDie()
output := s.NewDockerCommand("ps").ExecOrDie()
Lines := Lines(output)
Expect(len(Lines)).To(Equal(4))
webChecked := false
@ -134,9 +145,9 @@ func main() {
Expect(exposedIP).To(ContainSubstring(":80->80/tcp"))
url := strings.ReplaceAll(exposedIP, "->80/tcp", "")
output = NewCommand("curl", url).ExecOrDie()
output = s.NewCommand("curl", url).ExecOrDie()
Expect(output).To(ContainSubstring("Docker Compose demo"))
output = NewCommand("curl", url+"/words/noun").ExecOrDie()
output = s.NewCommand("curl", url+"/words/noun").ExecOrDie()
Expect(output).To(ContainSubstring("\"word\":"))
}
}
@ -145,20 +156,20 @@ func main() {
})
It("get logs from web service", func() {
output := NewDockerCommand("logs", "acidemo_web").ExecOrDie()
output := s.NewDockerCommand("logs", "acidemo_web").ExecOrDie()
Expect(output).To(ContainSubstring("Listening on port 80"))
})
It("shutdown compose app", func() {
NewDockerCommand("compose", "down", "-f", "./tests/composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
s.NewDockerCommand("compose", "down", "-f", "../composefiles/aci-demo/aci_demo_port.yaml", "--name", "acidemo").ExecOrDie()
})
It("switches back to default context", func() {
output := NewCommand("docker", "context", "use", "default").ExecOrDie()
output := s.NewCommand("docker", "context", "use", "default").ExecOrDie()
Expect(output).To(ContainSubstring("default"))
})
It("deletes test context", func() {
output := NewCommand("docker", "context", "rm", contextName).ExecOrDie()
output := s.NewCommand("docker", "context", "rm", contextName).ExecOrDie()
Expect(output).To(ContainSubstring(contextName))
})
}
@ -212,6 +223,10 @@ func uploadFile(credential azfile.SharedKeyCredential, baseURL, fileName, fileCo
Expect(err).To(BeNil())
}
func TestE2eACI(t *testing.T) {
suite.Run(t, new(E2eACISuite))
}
func setupTestResourceGroup(groupName string) {
log.Println("Creating resource group " + resourceGroupName)
ctx := context.TODO()

View File

@ -1,81 +1,85 @@
package main
import (
"fmt"
"os"
"os/exec"
"testing"
"time"
. "github.com/onsi/gomega"
"github.com/stretchr/testify/suite"
. "github.com/docker/api/tests/framework"
)
func main() {
SetupTest()
type E2eSuite struct {
Suite
}
func (s *E2eSuite) TestContextHelp() {
It("ensures context command includes azure-login and aci-create", func() {
output := NewDockerCommand("context", "create", "--help").ExecOrDie()
output := s.NewDockerCommand("context", "create", "--help").ExecOrDie()
Expect(output).To(ContainSubstring("docker context create CONTEXT BACKEND [OPTIONS] [flags]"))
Expect(output).To(ContainSubstring("--aci-location"))
Expect(output).To(ContainSubstring("--aci-subscription-id"))
Expect(output).To(ContainSubstring("--aci-resource-group"))
})
}
func (s *E2eSuite) TestContextDefault() {
It("should be initialized with default context", func() {
NewDockerCommand("context", "use", "default").ExecOrDie()
output := NewDockerCommand("context", "show").ExecOrDie()
s.NewDockerCommand("context", "use", "default").ExecOrDie()
output := s.NewDockerCommand("context", "show").ExecOrDie()
Expect(output).To(ContainSubstring("default"))
output = NewCommand("docker", "context", "ls").ExecOrDie()
output = s.NewCommand("docker", "context", "ls").ExecOrDie()
Expect(output).To(Not(ContainSubstring("test-example")))
Expect(output).To(ContainSubstring("default *"))
})
}
func (s *E2eSuite) TestLegacy() {
It("should list all legacy commands", func() {
output := NewDockerCommand("--help").ExecOrDie()
output := s.NewDockerCommand("--help").ExecOrDie()
Expect(output).To(ContainSubstring("swarm"))
})
It("should execute legacy commands", func() {
output, _ := NewDockerCommand("swarm", "join").Exec()
output, _ := s.NewDockerCommand("swarm", "join").Exec()
Expect(output).To(ContainSubstring("\"docker swarm join\" requires exactly 1 argument."))
})
It("should run local container in less than 5 secs", func() {
NewDockerCommand("pull", "hello-world").ExecOrDie()
output := NewDockerCommand("run", "hello-world").WithTimeout(time.NewTimer(5 * time.Second).C).ExecOrDie()
It("should run local container in less than 10 secs", func() {
s.NewDockerCommand("pull", "hello-world").ExecOrDie()
output := s.NewDockerCommand("run", "--rm", "hello-world").WithTimeout(time.NewTimer(10 * time.Second).C).ExecOrDie()
Expect(output).To(ContainSubstring("Hello from Docker!"))
})
}
It("should list local container", func() {
output := NewDockerCommand("ps", "-a").ExecOrDie()
Expect(output).To(ContainSubstring("hello-world"))
})
func (s *E2eSuite) TestMockBackend() {
It("creates a new test context to hardcoded example backend", func() {
NewDockerCommand("context", "create", "test-example", "example").ExecOrDie()
s.NewDockerCommand("context", "create", "test-example", "example").ExecOrDie()
// Expect(output).To(ContainSubstring("test-example context acitest created"))
})
defer NewDockerCommand("context", "rm", "test-example").ExecOrDie()
defer NewDockerCommand("context", "use", "default").ExecOrDie()
It("uses the test context", func() {
currentContext := NewDockerCommand("context", "use", "test-example").ExecOrDie()
currentContext := s.NewDockerCommand("context", "use", "test-example").ExecOrDie()
Expect(currentContext).To(ContainSubstring("test-example"))
output := NewDockerCommand("context", "ls").ExecOrDie()
output := s.NewDockerCommand("context", "ls").ExecOrDie()
Expect(output).To(ContainSubstring("test-example *"))
output = NewDockerCommand("context", "show").ExecOrDie()
output = s.NewDockerCommand("context", "show").ExecOrDie()
Expect(output).To(ContainSubstring("test-example"))
})
It("can run ps command", func() {
output := NewDockerCommand("ps").ExecOrDie()
output := s.NewDockerCommand("ps").ExecOrDie()
lines := Lines(output)
Expect(len(lines)).To(Equal(3))
Expect(lines[2]).To(ContainSubstring("1234 alpine"))
})
It("can run quiet ps command", func() {
output := NewDockerCommand("ps", "-q").ExecOrDie()
output := s.NewDockerCommand("ps", "-q").ExecOrDie()
lines := Lines(output)
Expect(len(lines)).To(Equal(2))
Expect(lines[0]).To(Equal("id"))
@ -83,7 +87,7 @@ func main() {
})
It("can run ps command with all ", func() {
output := NewDockerCommand("ps", "-q", "--all").ExecOrDie()
output := s.NewDockerCommand("ps", "-q", "--all").ExecOrDie()
lines := Lines(output)
Expect(len(lines)).To(Equal(3))
Expect(lines[0]).To(Equal("id"))
@ -92,28 +96,42 @@ func main() {
})
It("can run 'run' command", func() {
output := NewDockerCommand("run", "nginx", "-p", "80:80").ExecOrDie()
output := s.NewDockerCommand("run", "nginx", "-p", "80:80").ExecOrDie()
Expect(output).To(ContainSubstring("Running container \"nginx\" with name"))
})
}
func (s *E2eSuite) TestAPIServer() {
_, err := exec.LookPath("yarn")
if err != nil || os.Getenv("SKIP_NODE") != "" {
s.T().Skip("skipping, yarn not installed")
}
It("can run 'serve' command", func() {
server, err := startCliServer()
Expect(err).To(BeNil())
defer killCliServer(server)
cName := "test-example"
s.NewDockerCommand("context", "create", cName, "example").ExecOrDie()
NewCommand("yarn", "install").WithinDirectory("tests/node-client").ExecOrDie()
output := NewCommand("yarn", "run", "start", "test-example").WithinDirectory("tests/node-client").ExecOrDie()
sPath := fmt.Sprintf("unix:///%s/docker.sock", s.ConfigDir)
server, err := serveAPI(s.ConfigDir, sPath)
Expect(err).To(BeNil())
defer killProcess(server)
s.NewCommand("yarn", "install").WithinDirectory("../node-client").ExecOrDie()
output := s.NewCommand("yarn", "run", "start", cName, sPath).WithinDirectory("../node-client").ExecOrDie()
Expect(output).To(ContainSubstring("nginx"))
})
}
func killCliServer(process *os.Process) {
func TestE2e(t *testing.T) {
suite.Run(t, new(E2eSuite))
}
func killProcess(process *os.Process) {
err := process.Kill()
Expect(err).To(BeNil())
}
func startCliServer() (*os.Process, error) {
cmd := exec.Command("./bin/docker", "serve", "--address", "unix:///tmp/backend.sock")
func serveAPI(configDir string, address string) (*os.Process, error) {
cmd := exec.Command("../../bin/docker", "--config", configDir, "serve", "--address", address)
err := cmd.Start()
if err != nil {
return nil, err

View File

@ -4,13 +4,18 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
func (b CmdContext) makeCmd() *exec.Cmd {
@ -35,25 +40,79 @@ type RetriesContext struct {
interval time.Duration
}
// Suite is used to store context information for e2e tests
type Suite struct {
suite.Suite
ConfigDir string
BinDir string
}
// SetupSuite is run before running any tests
func (s *Suite) SetupSuite() {
d, _ := ioutil.TempDir("", "")
s.BinDir = d
gomega.RegisterFailHandler(func(message string, callerSkip ...int) {
log.Error(message)
cp := filepath.Join(s.ConfigDir, "config.json")
d, _ := ioutil.ReadFile(cp)
fmt.Printf("Contents of %s:\n%s\n\nContents of config dir:\n", cp, string(d))
out, _ := s.NewCommand("find", s.ConfigDir).Exec()
fmt.Println(out)
s.T().Fail()
})
s.linkClassicDocker()
}
// TearDownSuite is run after all tests
func (s *Suite) TearDownSuite() {
_ = os.RemoveAll(s.BinDir)
}
func (s *Suite) linkClassicDocker() {
p, err := exec.LookPath("docker")
gomega.Expect(err).To(gomega.BeNil())
err = os.Symlink(p, filepath.Join(s.BinDir, "docker-classic"))
gomega.Expect(err).To(gomega.BeNil())
err = os.Setenv("PATH", fmt.Sprintf("%s:%s", s.BinDir, os.Getenv("PATH")))
gomega.Expect(err).To(gomega.BeNil())
}
// BeforeTest is run before each test
func (s *Suite) BeforeTest(suite, test string) {
d, _ := ioutil.TempDir("", "")
s.ConfigDir = d
}
// AfterTest is run after each test
func (s *Suite) AfterTest(suite, test string) {
err := os.RemoveAll(s.ConfigDir)
require.NoError(s.T(), err)
}
// NewCommand creates a command context.
func NewCommand(command string, args ...string) *CmdContext {
func (s *Suite) NewCommand(command string, args ...string) *CmdContext {
var envs []string
if s.ConfigDir != "" {
envs = append(os.Environ(), fmt.Sprintf("DOCKER_CONFIG=%s", s.ConfigDir))
}
return &CmdContext{
command: command,
args: args,
envs: envs,
retries: RetriesContext{interval: time.Second},
}
}
func dockerExecutable() string {
if runtime.GOOS == "windows" {
return "./bin/docker.exe"
return "../../bin/docker.exe"
}
return "./bin/docker"
return "../../bin/docker"
}
// NewDockerCommand creates a docker builder.
func NewDockerCommand(args ...string) *CmdContext {
return NewCommand(dockerExecutable(), args...)
func (s *Suite) NewDockerCommand(args ...string) *CmdContext {
return s.NewCommand(dockerExecutable(), args...)
}
// WithinDirectory tells Docker the cwd.

View File

@ -1,14 +1,10 @@
package framework
import (
"fmt"
"log"
"os"
"strings"
"github.com/robpike/filter"
"github.com/onsi/gomega"
)
func nonEmptyString(s string) bool {
@ -30,28 +26,3 @@ func It(description string, test func()) {
test()
log.Print("Passed: ", description)
}
func gomegaFailHandler(message string, callerSkip ...int) {
log.Fatal(message)
}
//SetupTest Init gomega fail handler
func SetupTest() {
gomega.RegisterFailHandler(gomegaFailHandler)
linkClassicDocker()
}
func linkClassicDocker() {
dockerOriginal := strings.TrimSuffix(NewCommand("which", "docker").ExecOrDie(), "\n")
_, err := NewCommand("rm", "-r", "./bin/tests").Exec()
if err == nil {
fmt.Println("Removing existing /bin/tests folder before running tests")
}
_, err = NewCommand("mkdir", "-p", "./bin/tests").Exec()
gomega.Expect(err).To(gomega.BeNil())
NewCommand("ln", "-s", dockerOriginal, "./bin/tests/docker-classic").ExecOrDie()
newPath := "./bin/tests:" + os.Getenv("PATH")
err = os.Setenv("PATH", newPath)
gomega.Expect(err).To(gomega.BeNil())
}

View File

@ -3,8 +3,10 @@ import * as continersPb from "./grpc/containers_grpc_pb";
import { IContainersClient } from './grpc/containers_grpc_pb';
import { ListRequest, ListResponse } from "./grpc/containers_pb";
let address = process.argv[3] || "unix:///tmp/backend.sock";
const ContainersServiceClient = grpc.makeClientConstructor(continersPb["com.docker.api.containers.v1.Containers"], "ContainersClient");
const client = new ContainersServiceClient("unix:///tmp/backend.sock", grpc.credentials.createInsecure()) as unknown as IContainersClient;
const client = new ContainersServiceClient(address, grpc.credentials.createInsecure()) as unknown as IContainersClient;
let backend = process.argv[2] || "moby";
const meta = new grpc.Metadata();