diff --git a/Makefile b/Makefile index a5a3d6023..8cecf6f8a 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ ifeq ($(UNAME_S),Darwin) endif GIT_TAG?=$(shell git describe --tags --match "v[0-9]*") +TESTIFY_OPTS=$(if $(TESTIFY),-testify.m $(TESTIFY),) all: cli @@ -37,14 +38,14 @@ cli: ## Compile the cli --build-arg GIT_TAG=$(GIT_TAG) \ --output ./bin -e2e-local: ## Run End to end local tests - go test -v ./tests/e2e ./tests/skip-win-ci-e2e ./local/e2e +e2e-local: ## Run End to end local tests. set env TESTIFY=Test1 for running single test + go test -v ./tests/e2e ./tests/skip-win-ci-e2e ./local/e2e $(TESTIFY_OPTS) -e2e-win-ci: ## Run End to end local tests on windows CI, no docker for linux containers available ATM - go test -v ./tests/e2e +e2e-win-ci: ## Run End to end local tests on windows CI, no docker for linux containers available ATM. set env TESTIFY=Test1 for running single test + go test -v ./tests/e2e $(TESTIFY_OPTS) -e2e-aci: ## Run End to end ACI tests (requires azure login) - go test -v ./tests/aci-e2e +e2e-aci: ## Run End to end ACI tests. set env TESTIFY=Test1 for running single test + go test -v ./tests/aci-e2e $(TESTIFY_OPTS) cross: ## Compile the CLI for linux, darwin and windows @docker build . --target cross \ diff --git a/README.md b/README.md index f406cc482..e083cb843 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,11 @@ az ad sp create-for-rbac --name 'MyTestServicePrincipal' --sdk-auth Running aci e2e tests will override your local login, the service principal credentials use a token that cannot be refreshed automatically. You might need to run again `docker login azure` to properly use the command line after running ACI e2e tests. +You can also run a single ACI test from the test suite : +``` +TESTIFY=TestACIRunSingleContainer AZURE_TENANT_ID="xxx" AZURE_CLIENT_ID="yyy" AZURE_CLIENT_SECRET="yyy" make e2e-aci +``` + ## Release To create a new release: diff --git a/tests/aci-e2e/e2e-aci_test.go b/tests/aci-e2e/e2e-aci_test.go index dfd6d3835..f9a85ac8e 100644 --- a/tests/aci-e2e/e2e-aci_test.go +++ b/tests/aci-e2e/e2e-aci_test.go @@ -51,77 +51,31 @@ const ( ) var ( - subscriptionID string - resourceGroupName = "resourceGroupTestE2E-" + RandStringBytes(10) - testStorageAccountName = "storageteste2e" + RandStringBytes(6) // "between 3 and 24 characters in length and use numbers and lower-case letters only" + subscriptionID string ) type E2eACISuite struct { Suite } -func (s *E2eACISuite) TestContextDefault() { - s.T().Run("should be initialized with default context", func(t *testing.T) { - _, err := s.NewCommand("docker", "context", "rm", "-f", contextName).Exec() - if err == nil { - log.Println("Cleaning existing test context") - } - - s.NewCommand("docker", "context", "use", "default").ExecOrDie() - output := s.NewCommand("docker", "context", "ls").ExecOrDie() - Expect(output).To(Not(ContainSubstring(contextName))) - Expect(output).To(ContainSubstring("default *")) - }) -} - -func (s *E2eACISuite) TestACIBackend() { - s.T().Run("Logs in azure using service principal credentials", func(t *testing.T) { - login, err := login.NewAzureLoginService() - Expect(err).To(BeNil()) - // in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth` - clientID := os.Getenv("AZURE_CLIENT_ID") - clientSecret := os.Getenv("AZURE_CLIENT_SECRET") - tenantID := os.Getenv("AZURE_TENANT_ID") - err = login.TestLoginFromServicePrincipal(clientID, clientSecret, tenantID) - Expect(err).To(BeNil()) - }) - - s.T().Run("creates a new aci context for tests", func(t *testing.T) { - setupTestResourceGroup(resourceGroupName) - helper := azure.NewACIResourceGroupHelper() - models, err := helper.GetSubscriptionIDs(context.TODO()) - Expect(err).To(BeNil()) - subscriptionID = *models[0].SubscriptionID - - s.NewDockerCommand("context", "create", "aci", contextName, "--subscription-id", subscriptionID, "--resource-group", resourceGroupName, "--location", location).ExecOrDie() - }) - +func (s *E2eACISuite) TestACIRunSingleContainer() { + resourceGroupName := s.setupTestResourceGroup() defer deleteResourceGroup(resourceGroupName) - s.T().Run("uses the aci context", func(t *testing.T) { - currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie() - Expect(currentContext).To(ContainSubstring(contextName)) - output := s.NewCommand("docker", "context", "ls").ExecOrDie() - Expect(output).To(ContainSubstring("acitest *")) - }) - - It("ensures no container is running initially", func() { - output := s.NewDockerCommand("ps").ExecOrDie() - Expect(len(Lines(output))).To(Equal(1)) - }) - var nginxExposedURL string - s.T().Run("runs nginx on port 80", func(t *testing.T) { + s.Step("runs nginx on port 80", func() { aciContext := store.AciContext{ SubscriptionID: subscriptionID, Location: location, ResourceGroup: resourceGroupName, } + + testStorageAccountName := "storageteste2e" + RandStringBytes(6) // "between 3 and 24 characters in length and use numbers and lower-case letters only" createStorageAccount(aciContext, testStorageAccountName) - defer deleteStorageAccount(aciContext) + defer deleteStorageAccount(aciContext, testStorageAccountName) keys := getStorageKeys(aciContext, testStorageAccountName) firstKey := *keys[0].Value - credential, u := createFileShare(firstKey, testShareName) + credential, u := createFileShare(firstKey, testShareName, testStorageAccountName) uploadFile(credential, u.String(), testFileName, testFileContent) mountTarget := "/usr/share/nginx/html" @@ -150,13 +104,13 @@ func (s *E2eACISuite) TestACIBackend() { Expect(output).To(ContainSubstring("GET")) }) - s.T().Run("exec command", func(t *testing.T) { + s.Step("exec command", func() { _, err := s.NewDockerCommand("exec", testContainerName, "echo", "fail_with_argument").Exec() Expect(err.Error()).To(ContainSubstring("ACI exec command does not accept arguments to the command. " + "Only the binary should be specified")) }) - s.T().Run("follow logs from nginx", func(t *testing.T) { + s.Step("follow logs from nginx", func() { timeChan := make(chan time.Time) ctx := s.NewDockerCommand("logs", "--follow", testContainerName).WithTimeout(timeChan) @@ -178,12 +132,12 @@ func (s *E2eACISuite) TestACIBackend() { Expect(output).To(ContainSubstring("/test")) }) - s.T().Run("removes container nginx", func(t *testing.T) { + s.Step("removes container nginx", func() { output := s.NewDockerCommand("rm", testContainerName).ExecOrDie() Expect(Lines(output)[0]).To(Equal(testContainerName)) }) - s.T().Run("re-run nginx with modified cpu/mem, and without --detach and follow logs", func(t *testing.T) { + s.Step("re-run nginx with modified cpu/mem, and without --detach and follow logs", func() { shutdown := make(chan time.Time) errs := make(chan error) outChan := make(chan string) @@ -224,10 +178,14 @@ func (s *E2eACISuite) TestACIBackend() { Expect(output).To(ContainSubstring("/test")) }) - s.T().Run("removes container nginx", func(t *testing.T) { + s.Step("removes container nginx", func() { output := s.NewDockerCommand("rm", testContainerName).ExecOrDie() Expect(Lines(output)[0]).To(Equal(testContainerName)) }) +} + +func (s *E2eACISuite) TestACIComposeApplication() { + defer deleteResourceGroup(s.setupTestResourceGroup()) var exposedURL string const composeFile = "../composefiles/aci-demo/aci_demo_port.yaml" @@ -236,7 +194,7 @@ func (s *E2eACISuite) TestACIBackend() { const serverContainer = composeProjectName + "_web" const wordsContainer = composeProjectName + "_words" - s.T().Run("deploys a compose app", func(t *testing.T) { + s.Step("deploys a compose app", func() { // specifically do not specify project name here, it will be derived from current folder "acie2e" s.NewDockerCommand("compose", "up", "-f", composeFile).ExecOrDie() output := s.NewDockerCommand("ps").ExecOrDie() @@ -263,12 +221,12 @@ func (s *E2eACISuite) TestACIBackend() { Expect(webChecked).To(BeTrue()) }) - s.T().Run("get logs from web service", func(t *testing.T) { + s.Step("get logs from web service", func() { output := s.NewDockerCommand("logs", serverContainer).ExecOrDie() Expect(output).To(ContainSubstring("Listening on port 80")) }) - s.T().Run("updates a compose app", func(t *testing.T) { + s.Step("updates a compose app", func() { s.NewDockerCommand("compose", "up", "-f", composeFileMultiplePorts, "--project-name", composeProjectName).ExecOrDie() // Expect(output).To(ContainSubstring("Successfully deployed")) output := s.NewDockerCommand("ps").ExecOrDie() @@ -304,11 +262,15 @@ func (s *E2eACISuite) TestACIBackend() { Expect(wordsChecked).To(BeTrue()) }) - s.T().Run("shutdown compose app", func(t *testing.T) { + s.Step("shutdown compose app", func() { s.NewDockerCommand("compose", "down", "--project-name", composeProjectName).ExecOrDie() }) +} - s.T().Run("runs mysql with env variables", func(t *testing.T) { +func (s *E2eACISuite) TestACIDeployMySQlwithEnvVars() { + defer deleteResourceGroup(s.setupTestResourceGroup()) + + s.Step("runs mysql with env variables", func() { err := os.Setenv("MYSQL_USER", "user1") Expect(err).To(BeNil()) s.NewDockerCommand("run", "-d", "mysql:5.7", "-e", "MYSQL_ROOT_PASSWORD=rootpwd", "-e", "MYSQL_DATABASE=mytestdb", "-e", "MYSQL_USER", "-e", "MYSQL_PASSWORD=userpwd").ExecOrDie() @@ -330,17 +292,69 @@ func (s *E2eACISuite) TestACIBackend() { Expect(err).To(BeNil()) }) - s.T().Run("switches back to default context", func(t *testing.T) { + s.Step("switches back to default context", func() { output := s.NewCommand("docker", "context", "use", "default").ExecOrDie() Expect(output).To(ContainSubstring("default")) }) - s.T().Run("deletes test context", func(t *testing.T) { + s.Step("deletes test context", func() { output := s.NewCommand("docker", "context", "rm", contextName).ExecOrDie() Expect(output).To(ContainSubstring(contextName)) }) } +func (s *E2eACISuite) setupTestResourceGroup() string { + var resourceGroupName = randomResourceGroup() + s.Step("should be initialized with default context", s.checkDefaultContext) + s.Step("Logs in azure using service principal credentials", azureLogin) + s.Step("creates a new aci context for tests and use it", s.createAciContextAndUseIt(resourceGroupName)) + s.Step("ensures no container is running initially", s.checkNoContainnersRunning) + return resourceGroupName +} + +func (s *E2eACISuite) checkDefaultContext() { + output := s.NewCommand("docker", "context", "ls").ExecOrDie() + Expect(output).To(Not(ContainSubstring(contextName))) + Expect(output).To(ContainSubstring("default *")) +} + +func azureLogin() { + login, err := login.NewAzureLoginService() + Expect(err).To(BeNil()) + // in order to create new service principal and get these 3 values : `az ad sp create-for-rbac --name 'TestServicePrincipal' --sdk-auth` + clientID := os.Getenv("AZURE_CLIENT_ID") + clientSecret := os.Getenv("AZURE_CLIENT_SECRET") + tenantID := os.Getenv("AZURE_TENANT_ID") + err = login.TestLoginFromServicePrincipal(clientID, clientSecret, tenantID) + Expect(err).To(BeNil()) +} + +func (s *E2eACISuite) createAciContextAndUseIt(resourceGroupName string) func() { + return func() { + setupTestResourceGroup(resourceGroupName) + helper := azure.NewACIResourceGroupHelper() + models, err := helper.GetSubscriptionIDs(context.TODO()) + Expect(err).To(BeNil()) + subscriptionID = *models[0].SubscriptionID + + s.NewDockerCommand("context", "create", "aci", contextName, "--subscription-id", subscriptionID, "--resource-group", resourceGroupName, "--location", location).ExecOrDie() + + currentContext := s.NewCommand("docker", "context", "use", contextName).ExecOrDie() + Expect(currentContext).To(ContainSubstring(contextName)) + output := s.NewCommand("docker", "context", "ls").ExecOrDie() + Expect(output).To(ContainSubstring("acitest *")) + } +} + +func (s *E2eACISuite) checkNoContainnersRunning() { + output := s.NewDockerCommand("ps").ExecOrDie() + Expect(len(Lines(output))).To(Equal(1)) +} + +func randomResourceGroup() string { + return "resourceGroupTestE2E-" + RandStringBytes(10) +} + func createStorageAccount(aciContext store.AciContext, accountName string) azure_storage.Account { log.Println("Creating storage account " + accountName) storageAccount, err := storage.CreateStorageAccount(context.TODO(), aciContext, accountName) @@ -358,13 +372,13 @@ func getStorageKeys(aciContext store.AciContext, storageAccountName string) []az return *list.Keys } -func deleteStorageAccount(aciContext store.AciContext) { +func deleteStorageAccount(aciContext store.AciContext, testStorageAccountName string) { log.Println("Deleting storage account " + testStorageAccountName) _, err := storage.DeleteStorageAccount(context.TODO(), aciContext, testStorageAccountName) Expect(err).To(BeNil()) } -func createFileShare(key, shareName string) (azfile.SharedKeyCredential, url.URL) { +func createFileShare(key, shareName string, testStorageAccountName string) (azfile.SharedKeyCredential, url.URL) { // Create a ShareURL object that wraps a soon-to-be-created share's URL and a default pipeline. u, _ := url.Parse(fmt.Sprintf("https://%s.file.core.windows.net/%s", testStorageAccountName, shareName)) credential, err := azfile.NewSharedKeyCredential(testStorageAccountName, key) @@ -389,25 +403,25 @@ func TestE2eACI(t *testing.T) { suite.Run(t, new(E2eACISuite)) } -func setupTestResourceGroup(groupName string) { +func setupTestResourceGroup(resourceGroupName string) { log.Println("Creating resource group " + resourceGroupName) ctx := context.TODO() helper := azure.NewACIResourceGroupHelper() models, err := helper.GetSubscriptionIDs(ctx) Expect(err).To(BeNil()) - _, err = helper.CreateOrUpdate(ctx, *models[0].SubscriptionID, groupName, resources.Group{ + _, err = helper.CreateOrUpdate(ctx, *models[0].SubscriptionID, resourceGroupName, resources.Group{ Location: to.StringPtr(location), }) Expect(err).To(BeNil()) } -func deleteResourceGroup(groupName string) { +func deleteResourceGroup(resourceGroupName string) { log.Println("Deleting resource group " + resourceGroupName) ctx := context.TODO() helper := azure.NewACIResourceGroupHelper() models, err := helper.GetSubscriptionIDs(ctx) Expect(err).To(BeNil()) - err = helper.DeleteAsync(ctx, *models[0].SubscriptionID, groupName) + err = helper.DeleteAsync(ctx, *models[0].SubscriptionID, resourceGroupName) Expect(err).To(BeNil()) } diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index b7a044d23..12b62d368 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -99,7 +99,7 @@ func (s *E2eSuite) TestContextLsFormat() { } func (s *E2eSuite) TestContextCreateParseErrorDoesNotDelegateToLegacy() { - It("should dispay new cli error when parsing context create flags", func() { + s.Step("should dispay new cli error when parsing context create flags", func() { _, err := s.NewDockerCommand("context", "create", "aci", "--subscription-id", "titi").Exec() Expect(err.Error()).NotTo(ContainSubstring("unknown flag")) Expect(err.Error()).To(ContainSubstring("accepts 1 arg(s), received 0")) @@ -154,7 +154,7 @@ func (s *E2eSuite) TestCloudLogin() { } func (s *E2eSuite) TestSetupError() { - It("should display an error if cannot shell out to com.docker.cli", func() { + s.Step("should display an error if cannot shell out to com.docker.cli", func() { err := os.Setenv("PATH", s.BinDir) Expect(err).To(BeNil()) err = os.Remove(filepath.Join(s.BinDir, DockerClassicExecutable())) @@ -167,23 +167,23 @@ func (s *E2eSuite) TestSetupError() { } func (s *E2eSuite) TestLegacy() { - It("should list all legacy commands", func() { + s.Step("should list all legacy commands", func() { output := s.NewDockerCommand("--help").ExecOrDie() Expect(output).To(ContainSubstring("swarm")) }) - It("should execute legacy commands", func() { + s.Step("should execute legacy commands", func() { output, _ := s.NewDockerCommand("swarm", "join").Exec() Expect(output).To(ContainSubstring("\"docker swarm join\" requires exactly 1 argument.")) }) - It("should run local container in less than 10 secs", func() { + s.Step("should run local container in less than 10 secs", func() { s.NewDockerCommand("pull", "hello-world").ExecOrDie() output := s.NewDockerCommand("run", "--rm", "hello-world").WithTimeout(time.NewTimer(20 * time.Second).C).ExecOrDie() Expect(output).To(ContainSubstring("Hello from Docker!")) }) - It("should execute legacy commands in other moby contexts", func() { + s.Step("should execute legacy commands in other moby contexts", func() { s.NewDockerCommand("context", "create", "mobyCtx", "--from=default").ExecOrDie() s.NewDockerCommand("context", "use", "mobyCtx").ExecOrDie() output, _ := s.NewDockerCommand("swarm", "join").Exec() @@ -256,12 +256,12 @@ func (s *E2eSuite) TestAllowsFormatFlagInVersion() { } func (s *E2eSuite) TestMockBackend() { - It("creates a new test context to hardcoded example backend", func() { + s.Step("creates a new test context to hardcoded example backend", func() { s.NewDockerCommand("context", "create", "example", "test-example").ExecOrDie() // Expect(output).To(ContainSubstring("test-example context acitest created")) }) - It("uses the test context", func() { + s.Step("uses the test context", func() { currentContext := s.NewDockerCommand("context", "use", "test-example").ExecOrDie() Expect(currentContext).To(ContainSubstring("test-example")) output := s.NewDockerCommand("context", "ls").ExecOrDie() @@ -270,14 +270,14 @@ func (s *E2eSuite) TestMockBackend() { Expect(output).To(ContainSubstring("test-example")) }) - It("can run ps command", func() { + s.Step("can run ps command", func() { output := s.NewDockerCommand("ps").ExecOrDie() lines := Lines(output) Expect(len(lines)).To(Equal(3)) Expect(lines[2]).To(ContainSubstring("1234 alpine")) }) - It("can run quiet ps command", func() { + s.Step("can run quiet ps command", func() { output := s.NewDockerCommand("ps", "-q").ExecOrDie() lines := Lines(output) Expect(len(lines)).To(Equal(2)) @@ -285,7 +285,7 @@ func (s *E2eSuite) TestMockBackend() { Expect(lines[1]).To(Equal("1234")) }) - It("can run ps command with all ", func() { + s.Step("can run ps command with all ", func() { output := s.NewDockerCommand("ps", "-q", "--all").ExecOrDie() lines := Lines(output) Expect(len(lines)).To(Equal(3)) @@ -294,11 +294,11 @@ func (s *E2eSuite) TestMockBackend() { Expect(lines[2]).To(Equal("stopped")) }) - It("can run inspect command on container", func() { + s.Step("can run inspect command on container", func() { golden.Assert(s.T(), s.NewDockerCommand("inspect", "id").ExecOrDie(), "inspect-id.golden") }) - It("can run 'run' command", func() { + s.Step("can run 'run' command", func() { output := s.NewDockerCommand("run", "-d", "nginx", "-p", "80:80").ExecOrDie() Expect(output).To(ContainSubstring("Running container \"nginx\" with name")) }) diff --git a/tests/framework/helper.go b/tests/framework/helper.go index 1eaddc6ec..d930b49aa 100644 --- a/tests/framework/helper.go +++ b/tests/framework/helper.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/robpike/filter" - "github.com/sirupsen/logrus" ) func nonEmptyString(s string) bool { @@ -50,9 +49,3 @@ func GoldenFile(name string) string { func IsWindows() bool { return runtime.GOOS == "windows" } - -// It runs func -func It(description string, test func()) { - test() - logrus.Print("Passed: ", description) -} diff --git a/tests/framework/suite.go b/tests/framework/suite.go index a3908705a..9648e8267 100644 --- a/tests/framework/suite.go +++ b/tests/framework/suite.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path/filepath" + "testing" "time" "github.com/onsi/gomega" @@ -133,6 +134,13 @@ func (s *Suite) NewCommand(command string, args ...string) *CmdContext { } } +// Step runs a step in a test, with an identified name and output in test results +func (s *Suite) Step(name string, test func()) { + s.T().Run(name, func(t *testing.T) { + test() + }) +} + func dockerExecutable() string { if IsWindows() { return "docker.exe" diff --git a/tests/skip-win-ci-e2e/skip_win_ci_test.go b/tests/skip-win-ci-e2e/skip_win_ci_test.go index 38195a349..b8de44377 100644 --- a/tests/skip-win-ci-e2e/skip_win_ci_test.go +++ b/tests/skip-win-ci-e2e/skip_win_ci_test.go @@ -35,7 +35,7 @@ type NonWinCIE2eSuite struct { } func (s *NonWinCIE2eSuite) TestKillChildOnCancel() { - It("should kill com.docker.cli if parent command is cancelled", func() { + s.Step("should kill com.docker.cli if parent command is cancelled", func() { imageName := "test-sleep-image" out := s.ListProcessesCommand().ExecOrDie() Expect(out).NotTo(ContainSubstring(imageName))