From 95d21fa7682e88a642e6847f33f10569695e72af Mon Sep 17 00:00:00 2001
From: Guillaume Tardif <guillaume.tardif@gmail.com>
Date: Mon, 1 Feb 2021 12:08:40 +0100
Subject: [PATCH] First kube e2e. Adapted context create kubernetes command to
allow non interactive mode.
Signed-off-by: Guillaume Tardif <guillaume.tardif@gmail.com>
---
.github/workflows/kube-tests.yml | 63 ++++++++++++++++++++++++++++++++
Makefile | 3 ++
cli/cmd/context/create_kube.go | 3 +-
kube/context.go | 21 +++++++----
kube/e2e/compose_test.go | 45 ++++++++++++++++-------
utils/e2e/framework.go | 31 ++++++++++++++++
6 files changed, 145 insertions(+), 21 deletions(-)
create mode 100644 .github/workflows/kube-tests.yml
diff --git a/.github/workflows/kube-tests.yml b/.github/workflows/kube-tests.yml
new file mode 100644
index 000000000..a39d8c6b4
--- /dev/null
+++ b/.github/workflows/kube-tests.yml
@@ -0,0 +1,63 @@
+name: Kube integration tests
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+
+jobs:
+ check-optional-tests:
+ name: Check if needs to run Kube tests
+ runs-on: ubuntu-latest
+ outputs:
+ trigger-kube: ${{steps.runkubetest.outputs.triggered}}
+ steps:
+ - uses: khan/pull-request-comment-trigger@master
+ name: Check if test Kube
+ if: github.event_name == 'pull_request'
+ id: runkubetest
+ with:
+ trigger: '/test-kube'
+
+
+ kube-tests:
+ name: Kube e2e tests
+ runs-on: ubuntu-latest
+ env:
+ GO111MODULE: "on"
+ needs: check-optional-tests
+ if: github.ref == 'refs/heads/main' || needs.check-optional-tests.outputs.trigger-kube == 'true'
+ steps:
+ - name: Set up Go 1.15
+ uses: actions/setup-go@v1
+ with:
+ go-version: 1.15
+ id: go
+
+ - name: Setup docker CLI
+ run: |
+ curl https://download.docker.com/linux/static/stable/x86_64/docker-20.10.2.tgz | tar xz
+ sudo cp ./docker/docker /usr/bin/ && rm -rf docker && docker version
+
+ - name: Setup Kube tools
+ run: |
+ sudo apt-get install jq && jq --version
+ curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.10.0/kind-linux-amd64 && chmod +x ./kind && sudo mv ./kind /usr/bin/ && kind version
+ curl -LO "https://dl.k8s.io/release/v1.20.2/bin/linux/amd64/kubectl" && sudo mv kubectl /usr/bin/ && kubectl version --client
+
+ - name: Checkout code into the Go module directory
+ uses: actions/checkout@v2
+
+ - uses: actions/cache@v2
+ with:
+ path: ~/go/pkg/mod
+ key: go-${{ hashFiles('**/go.sum') }}
+
+ - name: Build for Kube e2e tests
+ env:
+ BUILD_TAGS: kube
+ run: make -f builder.Makefile cli
+
+ - name: Kube e2e Test
+ run: make e2e-kube
diff --git a/Makefile b/Makefile
index 45486e63a..4c168a15d 100644
--- a/Makefile
+++ b/Makefile
@@ -49,6 +49,9 @@ e2e-local: ## Run End to end local tests. Set E2E_TEST=TestName to run a single
e2e-win-ci: ## Run end to end local tests on Windows CI, no Docker for Linux containers available ATM. Set E2E_TEST=TestName to run a single test
go test -count=1 -v $(TEST_FLAGS) ./local/e2e/cli-only
+e2e-kube: ## Run End to end Kube tests. Set E2E_TEST=TestName to run a single test
+ go test -timeout 10m -count=1 -v $(TEST_FLAGS) ./kube/e2e
+
e2e-aci: ## Run End to end ACI tests. Set E2E_TEST=TestName to run a single test
go test -timeout 15m -count=1 -v $(TEST_FLAGS) ./aci/e2e
diff --git a/cli/cmd/context/create_kube.go b/cli/cmd/context/create_kube.go
index b42370e9a..609716e38 100644
--- a/cli/cmd/context/create_kube.go
+++ b/cli/cmd/context/create_kube.go
@@ -50,7 +50,8 @@ func createKubeCommand() *cobra.Command {
}
addDescriptionFlag(cmd, &opts.Description)
- cmd.Flags().StringVar(&opts.KubeconfigPath, "kubeconfig", "", "The endpoint of the Kubernetes manager")
+ cmd.Flags().StringVar(&opts.KubeConfigPath, "kubeconfig", "", "The endpoint of the Kubernetes manager")
+ cmd.Flags().StringVar(&opts.KubeContextName, "kubecontext", "", "The name of the context to use in kubeconfig")
cmd.Flags().BoolVar(&opts.FromEnvironment, "from-env", false, "Get endpoint and creds from env vars")
return cmd
}
diff --git a/kube/context.go b/kube/context.go
index 0b9201b00..a9b4f180b 100644
--- a/kube/context.go
+++ b/kube/context.go
@@ -29,9 +29,9 @@ import (
// ContextParams options for creating a Kubernetes context
type ContextParams struct {
- ContextName string
+ KubeContextName string
Description string
- KubeconfigPath string
+ KubeConfigPath string
FromEnvironment bool
}
@@ -45,7 +45,7 @@ func (cp ContextParams) CreateContextData() (interface{}, string, error) {
}
user := prompt.User{}
selectContext := func() error {
- contexts, err := kubernetes.ListAvailableKubeConfigContexts(cp.KubeconfigPath)
+ contexts, err := kubernetes.ListAvailableKubeConfigContexts(cp.KubeConfigPath)
if err != nil {
return err
}
@@ -57,11 +57,18 @@ func (cp ContextParams) CreateContextData() (interface{}, string, error) {
}
return err
}
- cp.ContextName = contexts[selected]
+ cp.KubeContextName = contexts[selected]
return nil
}
- if cp.KubeconfigPath != "" {
+ if cp.KubeConfigPath != "" {
+ if cp.KubeContextName != "" {
+ return store.KubeContext{
+ ContextName: cp.KubeContextName,
+ KubeconfigPath: cp.KubeConfigPath,
+ FromEnvironment: cp.FromEnvironment,
+ }, cp.Description, nil
+ }
err := selectContext()
if err != nil {
return nil, "", err
@@ -95,8 +102,8 @@ func (cp ContextParams) CreateContextData() (interface{}, string, error) {
}
}
return store.KubeContext{
- ContextName: cp.ContextName,
- KubeconfigPath: cp.KubeconfigPath,
+ ContextName: cp.KubeContextName,
+ KubeconfigPath: cp.KubeConfigPath,
FromEnvironment: cp.FromEnvironment,
}, cp.Description, nil
}
diff --git a/kube/e2e/compose_test.go b/kube/e2e/compose_test.go
index d435310c5..688843e1c 100644
--- a/kube/e2e/compose_test.go
+++ b/kube/e2e/compose_test.go
@@ -18,8 +18,8 @@ package e2e
import (
"fmt"
- "net/http"
"os"
+ "path/filepath"
"strings"
"testing"
"time"
@@ -48,31 +48,50 @@ func TestComposeUp(t *testing.T) {
c := NewParallelE2eCLI(t, binDir)
const projectName = "compose-kube-demo"
+ kubeconfig := filepath.Join(c.ConfigDir, "kubeconfig")
+ kindClusterName := "e2e"
+ kubeContextName := "kind-" + kindClusterName
+ dockerContextName := "kube-e2e-ctx"
+
+ t.Run("create kube cluster", func(t *testing.T) {
+ c.RunCmd("kind", "create", "cluster", "--name", kindClusterName, "--kubeconfig", kubeconfig, "--wait", "180s")
+ })
+ defer func() {
+ c.RunDockerCmd("context", "use", "default")
+ c.RunCmd("kind", "delete", "cluster", "--name", kindClusterName, "--kubeconfig", kubeconfig)
+ }()
t.Run("create kube context", func(t *testing.T) {
- res := c.RunDockerCmd("context", "create", "kubernetes", "--kubeconfig", "/Users/gtardif/.kube/config", "--kubecontext", "docker-desktop", "kube-e2e")
- res.Assert(t, icmd.Expected{Out: `Successfully created kube context "kube-e2e"`})
- c.RunDockerCmd("context", "use", "kube-e2e")
+ res := c.RunDockerCmd("context", "create", "kubernetes", "--kubeconfig", kubeconfig, "--kubecontext", kubeContextName, dockerContextName)
+ res.Assert(t, icmd.Expected{Out: fmt.Sprintf("Successfully created kube context %q", dockerContextName)})
+ c.RunDockerCmd("context", "use", dockerContextName)
})
t.Run("up", func(t *testing.T) {
c.RunDockerCmd("compose", "-f", "./kube-simple-demo/demo_sentences.yaml", "--project-name", projectName, "up", "-d")
})
- t.Run("check running project", func(t *testing.T) {
- res := c.RunDockerCmd("compose", "-p", projectName, "ps")
- res.Assert(t, icmd.Expected{Out: `web`})
-
- endpoint := "http://localhost:95"
- output := HTTPGetWithRetry(t, endpoint+"/words/noun", http.StatusOK, 2*time.Second, 20*time.Second)
- assert.Assert(t, strings.Contains(output, `"word":`))
+ t.Run("compose ls", func(t *testing.T) {
+ res := c.RunDockerCmd("compose", "ls", "--format", "json")
+ res.Assert(t, icmd.Expected{Out: `[{"Name":"compose-kube-demo","Status":"deployed"}]`})
})
+
+ t.Run("check running project", func(t *testing.T) {
+ // Docker Desktop kube cluster automatically exposes ports on the host, this is not the case with kind on Desktop,
+ //we need to connect to the clusterIP, from the kind container
+ res := c.RunCmd("sh", "-c", "kubectl --kubeconfig "+kubeconfig+" get service/web -o json | jq -r '.spec.clusterIP'")
+ clusterIP := strings.ReplaceAll(strings.TrimSpace(res.Stdout()), `"`, "")
+
+ endpoint := fmt.Sprintf("http://%s:80/words/noun", clusterIP)
+ c.WaitForCmdResult(icmd.Command("docker", "--context", "default", "exec", "e2e-control-plane", "curl", endpoint), StdoutContains(`"word":`), 3*time.Minute, 3*time.Second)
+ })
+
t.Run("down", func(t *testing.T) {
_ = c.RunDockerCmd("compose", "--project-name", projectName, "down")
})
- t.Run("check containers after down", func(t *testing.T) {
- res := c.RunDockerCmd("ps", "--all")
+ t.Run("check stack after down", func(t *testing.T) {
+ res := c.RunDockerCmd("compose", "ls")
assert.Assert(t, !strings.Contains(res.Combined(), projectName), res.Combined())
})
}
diff --git a/utils/e2e/framework.go b/utils/e2e/framework.go
index a525bb9c8..ce9360e4a 100644
--- a/utils/e2e/framework.go
+++ b/utils/e2e/framework.go
@@ -203,6 +203,15 @@ func (c *E2eCLI) RunDockerOrExitError(args ...string) *icmd.Result {
return icmd.RunCmd(c.NewDockerCmd(args...))
}
+// RunCmd runs a command, expects no error and returns a result
+func (c *E2eCLI) RunCmd(args ...string) *icmd.Result {
+ fmt.Printf(" [%s] %s\n", c.test.Name(), strings.Join(args, " "))
+ assert.Assert(c.test, len(args) >= 1, "require at least one command in parameters")
+ res := icmd.RunCmd(c.NewCmd(args[0], args[1:]...))
+ res.Assert(c.test, icmd.Success)
+ return res
+}
+
// RunDockerCmd runs a docker command, expects no error and returns a result
func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
res := c.RunDockerOrExitError(args...)
@@ -210,6 +219,28 @@ func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
return res
}
+// StdoutContains returns a predicate on command result expecting a string in stdout
+func StdoutContains(expected string) func(*icmd.Result) bool {
+ return func(res *icmd.Result) bool {
+ return strings.Contains(res.Stdout(), expected)
+ }
+}
+
+// WaitForCmdResult try to execute a cmd until resulting output matches given predicate
+func (c *E2eCLI) WaitForCmdResult(command icmd.Cmd, predicate func(*icmd.Result) bool, timeout time.Duration, delay time.Duration) {
+ assert.Assert(c.test, timeout.Nanoseconds() > delay.Nanoseconds(), "timeout must be greater than delay")
+ var res *icmd.Result
+ checkStopped := func(logt poll.LogT) poll.Result {
+ fmt.Printf(" [%s] %s\n", c.test.Name(), strings.Join(command.Command, " "))
+ res = icmd.RunCmd(command)
+ if !predicate(res) {
+ return poll.Continue("Cmd output did not match requirement: %q", res.Combined())
+ }
+ return poll.Success()
+ }
+ poll.WaitOn(c.test, checkStopped, poll.WithDelay(delay), poll.WithTimeout(timeout))
+}
+
// PathEnvVar returns path (os sensitive) for running test
func (c *E2eCLI) PathEnvVar() string {
path := c.BinDir + ":" + os.Getenv("PATH")