Merge pull request #544 from docker/simulation

Introduce ECS emulation mode
2020-09-01 15:18:32 +02:00 · 2020-09-01 15:18:32 +02:00 · c7e2db077e
parent eb0af4b711 4d11594df0
commit c7e2db077e
11 changed files with 355 additions and 9 deletions
--- a/cli/cmd/compose/up.go
+++ b/cli/cmd/compose/up.go
@ -40,7 +40,6 @@ func upCommand() *cobra.Command {
 	upCmd.Flags().StringArrayVarP(&opts.ConfigPaths, "file", "f", []string{}, "Compose configuration files")
 	upCmd.Flags().StringArrayVarP(&opts.Environment, "environment", "e", []string{}, "Environment variables")
 	upCmd.Flags().BoolP("detach", "d", true, " Detached mode: Run containers in the background")
-
 	return upCmd
 }

--- a/cli/cmd/context/create_ecs.go
+++ b/cli/cmd/context/create_ecs.go
@ -38,17 +38,22 @@ $ docker context create ecs CONTEXT [flags]
 }

 func createEcsCommand() *cobra.Command {
+	var localSimulation bool
 	var opts ecs.ContextParams
 	cmd := &cobra.Command{
 		Use:   "ecs CONTEXT [flags]",
 		Short: "Create a context for Amazon ECS",
 		Args:  cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if localSimulation {
+				return runCreateLocalSimulation(cmd.Context(), args[0], opts)
+			}
 			return runCreateEcs(cmd.Context(), args[0], opts)
 		},
 	}

 	addDescriptionFlag(cmd, &opts.Description)
+	cmd.Flags().BoolVar(&localSimulation, "local-simulation", false, "Create context for ECS local simulation endpoints")
 	cmd.Flags().StringVar(&opts.Profile, "profile", "", "Profile")
 	cmd.Flags().StringVar(&opts.Region, "region", "", "Region")
 	cmd.Flags().StringVar(&opts.AwsID, "key-id", "", "AWS Access Key ID")
@ -56,6 +61,21 @@ func createEcsCommand() *cobra.Command {
 	return cmd
 }

+func runCreateLocalSimulation(ctx context.Context, contextName string, opts ecs.ContextParams) error {
+	if contextExists(ctx, contextName) {
+		return errors.Wrapf(errdefs.ErrAlreadyExists, "context %q", contextName)
+	}
+	cs, err := client.GetCloudService(ctx, store.EcsLocalSimulationContextType)
+	if err != nil {
+		return errors.Wrap(err, "cannot connect to ECS backend")
+	}
+	data, description, err := cs.CreateContextData(ctx, opts)
+	if err != nil {
+		return err
+	}
+	return createDockerContext(ctx, contextName, store.EcsLocalSimulationContextType, description, data)
+}
+
 func runCreateEcs(ctx context.Context, contextName string, opts ecs.ContextParams) error {
 	if contextExists(ctx, contextName) {
 		return errors.Wrapf(errdefs.ErrAlreadyExists, "context %q", contextName)
@ -71,7 +91,7 @@ func runCreateEcs(ctx context.Context, contextName string, opts ecs.ContextParam
 func getEcsContextData(ctx context.Context, opts ecs.ContextParams) (interface{}, string, error) {
 	cs, err := client.GetCloudService(ctx, store.EcsContextType)
 	if err != nil {
-		return nil, "", errors.Wrap(err, "cannot connect to AWS backend")
+		return nil, "", errors.Wrap(err, "cannot connect to ECS backend")
 	}
 	return cs.CreateContextData(ctx, opts)
 }
--- a/cli/main.go
+++ b/cli/main.go
@ -27,6 +27,8 @@ import (
 	"syscall"
 	"time"

+	"github.com/docker/compose-cli/cli/cmd/compose"
+
 	"github.com/docker/compose-cli/cli/cmd/logout"

 	"github.com/docker/compose-cli/errdefs"
@ -38,12 +40,12 @@ import (
 	// Backend registrations
 	_ "github.com/docker/compose-cli/aci"
 	_ "github.com/docker/compose-cli/ecs"
+	_ "github.com/docker/compose-cli/ecs/local"
 	_ "github.com/docker/compose-cli/example"
 	_ "github.com/docker/compose-cli/local"
 	"github.com/docker/compose-cli/metrics"

 	"github.com/docker/compose-cli/cli/cmd"
-	"github.com/docker/compose-cli/cli/cmd/compose"
 	contextcmd "github.com/docker/compose-cli/cli/cmd/context"
 	"github.com/docker/compose-cli/cli/cmd/login"
 	"github.com/docker/compose-cli/cli/cmd/run"
@ -121,12 +123,12 @@ func main() {
 		cmd.RmCommand(),
 		cmd.StartCommand(),
 		cmd.InspectCommand(),
-		compose.Command(),
 		login.Command(),
 		logout.Command(),
 		cmd.VersionCommand(version),
 		cmd.StopCommand(),
 		cmd.SecretCommand(),
+		compose.Command(),

 		// Place holders
 		cmd.EcsCommand(),
--- a/context/store/store.go
+++ b/context/store/store.go
@ -44,6 +44,11 @@ const (
 	// EcsContextType is the endpoint key in the context endpoints for an ECS
 	// backend
 	EcsContextType = "ecs"
+
+	// EcsLocalSimulationContextType is the endpoint key in the context endpoints for an ECS backend
+	// running local simulation endpoints
+	EcsLocalSimulationContextType = "ecs-local"
+
 	// AciContextType is the endpoint key in the context endpoints for an ACI
 	// backend
 	AciContextType = "aci"
--- a/ecs/backend.go
+++ b/ecs/backend.go
@ -22,8 +22,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"

-	"github.com/docker/compose-cli/secrets"
-
 	"github.com/docker/compose-cli/backend"
 	"github.com/docker/compose-cli/compose"
 	"github.com/docker/compose-cli/containers"
@ -31,6 +29,7 @@ import (
 	"github.com/docker/compose-cli/context/cloud"
 	"github.com/docker/compose-cli/context/store"
 	"github.com/docker/compose-cli/errdefs"
+	"github.com/docker/compose-cli/secrets"
 )

 const backendType = store.EcsContextType
--- a/ecs/local/backend.go
+++ b/ecs/local/backend.go
@ -0,0 +1,67 @@
+/*
+   Copyright 2020 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package local
+
+import (
+	"context"
+
+	"github.com/docker/compose-cli/compose"
+	"github.com/docker/compose-cli/containers"
+	"github.com/docker/compose-cli/secrets"
+	"github.com/docker/docker/client"
+
+	"github.com/docker/compose-cli/backend"
+	"github.com/docker/compose-cli/context/cloud"
+	"github.com/docker/compose-cli/context/store"
+)
+
+const backendType = store.EcsLocalSimulationContextType
+
+func init() {
+	backend.Register(backendType, backendType, service, getCloudService)
+}
+
+type ecsLocalSimulation struct {
+	moby *client.Client
+}
+
+func service(ctx context.Context) (backend.Service, error) {
+	apiClient, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ecsLocalSimulation{
+		moby: apiClient,
+	}, nil
+}
+
+func getCloudService() (cloud.Service, error) {
+	return ecsLocalSimulation{}, nil
+}
+
+func (e ecsLocalSimulation) ContainerService() containers.Service {
+	return nil
+}
+
+func (e ecsLocalSimulation) SecretsService() secrets.Service {
+	return nil
+}
+
+func (e ecsLocalSimulation) ComposeService() compose.Service {
+	return e
+}
--- a/ecs/local/compose.go
+++ b/ecs/local/compose.go
@ -0,0 +1,145 @@
+/*
+   Copyright 2020 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package local
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/compose-cli/compose"
+	"github.com/docker/compose-cli/errdefs"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/compose-spec/compose-go/types"
+	"github.com/pkg/errors"
+	"github.com/sanathkr/go-yaml"
+	"golang.org/x/mod/semver"
+)
+
+func (e ecsLocalSimulation) Up(ctx context.Context, project *types.Project) error {
+	cmd := exec.Command("docker-compose", "version", "--short")
+	b := bytes.Buffer{}
+	b.WriteString("v")
+	cmd.Stdout = bufio.NewWriter(&b)
+	err := cmd.Run()
+	if err != nil {
+		return errors.Wrap(err, "ECS simulation mode require Docker-compose 1.27")
+	}
+	version := semver.MajorMinor(strings.TrimSpace(b.String()))
+	if version == "" {
+		return fmt.Errorf("can't parse docker-compose version: %s", b.String())
+	}
+	if semver.Compare(version, "v1.27") < 0 {
+		return fmt.Errorf("ECS simulation mode require Docker-compose 1.27, found %s", version)
+	}
+
+	converted, err := e.Convert(ctx, project)
+	if err != nil {
+		return err
+	}
+
+	cmd = exec.Command("docker-compose", "--context", "default", "--project-directory", project.WorkingDir, "--project-name", project.Name, "-f", "-", "up")
+	cmd.Stdin = strings.NewReader(string(converted))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func (e ecsLocalSimulation) Convert(ctx context.Context, project *types.Project) ([]byte, error) {
+	project.Networks["credentials_network"] = types.NetworkConfig{
+		Driver: "bridge",
+		Ipam: types.IPAMConfig{
+			Config: []*types.IPAMPool{
+				{
+					Subnet:  "169.254.170.0/24",
+					Gateway: "169.254.170.1",
+				},
+			},
+		},
+	}
+
+	// On Windows, this directory can be found at "%UserProfile%\.aws"
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return nil, err
+	}
+
+	for i, service := range project.Services {
+		service.Networks["credentials_network"] = &types.ServiceNetworkConfig{
+			Ipv4Address: fmt.Sprintf("169.254.170.%d", i+3),
+		}
+		service.DependsOn = append(service.DependsOn, "ecs-local-endpoints")
+		service.Environment["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = aws.String("/creds")
+		service.Environment["ECS_CONTAINER_METADATA_URI"] = aws.String("http://169.254.170.2/v3")
+		project.Services[i] = service
+	}
+
+	project.Services = append(project.Services, types.ServiceConfig{
+		Name:  "ecs-local-endpoints",
+		Image: "amazon/amazon-ecs-local-container-endpoints",
+		Volumes: []types.ServiceVolumeConfig{
+			{
+				Type:   types.VolumeTypeBind,
+				Source: "/var/run",
+				Target: "/var/run",
+			},
+			{
+				Type:   types.VolumeTypeBind,
+				Source: filepath.Join(home, ".aws"),
+				Target: "/home/.aws",
+			},
+		},
+		Environment: map[string]*string{
+			"HOME":        aws.String("/home"),
+			"AWS_PROFILE": aws.String("default"),
+		},
+		Networks: map[string]*types.ServiceNetworkConfig{
+			"credentials_network": {
+				Ipv4Address: "169.254.170.2",
+			},
+		},
+	})
+
+	delete(project.Networks, "default")
+	config := map[string]interface{}{
+		"services": project.Services,
+		"networks": project.Networks,
+		"volumes":  project.Volumes,
+		"secrets":  project.Secrets,
+		"configs":  project.Configs,
+	}
+	return yaml.Marshal(config)
+}
+
+func (e ecsLocalSimulation) Down(ctx context.Context, projectName string) error {
+	return errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose down")
+}
+
+func (e ecsLocalSimulation) Logs(ctx context.Context, projectName string, w io.Writer) error {
+	return errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose logs")
+}
+
+func (e ecsLocalSimulation) Ps(ctx context.Context, projectName string) ([]compose.ServiceStatus, error) {
+	return nil, errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose ps")
+}
--- a/ecs/local/context.go
+++ b/ecs/local/context.go
@ -0,0 +1,43 @@
+/*
+   Copyright 2020 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package local
+
+import (
+	"context"
+
+	"github.com/docker/compose-cli/context/cloud"
+	"github.com/docker/compose-cli/ecs"
+	"github.com/docker/compose-cli/errdefs"
+)
+
+var _ cloud.Service = ecsLocalSimulation{}
+
+func (e ecsLocalSimulation) Login(ctx context.Context, params interface{}) error {
+	return errdefs.ErrNotImplemented
+}
+
+func (e ecsLocalSimulation) Logout(ctx context.Context) error {
+	return errdefs.ErrNotImplemented
+}
+
+func (e ecsLocalSimulation) CreateContextData(ctx context.Context, params interface{}) (contextData interface{}, description string, err error) {
+	opts := params.(ecs.ContextParams)
+	if opts.Description == "" {
+		opts.Description = "ECS local endpoints"
+	}
+	return struct{}{}, opts.Description, nil
+}
--- a/go.mod
+++ b/go.mod
@ -22,7 +22,7 @@ require (
 	github.com/aws/aws-sdk-go v1.34.8
 	github.com/awslabs/goformation/v4 v4.14.0
 	github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
-	github.com/compose-spec/compose-go v0.0.0-20200818070525-eb1188aae4a2
+	github.com/compose-spec/compose-go v0.0.0-20200824075806-a70cd5945c25
 	github.com/containerd/console v1.0.0
 	github.com/containerd/containerd v1.3.5 // indirect
 	github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8
@ -46,11 +46,13 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b
 	github.com/sirupsen/logrus v1.6.0
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
+	golang.org/x/mod v0.3.0
 	golang.org/x/net v0.0.0-20200625001655-4c5254603344
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
--- a/go.sum
+++ b/go.sum
@ -87,8 +87,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/compose-spec/compose-go v0.0.0-20200818070525-eb1188aae4a2 h1:b3JmHJVJt8zXy112yGtRq74G32sPQ8XLJxfHKaP/DOg=
-github.com/compose-spec/compose-go v0.0.0-20200818070525-eb1188aae4a2/go.mod h1:P7PZ0svgjrZ8nv/XvxObbl8o0DCIE9ZbL8pllg6uL4w=
+github.com/compose-spec/compose-go v0.0.0-20200824075806-a70cd5945c25 h1:mVlGrHJuNGPJNEvCCIrDIZX5FYtNTwFd++y+fJaGTXM=
+github.com/compose-spec/compose-go v0.0.0-20200824075806-a70cd5945c25/go.mod h1:P7PZ0svgjrZ8nv/XvxObbl8o0DCIE9ZbL8pllg6uL4w=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
@ -417,6 +417,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
--- a/tests/ecs-local-e2e/context_test.go
+++ b/tests/ecs-local-e2e/context_test.go
@ -0,0 +1,63 @@
+/*
+   Copyright 2020 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	. "github.com/docker/compose-cli/tests/framework"
+	"gotest.tools/v3/icmd"
+)
+
+const (
+	contextName = "ecs-local-test"
+)
+
+var binDir string
+
+func TestMain(m *testing.M) {
+	p, cleanup, err := SetupExistingCLI()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	binDir = p
+	exitCode := m.Run()
+	cleanup()
+	os.Exit(exitCode)
+}
+
+func TestCreateContext(t *testing.T) {
+	c := NewE2eCLI(t, binDir)
+
+	t.Run("create context", func(t *testing.T) {
+		c.RunDockerCmd("context", "create", "ecs", contextName, "--local-simulation")
+		res := c.RunDockerCmd("context", "use", contextName)
+		res.Assert(t, icmd.Expected{Out: contextName})
+		res = c.RunDockerCmd("context", "ls")
+		res.Assert(t, icmd.Expected{Out: contextName + " *"})
+	})
+	t.Run("delete context", func(t *testing.T) {
+		res := c.RunDockerCmd("context", "use", "default")
+		res.Assert(t, icmd.Expected{Out: "default"})
+
+		res = c.RunDockerCmd("context", "rm", contextName)
+		res.Assert(t, icmd.Expected{Out: contextName})
+	})
+}