Create CloudFormation template with parameters

so we don't need AWS API to resolve IDs and can run conversion offline

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

ecs/cmd/commands/compose.go

@@ -49,7 +49,7 @@ func ConvertCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions)
 			if err != nil {
 				return err
 			}
-			template, err := client.Convert(context.Background(), project)
+			template, err := client.Convert(project)
 			if err != nil {
 				return err
 			}

ecs/go.sum

@@ -305,6 +305,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz+cjWoom6l0=

ecs/pkg/amazon/api.go

@@ -5,6 +5,5 @@ package amazon
 type API interface {
 	downAPI
 	upAPI
-	convertAPI
 	secretsAPI
 }

ecs/pkg/amazon/cloudformation.go

@@ -1,8 +1,6 @@
 package amazon
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"strings"
 
@@ -19,20 +17,31 @@ import (
 	"github.com/docker/ecs-plugin/pkg/compose"
 )
 
-func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) {
+func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) {
 	template := cloudformation.NewTemplate()
-	vpc, err := c.GetVPC(ctx, project)
+	template.Parameters["VPCId"] = cloudformation.Parameter{
-	if err != nil {
+		Type:        "AWS::EC2::VPC::Id",
-		return nil, err
+		Description: "ID of the VPC",
 	}
 
-	subnets, err := c.api.GetSubNets(ctx, vpc)
+	/*
-	if err != nil {
+		FIXME can't set subnets: Ref("SubnetIds") see https://github.com/awslabs/goformation/issues/282
-		return nil, err
+		template.Parameters["SubnetIds"] = cloudformation.Parameter{
+			Type:        "List<AWS::EC2::Subnet::Id>",
+			Description: "The list of SubnetIds, for at least two Availability Zones in the region in your VPC",
+		}
+	*/
+	template.Parameters["Subnet1Id"] = cloudformation.Parameter{
+		Type:        "AWS::EC2::Subnet::Id",
+		Description: "SubnetId,for Availability Zone 1 in the region in your VPC",
+	}
+	template.Parameters["Subnet2Id"] = cloudformation.Parameter{
+		Type:        "AWS::EC2::Subnet::Id",
+		Description: "SubnetId,for Availability Zone 1 in the region in your VPC",
 	}
 
 	for net := range project.Networks {
-		name, resource := convertNetwork(project, net, vpc)
+		name, resource := convertNetwork(project, net, cloudformation.Ref("VPCId"))
 		template.Resources[name] = resource
 	}
 
@@ -45,7 +54,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo
 	template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{
 		Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name),
 		Name:        fmt.Sprintf("%s.local", project.Name),
-		Vpc:         vpc,
+		Vpc:         cloudformation.Ref("VPCId"),
 	}
 
 	for _, service := range project.Services {
@@ -55,7 +64,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo
 		}
 
 		taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", service.Name)
-		policy, err := c.getPolicy(ctx, definition)
+		policy, err := c.getPolicy(definition)
 		if err != nil {
 			return nil, err
 		}
@@ -115,7 +124,10 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo
 				AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{
 					AssignPublicIp: ecsapi.AssignPublicIpEnabled,
 					SecurityGroups: serviceSecurityGroups,
-					Subnets:        subnets,
+					Subnets: []string{
+						cloudformation.Ref("Subnet1Id"),
+						cloudformation.Ref("Subnet2Id"),
+					},
 				},
 			},
 			SchedulingStrategy: ecsapi.SchedulingStrategyReplica,
@@ -171,29 +183,7 @@ func networkResourceName(project *compose.Project, network string) string {
 	return fmt.Sprintf("%s%sNetwork", project.Name, strings.Title(network))
 }
 
-func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) {
+func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) {
-	//check compose file for the default external network
-	if net, ok := project.Networks["default"]; ok {
-		if net.External.External {
-			vpc := net.Name
-			ok, err := c.api.VpcExists(ctx, vpc)
-			if err != nil {
-				return "", err
-			}
-			if !ok {
-				return "", errors.New("Vpc does not exist: " + vpc)
-			}
-			return vpc, nil
-		}
-	}
-	defaultVPC, err := c.api.GetDefaultVPC(ctx)
-	if err != nil {
-		return "", err
-	}
-	return defaultVPC, nil
-}
-
-func (c client) getPolicy(ctx context.Context, taskDef *ecs.TaskDefinition) (*PolicyDocument, error) {
 
 	arns := []string{}
 	for _, container := range taskDef.ContainerDefinitions {
@@ -212,17 +202,10 @@ func (c client) getPolicy(ctx context.Context, taskDef *ecs.TaskDefinition) (*Po
 			Statement: []PolicyStatement{
 				{
 					Effect:   "Allow",
-					Action:   []string{"secretsmanager:GetSecretValue", "ssm:GetParameters", "kms:Decrypt"},
+					Action:   []string{ActionGetSecretValue, ActionGetParameters, ActionDecrypt},
 					Resource: arns,
 				}},
 		}, nil
 	}
 	return nil, nil
 }
-
-type convertAPI interface {
-	GetDefaultVPC(ctx context.Context) (string, error)
-	VpcExists(ctx context.Context, vpcID string) (bool, error)
-	GetSubNets(ctx context.Context, vpcID string) ([]string, error)
-	GetRoleArn(ctx context.Context, name string) (string, error)
-}

ecs/pkg/amazon/iam.go

@@ -1,6 +1,12 @@
 package amazon
 
-const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+const (
+	ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+
+	ActionGetSecretValue = "secretsmanager:GetSecretValue"
+	ActionGetParameters  = "ssm:GetParameters"
+	ActionDecrypt        = "kms:Decrypt"
+)
 
 var assumeRolePolicyDocument = PolicyDocument{
 	Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html

ecs/pkg/amazon/mock/api.go

@@ -6,12 +6,11 @@ package mock
 
 import (
 	context "context"
-	reflect "reflect"
-
 	cloudformation "github.com/aws/aws-sdk-go/service/cloudformation"
 	cloudformation0 "github.com/awslabs/goformation/v4/cloudformation"
 	docker "github.com/docker/ecs-plugin/pkg/docker"
 	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
 )
 
 // MockAPI is a mock of API interface
@@ -77,23 +76,23 @@ func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 docker.Secret) (string
 }
 
 // CreateSecret indicates an expected call of CreateSecret
-func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 docker.Secret) *gomock.Call {
+func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecret", reflect.TypeOf((*MockAPI)(nil).CreateSecret), arg0, arg1)
 }
 
 // CreateStack mocks base method
-func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template) error {
+func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template, arg3 map[string]string) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2)
+	ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // CreateStack indicates an expected call of CreateStack
-func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2, arg3)
 }
 
 // DeleteCluster mocks base method
@@ -168,21 +167,6 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC), arg0)
 }
 
-// GetRoleArn mocks base method
-func (m *MockAPI) GetRoleArn(arg0 context.Context, arg1 string) (string, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetRoleArn", arg0, arg1)
-	ret0, _ := ret[0].(string)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// GetRoleArn indicates an expected call of GetRoleArn
-func (mr *MockAPIMockRecorder) GetRoleArn(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), arg0, arg1)
-}
-
 // GetStackID mocks base method
 func (m *MockAPI) GetStackID(arg0 context.Context, arg1 string) (string, error) {
 	m.ctrl.T.Helper()

ecs/pkg/amazon/sdk.go

@@ -153,17 +153,27 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) {
 	return len(stacks.Stacks) > 0, nil
 }
 
-func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template) error {
+func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template, parameters map[string]string) error {
 	logrus.Debug("Create CloudFormation stack")
 	json, err := template.JSON()
 	if err != nil {
 		return err
 	}
 
+	param := []*cloudformation.Parameter{}
+	for name, value := range parameters {
+		param = append(param, &cloudformation.Parameter{
+			ParameterKey:     aws.String(name),
+			ParameterValue:   aws.String(value),
+			UsePreviousValue: aws.Bool(true),
+		})
+	}
+
 	_, err = s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
 		OnFailure:        aws.String("DELETE"),
 		StackName:        aws.String(name),
 		TemplateBody:     aws.String(string(json)),
+		Parameters:       param,
 		TimeoutInMinutes: aws.Int64(10),
 		Capabilities: []*string{
 			aws.String(cloudformation.CapabilityCapabilityIam),

ecs/pkg/amazon/up.go

@@ -29,12 +29,28 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error
 		return err
 	}
 
-	template, err := c.Convert(ctx, project)
+	template, err := c.Convert(project)
 	if err != nil {
 		return err
 	}
 
-	err = c.api.CreateStack(ctx, project.Name, template)
+	vpc, err := c.GetVPC(ctx, project)
+	if err != nil {
+		return err
+	}
+
+	subNets, err := c.api.GetSubNets(ctx, vpc)
+	if err != nil {
+		return err
+	}
+
+	parameters := map[string]string{
+		"VPCId":     vpc,
+		"Subnet1Id": subNets[0],
+		"Subnet2Id": subNets[1],
+	}
+
+	err = c.api.CreateStack(ctx, project.Name, template, parameters)
 	if err != nil {
 		return err
 	}
@@ -42,10 +58,36 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error
 	return c.WaitStackCompletion(ctx, project.Name, StackCreate)
 }
 
+func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) {
+	//check compose file for the default external network
+	if net, ok := project.Networks["default"]; ok {
+		if net.External.External {
+			vpc := net.Name
+			ok, err := c.api.VpcExists(ctx, vpc)
+			if err != nil {
+				return "", err
+			}
+			if !ok {
+				return "", fmt.Errorf("VPC does not exist: %s", vpc)
+			}
+			return vpc, nil
+		}
+	}
+	defaultVPC, err := c.api.GetDefaultVPC(ctx)
+	if err != nil {
+		return "", err
+	}
+	return defaultVPC, nil
+}
+
 type upAPI interface {
 	waitAPI
+	GetDefaultVPC(ctx context.Context) (string, error)
+	VpcExists(ctx context.Context, vpcID string) (bool, error)
+	GetSubNets(ctx context.Context, vpcID string) ([]string, error)
+
 	ClusterExists(ctx context.Context, name string) (bool, error)
 	CreateCluster(ctx context.Context, name string) (string, error)
 	StackExists(ctx context.Context, name string) (bool, error)
-	CreateStack(ctx context.Context, name string, template *cloudformation.Template) error
+	CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error
 }

ecs/pkg/compose/api.go

@@ -8,7 +8,7 @@ import (
 )
 
 type API interface {
-	Convert(ctx context.Context, project *Project) (*cloudformation.Template, error)
+	Convert(project *Project) (*cloudformation.Template, error)
 	ComposeUp(ctx context.Context, project *Project) error
 	ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error