tyler.durden
/
compose
mirror of https://github.com/docker/compose.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #869 from docker/s3 

publish Cloudformation template on s3
This commit is contained in:
Nicolas De loof 2020-11-10 15:45:31 +01:00 committed by GitHub
parent 1cc0ccb189 71ecbda48f
commit 16c3223623
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ecs/aws.go
View File

@ -42,8 +42,8 @@ type API interface {
GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error)
GetRoleArn(ctx context.Context, name string) (string, error)
StackExists(ctx context.Context, name string) (bool, error)
CreateStack(ctx context.Context, name string, template []byte) error
CreateChangeSet(ctx context.Context, name string, template []byte) (string, error)
CreateStack(ctx context.Context, name string, region string, template []byte) error
CreateChangeSet(ctx context.Context, name string, region string, template []byte) (string, error)
UpdateStack(ctx context.Context, changeset string) error
WaitStackComplete(ctx context.Context, name string, operation int) error
GetStackID(ctx context.Context, name string) (string, error)

16
ecs/aws_mock.go
View File

@ -66,18 +66,18 @@ func (mr *MockAPIMockRecorder) CheckVPC(arg0, arg1 interface{}) *gomock.Call {
}
// CreateChangeSet mocks base method
func (m *MockAPI) CreateChangeSet(arg0 context.Context, arg1 string, arg2 []byte) (string, error) {
func (m *MockAPI) CreateChangeSet(arg0 context.Context, arg1, arg2 string, arg3 []byte) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CreateChangeSet", arg0, arg1, arg2)
ret := m.ctrl.Call(m, "CreateChangeSet", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// CreateChangeSet indicates an expected call of CreateChangeSet
func (mr *MockAPIMockRecorder) CreateChangeSet(arg0, arg1, arg2 interface{}) *gomock.Call {
func (mr *MockAPIMockRecorder) CreateChangeSet(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateChangeSet", reflect.TypeOf((*MockAPI)(nil).CreateChangeSet), arg0, arg1, arg2)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateChangeSet", reflect.TypeOf((*MockAPI)(nil).CreateChangeSet), arg0, arg1, arg2, arg3)
}
// CreateCluster mocks base method
@ -126,17 +126,17 @@ func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 interface{}) *gomock.Call
}
// CreateStack mocks base method
func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 []byte) error {
func (m *MockAPI) CreateStack(arg0 context.Context, arg1, arg2 string, arg3 []byte) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2)
ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(error)
return ret0
}
// CreateStack indicates an expected call of CreateStack
func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2 interface{}) *gomock.Call {
func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2, arg3)
}
// DeleteAutoscalingGroup mocks base method

188
ecs/sdk.go
View File

@ -17,6 +17,7 @@
package ecs
import (
"bytes"
"context"
"encoding/json"
"fmt"
@ -30,6 +31,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
@ -48,26 +50,32 @@ import (
"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/iam/iamiface"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface"
"github.com/aws/aws-sdk-go/service/ssm"
"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-uuid"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type sdk struct {
ECS ecsiface.ECSAPI
EC2 ec2iface.EC2API
EFS efsiface.EFSAPI
ELB elbv2iface.ELBV2API
CW cloudwatchlogsiface.CloudWatchLogsAPI
IAM iamiface.IAMAPI
CF cloudformationiface.CloudFormationAPI
SM secretsmanageriface.SecretsManagerAPI
SSM ssmiface.SSMAPI
AG autoscalingiface.AutoScalingAPI
ECS ecsiface.ECSAPI
EC2 ec2iface.EC2API
EFS efsiface.EFSAPI
ELB elbv2iface.ELBV2API
CW cloudwatchlogsiface.CloudWatchLogsAPI
IAM iamiface.IAMAPI
CF cloudformationiface.CloudFormationAPI
SM secretsmanageriface.SecretsManagerAPI
SSM ssmiface.SSMAPI
AG autoscalingiface.AutoScalingAPI
S3 s3iface.S3API
uploader *s3manager.Uploader
}
// sdk implement API
@ -78,16 +86,18 @@ func newSDK(sess *session.Session) sdk {
request.AddToUserAgent(r, internal.ECSUserAgentName+"/"+internal.Version)
})
return sdk{
ECS: ecs.New(sess),
EC2: ec2.New(sess),
EFS: efs.New(sess),
ELB: elbv2.New(sess),
CW: cloudwatchlogs.New(sess),
IAM: iam.New(sess),
CF: cloudformation.New(sess),
SM: secretsmanager.New(sess),
SSM: ssm.New(sess),
AG: autoscaling.New(sess),
ECS: ecs.New(sess),
EC2: ec2.New(sess),
EFS: efs.New(sess),
ELB: elbv2.New(sess),
CW: cloudwatchlogs.New(sess),
IAM: iam.New(sess),
CF: cloudformation.New(sess),
SM: secretsmanager.New(sess),
SSM: ssm.New(sess),
AG: autoscaling.New(sess),
S3: s3.New(sess),
uploader: s3manager.NewUploader(sess),
}
}
@ -187,11 +197,9 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error
return nil, err
}
for _, subnet := range subnets.Subnets {
id := aws.StringValue(subnet.SubnetId)
logrus.Debugf("Found SubNet %s", id)
ids = append(ids, existingAWSResource{
arn: aws.StringValue(subnet.SubnetArn),
id: id,
id: aws.StringValue(subnet.SubnetId),
})
}
@ -226,39 +234,119 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) {
return len(stacks.Stacks) > 0, nil
}
func (s sdk) CreateStack(ctx context.Context, name string, template []byte) error {
logrus.Debug("Create CloudFormation stack")
type uploadedTemplateFunc func(body *string, url *string) (string, error)
_, err := s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
OnFailure: aws.String("DELETE"),
StackName: aws.String(name),
TemplateBody: aws.String(string(template)),
TimeoutInMinutes: nil,
Capabilities: []*string{
aws.String(cloudformation.CapabilityCapabilityIam),
},
Tags: []*cloudformation.Tag{
{
Key: aws.String(compose.ProjectTag),
Value: aws.String(name),
const cloudformationBytesLimit = 51200
func (s sdk) withTemplate(ctx context.Context, name string, template []byte, region string, fn uploadedTemplateFunc) (string, error) {
if len(template) < cloudformationBytesLimit {
return fn(aws.String(string(template)), nil)
}
logrus.Debug("Create s3 bucket to store cloudformation template")
var configuration *s3.CreateBucketConfiguration
if region != "us-east-1" {
configuration = &s3.CreateBucketConfiguration{
LocationConstraint: aws.String(region),
}
}
// CloudFormation will only allow URL from a same-region bucket
// to avoid conflicts we suffix bucket name by region, so we can create comparable buckets in other regions.
bucket := "com.docker.compose." + region
_, err := s.S3.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
CreateBucketConfiguration: configuration,
})
if err != nil {
ae, ok := err.(awserr.Error)
if !ok {
return "", err
}
if ae.Code() != s3.ErrCodeBucketAlreadyOwnedByYou {
return "", err
}
}
key, err := uuid.GenerateUUID()
if err != nil {
return "", err
}
upload, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Key: aws.String(key),
Body: bytes.NewReader(template),
Bucket: aws.String(bucket),
ContentType: aws.String("application/json"),
Tagging: aws.String(name),
})
if err != nil {
return "", err
}
defer s.S3.DeleteObjects(&s3.DeleteObjectsInput{ //nolint: errcheck
Bucket: aws.String(bucket),
Delete: &s3.Delete{
Objects: []*s3.ObjectIdentifier{
{
Key: aws.String(key),
VersionId: upload.VersionID,
},
},
},
})
return fn(nil, aws.String(upload.Location))
}
func (s sdk) CreateStack(ctx context.Context, name string, region string, template []byte) error {
logrus.Debug("Create CloudFormation stack")
stackID, err := s.withTemplate(ctx, name, template, region, func(body *string, url *string) (string, error) {
stack, err := s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
OnFailure: aws.String("DELETE"),
StackName: aws.String(name),
TemplateBody: body,
TemplateURL: url,
TimeoutInMinutes: nil,
Capabilities: []*string{
aws.String(cloudformation.CapabilityCapabilityIam),
},
Tags: []*cloudformation.Tag{
{
Key: aws.String(compose.ProjectTag),
Value: aws.String(name),
},
},
})
if err != nil {
return "", err
}
return aws.StringValue(stack.StackId), nil
})
logrus.Debugf("Stack %s created", stackID)
return err
}
func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte) (string, error) {
func (s sdk) CreateChangeSet(ctx context.Context, name string, region string, template []byte) (string, error) {
logrus.Debug("Create CloudFormation Changeset")
update := fmt.Sprintf("Update%s", time.Now().Format("2006-01-02-15-04-05"))
changeset, err := s.CF.CreateChangeSetWithContext(ctx, &cloudformation.CreateChangeSetInput{
ChangeSetName: aws.String(update),
ChangeSetType: aws.String(cloudformation.ChangeSetTypeUpdate),
StackName: aws.String(name),
TemplateBody: aws.String(string(template)),
Capabilities: []*string{
aws.String(cloudformation.CapabilityCapabilityIam),
},
changeset, err := s.withTemplate(ctx, name, template, region, func(body *string, url *string) (string, error) {
changeset, err := s.CF.CreateChangeSetWithContext(ctx, &cloudformation.CreateChangeSetInput{
ChangeSetName: aws.String(update),
ChangeSetType: aws.String(cloudformation.ChangeSetTypeUpdate),
StackName: aws.String(name),
TemplateBody: body,
TemplateURL: url,
Capabilities: []*string{
aws.String(cloudformation.CapabilityCapabilityIam),
},
})
if err != nil {
return "", err
}
return aws.StringValue(changeset.Id), err
})
if err != nil {
return "", err
@ -267,7 +355,7 @@ func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte)
// we have to WaitUntilChangeSetCreateComplete even this in fail with error `ResourceNotReady`
// so that we can invoke DescribeChangeSet to check status, and then we can know about the actual creation failure cause.
s.CF.WaitUntilChangeSetCreateCompleteWithContext(ctx, &cloudformation.DescribeChangeSetInput{ // nolint:errcheck
ChangeSetName: changeset.Id,
ChangeSetName: aws.String(changeset),
})
desc, err := s.CF.DescribeChangeSetWithContext(ctx, &cloudformation.DescribeChangeSetInput{
@ -275,10 +363,10 @@ func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte)
StackName: aws.String(name),
})
if aws.StringValue(desc.Status) == "FAILED" {
return *changeset.Id, fmt.Errorf(aws.StringValue(desc.StatusReason))
return changeset, fmt.Errorf(aws.StringValue(desc.StatusReason))
}
return *changeset.Id, err
return changeset, err
}
func (s sdk) UpdateStack(ctx context.Context, changeset string) error {

4
ecs/up.go
View File

@ -44,7 +44,7 @@ func (b *ecsAPIService) Up(ctx context.Context, project *types.Project, detach b
operation := stackCreate
if update {
operation = stackUpdate
changeset, err := b.aws.CreateChangeSet(ctx, project.Name, template)
changeset, err := b.aws.CreateChangeSet(ctx, project.Name, b.Region, template)
if err != nil {
return err
}
@ -53,7 +53,7 @@ func (b *ecsAPIService) Up(ctx context.Context, project *types.Project, detach b
return err
}
} else {
err = b.aws.CreateStack(ctx, project.Name, template)
err = b.aws.CreateStack(ctx, project.Name, b.Region, template)
if err != nil {
return err
}

1
go.mod
View File

@ -40,6 +40,7 @@ require (
github.com/google/uuid v1.1.2
github.com/gorilla/mux v1.7.4 // indirect
github.com/hashicorp/go-multierror v1.1.0
github.com/hashicorp/go-uuid v1.0.1
github.com/iancoleman/strcase v0.1.2
github.com/joho/godotenv v1.3.0
github.com/labstack/echo v3.3.10+incompatible

1
go.sum
View File

@ -298,6 +298,7 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=