tyler.durden/compose
1
0
Fork 0
mirror of https://github.com/docker/compose.git synced 2025-07-21 04:34:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #869 from docker/s3

Browse Source 
publish Cloudformation template on s3
This commit is contained in:
Nicolas De loof 2020-11-10 15:45:31 +01:00 committed by GitHub
commit 16c3223623
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ecs/aws.go
View File

@ -42,8 +42,8 @@ type API interface {
GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error)
GetRoleArn(ctx context.Context, name string) (string, error)
StackExists(ctx context.Context, name string) (bool, error)
CreateStack(ctx context.Context, name string, template []byte) error
CreateChangeSet(ctx context.Context, name string, template []byte) (string, error)
CreateStack(ctx context.Context, name string, region string, template []byte) error
CreateChangeSet(ctx context.Context, name string, region string, template []byte) (string, error)
UpdateStack(ctx context.Context, changeset string) error
WaitStackComplete(ctx context.Context, name string, operation int) error
GetStackID(ctx context.Context, name string) (string, error)

16
ecs/aws_mock.go
View File

@ -66,18 +66,18 @@ func (mr *MockAPIMockRecorder) CheckVPC(arg0, arg1 interface{}) *gomock.Call {
}
// CreateChangeSet mocks base method
func (m *MockAPI) CreateChangeSet(arg0 context.Context, arg1 string, arg2 []byte) (string, error) {
func (m *MockAPI) CreateChangeSet(arg0 context.Context, arg1, arg2 string, arg3 []byte) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CreateChangeSet", arg0, arg1, arg2)
ret := m.ctrl.Call(m, "CreateChangeSet", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// CreateChangeSet indicates an expected call of CreateChangeSet
func (mr *MockAPIMockRecorder) CreateChangeSet(arg0, arg1, arg2 interface{}) *gomock.Call {
func (mr *MockAPIMockRecorder) CreateChangeSet(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateChangeSet", reflect.TypeOf((*MockAPI)(nil).CreateChangeSet), arg0, arg1, arg2)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateChangeSet", reflect.TypeOf((*MockAPI)(nil).CreateChangeSet), arg0, arg1, arg2, arg3)
}
// CreateCluster mocks base method
@ -126,17 +126,17 @@ func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 interface{}) *gomock.Call
}
// CreateStack mocks base method
func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 []byte) error {
func (m *MockAPI) CreateStack(arg0 context.Context, arg1, arg2 string, arg3 []byte) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2)
ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(error)
return ret0
}
// CreateStack indicates an expected call of CreateStack
func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2 interface{}) *gomock.Call {
func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2, arg3)
}
// DeleteAutoscalingGroup mocks base method

112
ecs/sdk.go
View File

@ -17,6 +17,7 @@
package ecs
import (
"bytes"
"context"
"encoding/json"
"fmt"
@ -30,6 +31,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
@ -48,11 +50,15 @@ import (
"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/iam/iamiface"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface"
"github.com/aws/aws-sdk-go/service/ssm"
"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-uuid"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -68,6 +74,8 @@ type sdk struct {
SM secretsmanageriface.SecretsManagerAPI
SSM ssmiface.SSMAPI
AG autoscalingiface.AutoScalingAPI
S3 s3iface.S3API
uploader *s3manager.Uploader
}
// sdk implement API
@ -88,6 +96,8 @@ func newSDK(sess *session.Session) sdk {
SM: secretsmanager.New(sess),
SSM: ssm.New(sess),
AG: autoscaling.New(sess),
S3: s3.New(sess),
uploader: s3manager.NewUploader(sess),
}
}
@ -187,11 +197,9 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error
return nil, err
}
for _, subnet := range subnets.Subnets {
id := aws.StringValue(subnet.SubnetId)
logrus.Debugf("Found SubNet %s", id)
ids = append(ids, existingAWSResource{
arn: aws.StringValue(subnet.SubnetArn),
id: id,
id: aws.StringValue(subnet.SubnetId),
})
}
@ -226,13 +234,80 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) {
return len(stacks.Stacks) > 0, nil
}
func (s sdk) CreateStack(ctx context.Context, name string, template []byte) error {
type uploadedTemplateFunc func(body *string, url *string) (string, error)
const cloudformationBytesLimit = 51200
func (s sdk) withTemplate(ctx context.Context, name string, template []byte, region string, fn uploadedTemplateFunc) (string, error) {
if len(template) < cloudformationBytesLimit {
return fn(aws.String(string(template)), nil)
}
logrus.Debug("Create s3 bucket to store cloudformation template")
var configuration *s3.CreateBucketConfiguration
if region != "us-east-1" {
configuration = &s3.CreateBucketConfiguration{
LocationConstraint: aws.String(region),
}
}
// CloudFormation will only allow URL from a same-region bucket
// to avoid conflicts we suffix bucket name by region, so we can create comparable buckets in other regions.
bucket := "com.docker.compose." + region
_, err := s.S3.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
CreateBucketConfiguration: configuration,
})
if err != nil {
ae, ok := err.(awserr.Error)
if !ok {
return "", err
}
if ae.Code() != s3.ErrCodeBucketAlreadyOwnedByYou {
return "", err
}
}
key, err := uuid.GenerateUUID()
if err != nil {
return "", err
}
upload, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Key: aws.String(key),
Body: bytes.NewReader(template),
Bucket: aws.String(bucket),
ContentType: aws.String("application/json"),
Tagging: aws.String(name),
})
if err != nil {
return "", err
}
defer s.S3.DeleteObjects(&s3.DeleteObjectsInput{ //nolint: errcheck
Bucket: aws.String(bucket),
Delete: &s3.Delete{
Objects: []*s3.ObjectIdentifier{
{
Key: aws.String(key),
VersionId: upload.VersionID,
},
},
},
})
return fn(nil, aws.String(upload.Location))
}
func (s sdk) CreateStack(ctx context.Context, name string, region string, template []byte) error {
logrus.Debug("Create CloudFormation stack")
_, err := s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
stackID, err := s.withTemplate(ctx, name, template, region, func(body *string, url *string) (string, error) {
stack, err := s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
OnFailure: aws.String("DELETE"),
StackName: aws.String(name),
TemplateBody: aws.String(string(template)),
TemplateBody: body,
TemplateURL: url,
TimeoutInMinutes: nil,
Capabilities: []*string{
aws.String(cloudformation.CapabilityCapabilityIam),
@ -244,18 +319,26 @@ func (s sdk) CreateStack(ctx context.Context, name string, template []byte) erro
},
},
})
if err != nil {
return "", err
}
return aws.StringValue(stack.StackId), nil
})
logrus.Debugf("Stack %s created", stackID)
return err
}
func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte) (string, error) {
func (s sdk) CreateChangeSet(ctx context.Context, name string, region string, template []byte) (string, error) {
logrus.Debug("Create CloudFormation Changeset")
update := fmt.Sprintf("Update%s", time.Now().Format("2006-01-02-15-04-05"))
changeset, err := s.withTemplate(ctx, name, template, region, func(body *string, url *string) (string, error) {
changeset, err := s.CF.CreateChangeSetWithContext(ctx, &cloudformation.CreateChangeSetInput{
ChangeSetName: aws.String(update),
ChangeSetType: aws.String(cloudformation.ChangeSetTypeUpdate),
StackName: aws.String(name),
TemplateBody: aws.String(string(template)),
TemplateBody: body,
TemplateURL: url,
Capabilities: []*string{
aws.String(cloudformation.CapabilityCapabilityIam),
},
@ -263,11 +346,16 @@ func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte)
if err != nil {
return "", err
}
return aws.StringValue(changeset.Id), err
})
if err != nil {
return "", err
}
// we have to WaitUntilChangeSetCreateComplete even this in fail with error `ResourceNotReady`
// so that we can invoke DescribeChangeSet to check status, and then we can know about the actual creation failure cause.
s.CF.WaitUntilChangeSetCreateCompleteWithContext(ctx, &cloudformation.DescribeChangeSetInput{ // nolint:errcheck
ChangeSetName: changeset.Id,
ChangeSetName: aws.String(changeset),
})
desc, err := s.CF.DescribeChangeSetWithContext(ctx, &cloudformation.DescribeChangeSetInput{
@ -275,10 +363,10 @@ func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte)
StackName: aws.String(name),
})
if aws.StringValue(desc.Status) == "FAILED" {
return *changeset.Id, fmt.Errorf(aws.StringValue(desc.StatusReason))
return changeset, fmt.Errorf(aws.StringValue(desc.StatusReason))
}
return *changeset.Id, err
return changeset, err
}
func (s sdk) UpdateStack(ctx context.Context, changeset string) error {

4
ecs/up.go
View File

@ -44,7 +44,7 @@ func (b *ecsAPIService) Up(ctx context.Context, project *types.Project, detach b
operation := stackCreate
if update {
operation = stackUpdate
changeset, err := b.aws.CreateChangeSet(ctx, project.Name, template)
changeset, err := b.aws.CreateChangeSet(ctx, project.Name, b.Region, template)
if err != nil {
return err
}
@ -53,7 +53,7 @@ func (b *ecsAPIService) Up(ctx context.Context, project *types.Project, detach b
return err
}
} else {
err = b.aws.CreateStack(ctx, project.Name, template)
err = b.aws.CreateStack(ctx, project.Name, b.Region, template)
if err != nil {
return err
}

1
go.mod
View File

@ -40,6 +40,7 @@ require (
github.com/google/uuid v1.1.2
github.com/gorilla/mux v1.7.4 // indirect
github.com/hashicorp/go-multierror v1.1.0
github.com/hashicorp/go-uuid v1.0.1
github.com/iancoleman/strcase v0.1.2
github.com/joho/godotenv v1.3.0
github.com/labstack/echo v3.3.10+incompatible

1
go.sum
View File

@ -298,6 +298,7 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=