From 8379bf467ea298feefa04db372333c3085a2a183 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 14 Oct 2020 11:04:50 +0200 Subject: [PATCH] Let user pass EFS create option by driver_opts fixes #781 Signed-off-by: Nicolas De Loof --- ecs/awsResources.go | 85 +++++++++++++++++++++++++++++++++----- ecs/cloudformation.go | 9 +++- ecs/cloudformation_test.go | 53 ++++++++++++++---------- ecs/compatibility.go | 1 + 4 files changed, 116 insertions(+), 32 deletions(-) diff --git a/ecs/awsResources.go b/ecs/awsResources.go index af5a55374..566754eb6 100644 --- a/ecs/awsResources.go +++ b/ecs/awsResources.go @@ -19,15 +19,17 @@ package ecs import ( "context" "fmt" + "strconv" + "strings" "github.com/docker/compose-cli/api/compose" - "github.com/docker/compose-cli/errdefs" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" + "github.com/awslabs/goformation/v4/cloudformation/efs" "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/compose-spec/compose-go/types" "github.com/pkg/errors" @@ -199,24 +201,23 @@ func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types if err != nil { return nil, err } - if id == "" { - tags["Name"] = fmt.Sprintf("%s_%s", project.Name, vol.Name) - logrus.Debug("no EFS filesystem found, create a fresh new one") - id, err = b.aws.CreateFileSystem(ctx, tags) - if err != nil { - return nil, err - } + if id != "" { + filesystems[name] = id } - filesystems[name] = id } return filesystems, nil } // ensureResources create required resources in template if not yet defined -func (b *ecsAPIService) ensureResources(resources *awsResources, project *types.Project, template *cloudformation.Template) { +func (b *ecsAPIService) ensureResources(resources *awsResources, project *types.Project, template *cloudformation.Template) error { b.ensureCluster(resources, project, template) b.ensureNetworks(resources, project, template) + err := b.ensureVolumes(resources, project, template) + if err != nil { + return err + } b.ensureLoadBalancer(resources, project, template) + return nil } func (b *ecsAPIService) ensureCluster(r *awsResources, project *types.Project, template *cloudformation.Template) { @@ -257,6 +258,70 @@ func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project, } } +func (b *ecsAPIService) ensureVolumes(r *awsResources, project *types.Project, template *cloudformation.Template) error { + for name, volume := range project.Volumes { + if _, ok := r.filesystems[name]; ok { + continue + } + + var backupPolicy *efs.FileSystem_BackupPolicy + if backup, ok := volume.DriverOpts["backup_policy"]; ok { + backupPolicy = &efs.FileSystem_BackupPolicy{ + Status: backup, + } + } + + var lifecyclePolicies []efs.FileSystem_LifecyclePolicy + if policy, ok := volume.DriverOpts["lifecycle_policy"]; ok { + lifecyclePolicies = append(lifecyclePolicies, efs.FileSystem_LifecyclePolicy{ + TransitionToIA: strings.TrimSpace(policy), + }) + } + + var provisionedThroughputInMibps float64 + if t, ok := volume.DriverOpts["provisioned_throughput"]; ok { + v, err := strconv.ParseFloat(t, 64) + if err != nil { + return err + } + provisionedThroughputInMibps = v + } + + var performanceMode = volume.DriverOpts["performance_mode"] + var throughputMode = volume.DriverOpts["throughput_mode"] + var kmsKeyID = volume.DriverOpts["kms_key_id"] + + n := volumeResourceName(name) + template.Resources[n] = &efs.FileSystem{ + BackupPolicy: backupPolicy, + Encrypted: true, + FileSystemPolicy: nil, + FileSystemTags: []efs.FileSystem_ElasticFileSystemTag{ + { + Key: compose.ProjectTag, + Value: project.Name, + }, + { + Key: compose.VolumeTag, + Value: name, + }, + { + Key: "Name", + Value: fmt.Sprintf("%s_%s", project.Name, name), + }, + }, + KmsKeyId: kmsKeyID, + LifecyclePolicies: lifecyclePolicies, + PerformanceMode: performanceMode, + ProvisionedThroughputInMibps: provisionedThroughputInMibps, + ThroughputMode: throughputMode, + AWSCloudFormationDeletionPolicy: "Retain", + } + r.filesystems[name] = cloudformation.Ref(n) + } + return nil +} + func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Project, template *cloudformation.Template) { if r.loadBalancer != "" { return diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go index ec518ded4..5c116fb41 100644 --- a/ecs/cloudformation.go +++ b/ecs/cloudformation.go @@ -58,7 +58,10 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c } template := cloudformation.NewTemplate() - b.ensureResources(&resources, project, template) + err = b.ensureResources(&resources, project, template) + if err != nil { + return nil, err + } for name, secret := range project.Secrets { err := b.createSecret(project, name, secret, template) @@ -441,6 +444,10 @@ func serviceResourceName(service string) string { return fmt.Sprintf("%sService", normalizeResourceName(service)) } +func volumeResourceName(service string) string { + return fmt.Sprintf("%sFilesystem", normalizeResourceName(service)) +} + func normalizeResourceName(s string) string { return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) } diff --git a/ecs/cloudformation_test.go b/ecs/cloudformation_test.go index 23a089a80..06176822c 100644 --- a/ecs/cloudformation_test.go +++ b/ecs/cloudformation_test.go @@ -357,20 +357,8 @@ networks: assert.Check(t, s.NetworkConfiguration.AwsvpcConfiguration.SecurityGroups[0] == "sg-123abc") //nolint:staticcheck } -func testVolume(t *testing.T, yaml string, fn ...func(m *MockAPIMockRecorder)) { - template := convertYaml(t, yaml, fn...) - - s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget) - assert.Check(t, s != nil) - assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck - - s = template.Resources["DbdataNFSMountTargetOnSubnet2"].(*efs.MountTarget) - assert.Check(t, s != nil) - assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck -} - func TestUseExternalVolume(t *testing.T) { - testVolume(t, ` + template := convertYaml(t, ` services: test: image: nginx @@ -381,30 +369,50 @@ volumes: `, useDefaultVPC, func(m *MockAPIMockRecorder) { m.FileSystemExists(gomock.Any(), "fs-123abc").Return(true, nil) }) + s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget) + assert.Check(t, s != nil) + assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck + + s = template.Resources["DbdataNFSMountTargetOnSubnet2"].(*efs.MountTarget) + assert.Check(t, s != nil) + assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck } func TestCreateVolume(t *testing.T) { - testVolume(t, ` + template := convertYaml(t, ` services: test: image: nginx volumes: - db-data: {} + db-data: + driver_opts: + backup_policy: ENABLED + lifecycle_policy: AFTER_30_DAYS + performance_mode: maxIO + throughput_mode: provisioned + provisioned_throughput: 1024 `, useDefaultVPC, func(m *MockAPIMockRecorder) { m.FindFileSystem(gomock.Any(), map[string]string{ compose.ProjectTag: t.Name(), compose.VolumeTag: "db-data", }).Return("", nil) - m.CreateFileSystem(gomock.Any(), map[string]string{ - compose.ProjectTag: t.Name(), - compose.VolumeTag: "db-data", - "Name": fmt.Sprintf("%s_%s", t.Name(), "db-data"), - }).Return("fs-123abc", nil) }) + n := volumeResourceName("db-data") + f := template.Resources[n].(*efs.FileSystem) + assert.Check(t, f != nil) + assert.Equal(t, f.BackupPolicy.Status, "ENABLED") //nolint:staticcheck + assert.Equal(t, f.LifecyclePolicies[0].TransitionToIA, "AFTER_30_DAYS") //nolint:staticcheck + assert.Equal(t, f.PerformanceMode, "maxIO") //nolint:staticcheck + assert.Equal(t, f.ThroughputMode, "provisioned") //nolint:staticcheck + assert.Equal(t, f.ProvisionedThroughputInMibps, float64(1024)) //nolint:staticcheck + + s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget) + assert.Check(t, s != nil) + assert.Equal(t, s.FileSystemId, cloudformation.Ref(n)) //nolint:staticcheck } func TestReusePreviousVolume(t *testing.T) { - testVolume(t, ` + template := convertYaml(t, ` services: test: image: nginx @@ -416,6 +424,9 @@ volumes: compose.VolumeTag: "db-data", }).Return("fs-123abc", nil) }) + s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget) + assert.Check(t, s != nil) + assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck } func TestServiceMapping(t *testing.T) { diff --git a/ecs/compatibility.go b/ecs/compatibility.go index 339ff0e42..4c42fc434 100644 --- a/ecs/compatibility.go +++ b/ecs/compatibility.go @@ -98,6 +98,7 @@ var compatibleComposeAttributes = []string{ "volumes", "volumes.external", "volumes.name", + "volumes.driver_opts", "networks.external", "networks.name", }