Merge pull request #783 from docker/efs_opts

Let user pass EFS create option by driver_opts

ecs/awsResources.go

@@ -19,15 +19,17 @@ package ecs
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/docker/compose-cli/api/compose"
-
 	"github.com/docker/compose-cli/errdefs"
 
 	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/awslabs/goformation/v4/cloudformation"
 	"github.com/awslabs/goformation/v4/cloudformation/ec2"
 	"github.com/awslabs/goformation/v4/cloudformation/ecs"
+	"github.com/awslabs/goformation/v4/cloudformation/efs"
 	"github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2"
 	"github.com/compose-spec/compose-go/types"
 	"github.com/pkg/errors"
@@ -199,24 +201,23 @@ func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types
 		if err != nil {
 			return nil, err
 		}
-		if id == "" {
-			tags["Name"] = fmt.Sprintf("%s_%s", project.Name, vol.Name)
-			logrus.Debug("no EFS filesystem found, create a fresh new one")
-			id, err = b.aws.CreateFileSystem(ctx, tags)
-			if err != nil {
-				return nil, err
-			}
+		if id != "" {
+			filesystems[name] = id
 		}
-		filesystems[name] = id
 	}
 	return filesystems, nil
 }
 
 // ensureResources create required resources in template if not yet defined
-func (b *ecsAPIService) ensureResources(resources *awsResources, project *types.Project, template *cloudformation.Template) {
+func (b *ecsAPIService) ensureResources(resources *awsResources, project *types.Project, template *cloudformation.Template) error {
 	b.ensureCluster(resources, project, template)
 	b.ensureNetworks(resources, project, template)
+	err := b.ensureVolumes(resources, project, template)
+	if err != nil {
+		return err
+	}
 	b.ensureLoadBalancer(resources, project, template)
+	return nil
 }
 
 func (b *ecsAPIService) ensureCluster(r *awsResources, project *types.Project, template *cloudformation.Template) {
@@ -257,6 +258,70 @@ func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project,
 	}
 }
 
+func (b *ecsAPIService) ensureVolumes(r *awsResources, project *types.Project, template *cloudformation.Template) error {
+	for name, volume := range project.Volumes {
+		if _, ok := r.filesystems[name]; ok {
+			continue
+		}
+
+		var backupPolicy *efs.FileSystem_BackupPolicy
+		if backup, ok := volume.DriverOpts["backup_policy"]; ok {
+			backupPolicy = &efs.FileSystem_BackupPolicy{
+				Status: backup,
+			}
+		}
+
+		var lifecyclePolicies []efs.FileSystem_LifecyclePolicy
+		if policy, ok := volume.DriverOpts["lifecycle_policy"]; ok {
+			lifecyclePolicies = append(lifecyclePolicies, efs.FileSystem_LifecyclePolicy{
+				TransitionToIA: strings.TrimSpace(policy),
+			})
+		}
+
+		var provisionedThroughputInMibps float64
+		if t, ok := volume.DriverOpts["provisioned_throughput"]; ok {
+			v, err := strconv.ParseFloat(t, 64)
+			if err != nil {
+				return err
+			}
+			provisionedThroughputInMibps = v
+		}
+
+		var performanceMode = volume.DriverOpts["performance_mode"]
+		var throughputMode = volume.DriverOpts["throughput_mode"]
+		var kmsKeyID = volume.DriverOpts["kms_key_id"]
+
+		n := volumeResourceName(name)
+		template.Resources[n] = &efs.FileSystem{
+			BackupPolicy:     backupPolicy,
+			Encrypted:        true,
+			FileSystemPolicy: nil,
+			FileSystemTags: []efs.FileSystem_ElasticFileSystemTag{
+				{
+					Key:   compose.ProjectTag,
+					Value: project.Name,
+				},
+				{
+					Key:   compose.VolumeTag,
+					Value: name,
+				},
+				{
+					Key:   "Name",
+					Value: fmt.Sprintf("%s_%s", project.Name, name),
+				},
+			},
+			KmsKeyId:                        kmsKeyID,
+			LifecyclePolicies:               lifecyclePolicies,
+			PerformanceMode:                 performanceMode,
+			ProvisionedThroughputInMibps:    provisionedThroughputInMibps,
+			ThroughputMode:                  throughputMode,
+			AWSCloudFormationDeletionPolicy: "Retain",
+		}
+		r.filesystems[name] = cloudformation.Ref(n)
+	}
+	return nil
+}
+
 func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Project, template *cloudformation.Template) {
 	if r.loadBalancer != "" {
 		return

ecs/cloudformation.go

@@ -58,7 +58,10 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c
 	}
 
 	template := cloudformation.NewTemplate()
-	b.ensureResources(&resources, project, template)
+	err = b.ensureResources(&resources, project, template)
+	if err != nil {
+		return nil, err
+	}
 
 	for name, secret := range project.Secrets {
 		err := b.createSecret(project, name, secret, template)
@@ -441,6 +444,10 @@ func serviceResourceName(service string) string {
 	return fmt.Sprintf("%sService", normalizeResourceName(service))
 }
 
+func volumeResourceName(service string) string {
+	return fmt.Sprintf("%sFilesystem", normalizeResourceName(service))
+}
+
 func normalizeResourceName(s string) string {
 	return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, ""))
 }

ecs/cloudformation_test.go

@@ -357,20 +357,8 @@ networks:
 	assert.Check(t, s.NetworkConfiguration.AwsvpcConfiguration.SecurityGroups[0] == "sg-123abc") //nolint:staticcheck
 }
 
-func testVolume(t *testing.T, yaml string, fn ...func(m *MockAPIMockRecorder)) {
-	template := convertYaml(t, yaml, fn...)
-
-	s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
-	assert.Check(t, s != nil)
-	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
-
-	s = template.Resources["DbdataNFSMountTargetOnSubnet2"].(*efs.MountTarget)
-	assert.Check(t, s != nil)
-	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
-}
-
 func TestUseExternalVolume(t *testing.T) {
-	testVolume(t, `
+	template := convertYaml(t, `
 services:
   test:
     image: nginx
@@ -381,30 +369,50 @@ volumes:
 `, useDefaultVPC, func(m *MockAPIMockRecorder) {
 		m.FileSystemExists(gomock.Any(), "fs-123abc").Return(true, nil)
 	})
+	s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
+	assert.Check(t, s != nil)
+	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
+
+	s = template.Resources["DbdataNFSMountTargetOnSubnet2"].(*efs.MountTarget)
+	assert.Check(t, s != nil)
+	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
 }
 
 func TestCreateVolume(t *testing.T) {
-	testVolume(t, `
+	template := convertYaml(t, `
 services:
   test:
     image: nginx
 volumes:
-  db-data: {}
+  db-data: 
+    driver_opts:
+        backup_policy: ENABLED
+        lifecycle_policy: AFTER_30_DAYS
+        performance_mode: maxIO
+        throughput_mode: provisioned
+        provisioned_throughput: 1024
 `, useDefaultVPC, func(m *MockAPIMockRecorder) {
 		m.FindFileSystem(gomock.Any(), map[string]string{
 			compose.ProjectTag: t.Name(),
 			compose.VolumeTag:  "db-data",
 		}).Return("", nil)
-		m.CreateFileSystem(gomock.Any(), map[string]string{
-			compose.ProjectTag: t.Name(),
-			compose.VolumeTag:  "db-data",
-			"Name":             fmt.Sprintf("%s_%s", t.Name(), "db-data"),
-		}).Return("fs-123abc", nil)
 	})
+	n := volumeResourceName("db-data")
+	f := template.Resources[n].(*efs.FileSystem)
+	assert.Check(t, f != nil)
+	assert.Equal(t, f.BackupPolicy.Status, "ENABLED")                       //nolint:staticcheck
+	assert.Equal(t, f.LifecyclePolicies[0].TransitionToIA, "AFTER_30_DAYS") //nolint:staticcheck
+	assert.Equal(t, f.PerformanceMode, "maxIO")                             //nolint:staticcheck
+	assert.Equal(t, f.ThroughputMode, "provisioned")                        //nolint:staticcheck
+	assert.Equal(t, f.ProvisionedThroughputInMibps, float64(1024))          //nolint:staticcheck
+
+	s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
+	assert.Check(t, s != nil)
+	assert.Equal(t, s.FileSystemId, cloudformation.Ref(n)) //nolint:staticcheck
 }
 
 func TestReusePreviousVolume(t *testing.T) {
-	testVolume(t, `
+	template := convertYaml(t, `
 services:
   test:
     image: nginx
@@ -416,6 +424,9 @@ volumes:
 			compose.VolumeTag:  "db-data",
 		}).Return("fs-123abc", nil)
 	})
+	s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
+	assert.Check(t, s != nil)
+	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
 }
 
 func TestServiceMapping(t *testing.T) {

ecs/compatibility.go

@@ -98,6 +98,7 @@ var compatibleComposeAttributes = []string{
 	"volumes",
 	"volumes.external",
 	"volumes.name",
+	"volumes.driver_opts",
 	"networks.external",
 	"networks.name",
 }