/* Copyright 2020 Docker Compose CLI authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package ecs import ( "context" "fmt" "io/ioutil" "regexp" "strings" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/awslabs/goformation/v4/cloudformation/logs" "github.com/awslabs/goformation/v4/cloudformation/secretsmanager" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/compose-spec/compose-go/compatibility" "github.com/compose-spec/compose-go/errdefs" "github.com/compose-spec/compose-go/types" "github.com/sirupsen/logrus" ) const ( parameterClusterName = "ParameterClusterName" parameterVPCId = "ParameterVPCId" parameterSubnet1Id = "ParameterSubnet1Id" parameterSubnet2Id = "ParameterSubnet2Id" parameterLoadBalancerARN = "ParameterLoadBalancerARN" ) func (b *ecsAPIService) Convert(ctx context.Context, project *types.Project) ([]byte, error) { template, networks, err := b.convert(project) if err != nil { return nil, err } // Create a NFS inbound rule on each mount target for volumes // as "source security group" use an arbitrary network attached to service(s) who mounts target volume for n, vol := range project.Volumes { err := b.SDK.WithVolumeSecurityGroups(ctx, vol.Name, func(securityGroups []string) error { target := securityGroups[0] for _, s := range project.Services { for _, v := range s.Volumes { if v.Source != n { continue } var source string for net := range s.Networks { network := project.Networks[net] if ext, ok := network.Extensions[extensionSecurityGroup]; ok { source = ext.(string) } else { source = networkResourceName(project, net) } break } name := fmt.Sprintf("%sNFSMount%s", s.Name, n) template.Resources[name] = &ec2.SecurityGroupIngress{ Description: fmt.Sprintf("Allow NFS mount for %s on %s", s.Name, n), GroupId: target, SourceSecurityGroupId: cloudformation.Ref(source), IpProtocol: "tcp", FromPort: 2049, ToPort: 2049, } service := template.Resources[serviceResourceName(s.Name)].(*ecs.Service) service.AWSCloudFormationDependsOn = append(service.AWSCloudFormationDependsOn, name) } } return nil }) if err != nil { return nil, err } } err = b.createCapacityProvider(ctx, project, networks, template) if err != nil { return nil, err } return marshall(template) } // Convert a compose project into a CloudFormation template func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Template, map[string]string, error) { //nolint:gocyclo var checker compatibility.Checker = &fargateCompatibilityChecker{ compatibility.AllowList{ Supported: compatibleComposeAttributes, }, } compatibility.Check(project, checker) for _, err := range checker.Errors() { if errdefs.IsIncompatibleError(err) { logrus.Error(err.Error()) } else { logrus.Warn(err.Error()) } } if !compatibility.IsCompatible(checker) { return nil, nil, fmt.Errorf("compose file is incompatible with Amazon ECS") } template := cloudformation.NewTemplate() template.Description = "CloudFormation template created by Docker for deploying applications on Amazon ECS" template.Parameters[parameterClusterName] = cloudformation.Parameter{ Type: "String", Description: "Name of the ECS cluster to deploy to (optional)", } template.Parameters[parameterVPCId] = cloudformation.Parameter{ Type: "AWS::EC2::VPC::Id", Description: "ID of the VPC", } /* FIXME can't set subnets: Ref("SubnetIds") see https://github.com/awslabs/goformation/issues/282 template.Parameters["SubnetIds"] = cloudformation.Parameter{ Type: "List", Description: "The list of SubnetIds, for at least two Availability Zones in the region in your VPC", } */ template.Parameters[parameterSubnet1Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", Description: "SubnetId, for Availability Zone 1 in the region in your VPC", } template.Parameters[parameterSubnet2Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", Description: "SubnetId, for Availability Zone 2 in the region in your VPC", } template.Parameters[parameterLoadBalancerARN] = cloudformation.Parameter{ Type: "String", Description: "Name of the LoadBalancer to connect to (optional)", } template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(parameterClusterName)) cluster := createCluster(project, template) networks := map[string]string{} for _, net := range project.Networks { networks[net.Name] = convertNetwork(project, net, cloudformation.Ref(parameterVPCId), template) } for i, s := range project.Secrets { if s.External.External { continue } secret, err := ioutil.ReadFile(s.File) if err != nil { return nil, nil, err } name := fmt.Sprintf("%sSecret", normalizeResourceName(s.Name)) template.Resources[name] = &secretsmanager.Secret{ Description: "", SecretString: string(secret), Tags: projectTags(project), } s.Name = cloudformation.Ref(name) project.Secrets[i] = s } createLogGroup(project, template) // Private DNS namespace will allow DNS name for the services to be ..local createCloudMap(project, template) loadBalancerARN := createLoadBalancer(project, template) for _, service := range project.Services { definition, err := convert(project, service) if err != nil { return nil, nil, err } taskExecutionRole := createTaskExecutionRole(service, definition, template) definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) taskRole := createTaskRole(service, template) if taskRole != "" { definition.TaskRoleArn = cloudformation.Ref(taskRole) } taskDefinition := fmt.Sprintf("%sTaskDefinition", normalizeResourceName(service.Name)) template.Resources[taskDefinition] = definition var healthCheck *cloudmap.Service_HealthCheckConfig serviceRegistry := createServiceRegistry(service, template, healthCheck) serviceSecurityGroups := []string{} for net := range service.Networks { serviceSecurityGroups = append(serviceSecurityGroups, networks[net]) } dependsOn := []string{} serviceLB := []ecs.Service_LoadBalancer{} if len(service.Ports) > 0 { for _, port := range service.Ports { protocol := strings.ToUpper(port.Protocol) if getLoadBalancerType(project) == elbv2.LoadBalancerTypeEnumApplication { // we don't set Https as a certificate must be specified for HTTPS listeners protocol = elbv2.ProtocolEnumHttp } if loadBalancerARN != "" { targetGroupName := createTargetGroup(project, service, port, template, protocol) listenerName := createListener(service, port, template, targetGroupName, loadBalancerARN, protocol) dependsOn = append(dependsOn, listenerName) serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, ContainerPort: int(port.Target), TargetGroupArn: cloudformation.Ref(targetGroupName), }) } } } desiredCount := 1 if service.Deploy != nil && service.Deploy.Replicas != nil { desiredCount = int(*service.Deploy.Replicas) } for dependency := range service.DependsOn { dependsOn = append(dependsOn, serviceResourceName(dependency)) } minPercent, maxPercent, err := computeRollingUpdateLimits(service) if err != nil { return nil, nil, err } assignPublicIP := ecsapi.AssignPublicIpEnabled launchType := ecsapi.LaunchTypeFargate platformVersion := "1.4.0" // LATEST which is set to 1.3.0 (?) which doesn’t allow efs volumes. if requireEC2(service) { assignPublicIP = ecsapi.AssignPublicIpDisabled launchType = ecsapi.LaunchTypeEc2 platformVersion = "" // The platform version must be null when specifying an EC2 launch type } template.Resources[serviceResourceName(service.Name)] = &ecs.Service{ AWSCloudFormationDependsOn: dependsOn, Cluster: cluster, DesiredCount: desiredCount, DeploymentController: &ecs.Service_DeploymentController{ Type: ecsapi.DeploymentControllerTypeEcs, }, DeploymentConfiguration: &ecs.Service_DeploymentConfiguration{ MaximumPercent: maxPercent, MinimumHealthyPercent: minPercent, }, LaunchType: launchType, // TODO we miss support for https://github.com/aws/containers-roadmap/issues/631 to select a capacity provider LoadBalancers: serviceLB, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: assignPublicIP, SecurityGroups: serviceSecurityGroups, Subnets: []string{ cloudformation.Ref(parameterSubnet1Id), cloudformation.Ref(parameterSubnet2Id), }, }, }, PlatformVersion: platformVersion, PropagateTags: ecsapi.PropagateTagsService, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, Tags: serviceTags(project, service), TaskDefinition: cloudformation.Ref(normalizeResourceName(taskDefinition)), } } return template, networks, nil } func createLogGroup(project *types.Project, template *cloudformation.Template) { retention := 0 if v, ok := project.Extensions[extensionRetention]; ok { retention = v.(int) } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) template.Resources["LogGroup"] = &logs.LogGroup{ LogGroupName: logGroup, RetentionInDays: retention, } } func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { maxPercent := 200 minPercent := 100 if service.Deploy == nil || service.Deploy.UpdateConfig == nil { return minPercent, maxPercent, nil } updateConfig := service.Deploy.UpdateConfig min, okMin := updateConfig.Extensions[extensionMinPercent] if okMin { minPercent = min.(int) } max, okMax := updateConfig.Extensions[extensionMaxPercent] if okMax { maxPercent = max.(int) } if okMin && okMax { return minPercent, maxPercent, nil } if updateConfig.Parallelism != nil { parallelism := int(*updateConfig.Parallelism) if service.Deploy.Replicas == nil { return minPercent, maxPercent, fmt.Errorf("rolling update configuration require deploy.replicas to be set") } replicas := int(*service.Deploy.Replicas) if replicas < parallelism { return minPercent, maxPercent, fmt.Errorf("deploy.replicas (%d) must be greater than deploy.update_config.parallelism (%d)", replicas, parallelism) } if !okMin { minPercent = (replicas - parallelism) * 100 / replicas } if !okMax { maxPercent = (replicas + parallelism) * 100 / replicas } } return minPercent, maxPercent, nil } func getLoadBalancerType(project *types.Project) string { for _, service := range project.Services { for _, port := range service.Ports { protocol := port.Protocol v, ok := port.Extensions[extensionProtocol] if ok { protocol = v.(string) } if protocol == "http" || protocol == "https" { continue } if port.Published != 80 && port.Published != 443 { return elbv2.LoadBalancerTypeEnumNetwork } } } return elbv2.LoadBalancerTypeEnumApplication } func getLoadBalancerSecurityGroups(project *types.Project, template *cloudformation.Template) []string { securityGroups := []string{} for _, network := range project.Networks { if !network.Internal { net := convertNetwork(project, network, cloudformation.Ref(parameterVPCId), template) securityGroups = append(securityGroups, net) } } return uniqueStrings(securityGroups) } func createLoadBalancer(project *types.Project, template *cloudformation.Template) string { ports := 0 for _, service := range project.Services { ports += len(service.Ports) } if ports == 0 { // Project do not expose any port (batch jobs?) // So no need to create a PortPublisher return "" } // load balancer names are limited to 32 characters total loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) // Create PortPublisher if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(parameterLoadBalancerARN)) loadBalancerType := getLoadBalancerType(project) securityGroups := []string{} if loadBalancerType == elbv2.LoadBalancerTypeEnumApplication { securityGroups = getLoadBalancerSecurityGroups(project, template) } template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ Name: loadBalancerName, Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, SecurityGroups: securityGroups, Subnets: []string{ cloudformation.Ref(parameterSubnet1Id), cloudformation.Ref(parameterSubnet2Id), }, Tags: projectTags(project), Type: loadBalancerType, AWSCloudFormationCondition: "CreateLoadBalancer", } return cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(parameterLoadBalancerARN)) } func createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerARN string, protocol string) string { listenerName := fmt.Sprintf( "%s%s%dListener", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), port.Target, ) //add listener to dependsOn //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ DefaultActions: []elasticloadbalancingv2.Listener_Action{ { ForwardConfig: &elasticloadbalancingv2.Listener_ForwardConfig{ TargetGroups: []elasticloadbalancingv2.Listener_TargetGroupTuple{ { TargetGroupArn: cloudformation.Ref(targetGroupName), }, }, }, Type: elbv2.ActionTypeEnumForward, }, }, LoadBalancerArn: loadBalancerARN, Protocol: protocol, Port: int(port.Target), } return listenerName } func createTargetGroup(project *types.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { targetGroupName := fmt.Sprintf( "%s%s%dTargetGroup", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), port.Published, ) template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ Port: int(port.Target), Protocol: protocol, Tags: projectTags(project), VpcId: cloudformation.Ref(parameterVPCId), TargetType: elbv2.TargetTypeEnumIp, } return targetGroupName } func createServiceRegistry(service types.ServiceConfig, template *cloudformation.Template, healthCheck *cloudmap.Service_HealthCheckConfig) ecs.Service_ServiceRegistry { serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) serviceRegistry := ecs.Service_ServiceRegistry{ RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), } template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), HealthCheckConfig: healthCheck, HealthCheckCustomConfig: &cloudmap.Service_HealthCheckCustomConfig{ FailureThreshold: 1, }, Name: service.Name, NamespaceId: cloudformation.Ref("CloudMap"), DnsConfig: &cloudmap.Service_DnsConfig{ DnsRecords: []cloudmap.Service_DnsRecord{ { TTL: 60, Type: cloudmapapi.RecordTypeA, }, }, RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, }, } return serviceRegistry } func createTaskExecutionRole(service types.ServiceConfig, definition *ecs.TaskDefinition, template *cloudformation.Template) string { taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) policies := createPolicies(service, definition) template.Resources[taskExecutionRole] = &iam.Role{ AssumeRolePolicyDocument: ecsTaskAssumeRolePolicyDocument, Policies: policies, ManagedPolicyArns: []string{ ecsTaskExecutionPolicy, ecrReadOnlyPolicy, }, } return taskExecutionRole } func createTaskRole(service types.ServiceConfig, template *cloudformation.Template) string { taskRole := fmt.Sprintf("%sTaskRole", normalizeResourceName(service.Name)) rolePolicies := []iam.Role_Policy{} if roles, ok := service.Extensions[extensionRole]; ok { rolePolicies = append(rolePolicies, iam.Role_Policy{ PolicyDocument: roles, }) } managedPolicies := []string{} if v, ok := service.Extensions[extensionManagedPolicies]; ok { for _, s := range v.([]interface{}) { managedPolicies = append(managedPolicies, s.(string)) } } if len(rolePolicies) == 0 && len(managedPolicies) == 0 { return "" } template.Resources[taskRole] = &iam.Role{ AssumeRolePolicyDocument: ecsTaskAssumeRolePolicyDocument, Policies: rolePolicies, ManagedPolicyArns: managedPolicies, } return taskRole } func createCluster(project *types.Project, template *cloudformation.Template) string { template.Resources["Cluster"] = &ecs.Cluster{ ClusterName: project.Name, Tags: projectTags(project), AWSCloudFormationCondition: "CreateCluster", } cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(parameterClusterName)) return cluster } func createCloudMap(project *types.Project, template *cloudformation.Template) { template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), Vpc: cloudformation.Ref(parameterVPCId), } } func convertNetwork(project *types.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { if net.External.External { return net.Name } if sg, ok := net.Extensions[extensionSecurityGroup]; ok { logrus.Warn("to use an existing security-group, set `network.external` and `network.name` in your compose file") logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) return sg.(string) } var ingresses []ec2.SecurityGroup_Ingress if !net.Internal { for _, service := range project.Services { if _, ok := service.Networks[net.Name]; ok { for _, port := range service.Ports { protocol := strings.ToUpper(port.Protocol) if protocol == "" { protocol = "-1" } ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ CidrIp: "0.0.0.0/0", Description: fmt.Sprintf("%s:%d/%s", service.Name, port.Target, port.Protocol), FromPort: int(port.Target), IpProtocol: protocol, ToPort: int(port.Target), }) } } } } securityGroup := networkResourceName(project, net.Name) template.Resources[securityGroup] = &ec2.SecurityGroup{ GroupDescription: fmt.Sprintf("%s %s Security Group", project.Name, net.Name), GroupName: securityGroup, SecurityGroupIngress: ingresses, VpcId: vpc, Tags: networkTags(project, net), } ingress := securityGroup + "Ingress" template.Resources[ingress] = &ec2.SecurityGroupIngress{ Description: fmt.Sprintf("Allow communication within network %s", net.Name), IpProtocol: "-1", // all protocols GroupId: cloudformation.Ref(securityGroup), SourceSecurityGroupId: cloudformation.Ref(securityGroup), } return cloudformation.Ref(securityGroup) } func networkResourceName(project *types.Project, network string) string { return fmt.Sprintf("%s%sNetwork", normalizeResourceName(project.Name), normalizeResourceName(network)) } func serviceResourceName(service string) string { return fmt.Sprintf("%sService", normalizeResourceName(service)) } func normalizeResourceName(s string) string { return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) } func createPolicies(service types.ServiceConfig, taskDef *ecs.TaskDefinition) []iam.Role_Policy { arns := []string{} for _, container := range taskDef.ContainerDefinitions { if container.RepositoryCredentials != nil { arns = append(arns, container.RepositoryCredentials.CredentialsParameter) } if len(container.Secrets) > 0 { for _, s := range container.Secrets { arns = append(arns, s.ValueFrom) } } } if len(arns) > 0 { return []iam.Role_Policy{ { PolicyDocument: &PolicyDocument{ Statement: []PolicyStatement{ { Effect: "Allow", Action: []string{actionGetSecretValue, actionGetParameters, actionDecrypt}, Resource: arns, }, }, }, PolicyName: fmt.Sprintf("%sGrantAccessToSecrets", service.Name), }, } } return nil } func uniqueStrings(items []string) []string { keys := make(map[string]bool) unique := []string{} for _, item := range items { if _, val := keys[item]; !val { keys[item] = true unique = append(unique, item) } } return unique }