/* Copyright 2020 Docker Compose CLI authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package ecs import ( "context" "fmt" "io/ioutil" "regexp" "strings" "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/config" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/awslabs/goformation/v4/cloudformation/logs" "github.com/awslabs/goformation/v4/cloudformation/secretsmanager" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/cnabio/cnab-to-oci/remotes" "github.com/compose-spec/compose-go/types" cliconfig "github.com/docker/cli/cli/config" "github.com/docker/distribution/reference" "golang.org/x/sync/errgroup" "sigs.k8s.io/kustomize/kyaml/yaml" "sigs.k8s.io/kustomize/kyaml/yaml/merge2" ) func (b *ecsAPIService) Convert(ctx context.Context, project *types.Project, options compose.ConvertOptions) ([]byte, error) { err := b.resolveServiceImagesDigests(ctx, project) if err != nil { return nil, err } template, err := b.convert(ctx, project) if err != nil { return nil, err } bytes, err := marshall(template, options.Format) if err != nil { return nil, err } x, ok := project.Extensions[extensionCloudFormation] if !ok { return bytes, nil } if options.Format != "yaml" { return nil, fmt.Errorf("format %q with overlays is not supported", options.Format) } nodes, err := yaml.Parse(string(bytes)) if err != nil { return nil, err } bytes, err = yaml.Marshal(x) if err != nil { return nil, err } overlay, err := yaml.Parse(string(bytes)) if err != nil { return nil, err } nodes, err = merge2.Merge(overlay, nodes, yaml.MergeOptions{ ListIncreaseDirection: yaml.MergeOptionsListPrepend, }) if err != nil { return nil, err } s, err := nodes.String() if err != nil { return nil, err } bytes = []byte(s) return bytes, err } func (b *ecsAPIService) resolveServiceImagesDigests(ctx context.Context, project *types.Project) error { configFile, err := cliconfig.Load(config.Dir(ctx)) if err != nil { return err } resolver := remotes.CreateResolver(configFile) eg := errgroup.Group{} for i, s := range project.Services { idx := i service := s eg.Go(func() error { named, err := reference.ParseDockerRef(service.Image) if err != nil { return err } _, desc, err := resolver.Resolve(ctx, named.String()) if err != nil { return err } digested, err := reference.WithDigest(named, desc.Digest) if err != nil { return err } fmt.Printf("%s resolved to %s\n", service.Image, digested) service.Image = digested.String() project.Services[idx] = service return nil }) } return eg.Wait() } func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*cloudformation.Template, error) { err := b.checkCompatibility(project) if err != nil { return nil, err } template := cloudformation.NewTemplate() resources, err := b.parse(ctx, project, template) if err != nil { return nil, err } err = b.ensureResources(&resources, project, template) if err != nil { return nil, err } for name, secret := range project.Secrets { err := b.createSecret(project, name, secret, template) if err != nil { return nil, err } } b.createLogGroup(project, template) // Private DNS namespace will allow DNS name for the services to be ..local b.createCloudMap(project, template, resources.vpc) b.createNFSMountTarget(project, resources, template) b.createAccessPoints(project, resources, template) for _, service := range project.Services { err := b.createService(project, service, template, resources) if err != nil { return nil, err } err = b.createAutoscalingPolicy(project, resources, template, service) if err != nil { return nil, err } } err = b.createCapacityProvider(ctx, project, template, resources) if err != nil { return nil, err } return template, nil } func (b *ecsAPIService) createService(project *types.Project, service types.ServiceConfig, template *cloudformation.Template, resources awsResources) error { taskExecutionRole := b.createTaskExecutionRole(project, service, template) taskRole := b.createTaskRole(project, service, template, resources) definition, err := b.createTaskDefinition(project, service, resources) if err != nil { return err } definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) if taskRole != "" { definition.TaskRoleArn = cloudformation.Ref(taskRole) } taskDefinition := fmt.Sprintf("%sTaskDefinition", normalizeResourceName(service.Name)) template.Resources[taskDefinition] = definition var healthCheck *cloudmap.Service_HealthCheckConfig serviceRegistry := b.createServiceRegistry(service, template, healthCheck) var ( dependsOn []string serviceLB []ecs.Service_LoadBalancer ) for _, port := range service.Ports { for net := range service.Networks { b.createIngress(service, net, port, template, resources) } protocol := strings.ToUpper(port.Protocol) if resources.loadBalancerType == elbv2.LoadBalancerTypeEnumApplication { // we don't set Https as a certificate must be specified for HTTPS listeners protocol = elbv2.ProtocolEnumHttp } targetGroupName := b.createTargetGroup(project, service, port, template, protocol, resources.vpc) listenerName := b.createListener(service, port, template, targetGroupName, resources.loadBalancer, protocol) dependsOn = append(dependsOn, listenerName) serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, ContainerPort: int(port.Target), TargetGroupArn: cloudformation.Ref(targetGroupName), }) } desiredCount := 1 if service.Deploy != nil && service.Deploy.Replicas != nil { desiredCount = int(*service.Deploy.Replicas) } for dependency := range service.DependsOn { dependsOn = append(dependsOn, serviceResourceName(dependency)) } for _, s := range service.Volumes { dependsOn = append(dependsOn, b.mountTargets(s.Source, resources)...) } minPercent, maxPercent, err := computeRollingUpdateLimits(service) if err != nil { return err } assignPublicIP := ecsapi.AssignPublicIpEnabled launchType := ecsapi.LaunchTypeFargate platformVersion := "1.4.0" // LATEST which is set to 1.3.0 (?) which doesn’t allow efs volumes. if requireEC2(service) { assignPublicIP = ecsapi.AssignPublicIpDisabled launchType = ecsapi.LaunchTypeEc2 platformVersion = "" // The platform version must be null when specifying an EC2 launch type } template.Resources[serviceResourceName(service.Name)] = &ecs.Service{ AWSCloudFormationDependsOn: dependsOn, Cluster: resources.cluster.ARN(), DesiredCount: desiredCount, DeploymentController: &ecs.Service_DeploymentController{ Type: ecsapi.DeploymentControllerTypeEcs, }, DeploymentConfiguration: &ecs.Service_DeploymentConfiguration{ MaximumPercent: maxPercent, MinimumHealthyPercent: minPercent, }, LaunchType: launchType, // TODO we miss support for https://github.com/aws/containers-roadmap/issues/631 to select a capacity provider LoadBalancers: serviceLB, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: assignPublicIP, SecurityGroups: resources.serviceSecurityGroups(service), Subnets: resources.subnetsIDs(), }, }, PlatformVersion: platformVersion, PropagateTags: ecsapi.PropagateTagsService, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, Tags: serviceTags(project, service), TaskDefinition: cloudformation.Ref(normalizeResourceName(taskDefinition)), } return nil } const allProtocols = "-1" func (b *ecsAPIService) createIngress(service types.ServiceConfig, net string, port types.ServicePortConfig, template *cloudformation.Template, resources awsResources) { protocol := strings.ToUpper(port.Protocol) if protocol == "" { protocol = allProtocols } ingress := fmt.Sprintf("%s%dIngress", normalizeResourceName(net), port.Target) template.Resources[ingress] = &ec2.SecurityGroupIngress{ CidrIp: "0.0.0.0/0", Description: fmt.Sprintf("%s:%d/%s on %s network", service.Name, port.Target, port.Protocol, net), GroupId: resources.securityGroups[net], FromPort: int(port.Target), IpProtocol: protocol, ToPort: int(port.Target), } } func (b *ecsAPIService) createSecret(project *types.Project, name string, s types.SecretConfig, template *cloudformation.Template) error { if s.External.External { return nil } sensitiveData, err := ioutil.ReadFile(s.File) if err != nil { return err } resource := fmt.Sprintf("%sSecret", normalizeResourceName(s.Name)) template.Resources[resource] = &secretsmanager.Secret{ Description: fmt.Sprintf("Secret %s", s.Name), SecretString: string(sensitiveData), Tags: projectTags(project), } s.Name = cloudformation.Ref(resource) project.Secrets[name] = s return nil } func (b *ecsAPIService) createLogGroup(project *types.Project, template *cloudformation.Template) { retention := 0 if v, ok := project.Extensions[extensionRetention]; ok { retention = v.(int) } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) template.Resources["LogGroup"] = &logs.LogGroup{ LogGroupName: logGroup, RetentionInDays: retention, } } func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { maxPercent := 200 minPercent := 100 if service.Deploy == nil || service.Deploy.UpdateConfig == nil { return minPercent, maxPercent, nil } updateConfig := service.Deploy.UpdateConfig min, okMin := updateConfig.Extensions[extensionMinPercent] if okMin { minPercent = min.(int) } max, okMax := updateConfig.Extensions[extensionMaxPercent] if okMax { maxPercent = max.(int) } if okMin && okMax { return minPercent, maxPercent, nil } if updateConfig.Parallelism != nil { parallelism := int(*updateConfig.Parallelism) if service.Deploy.Replicas == nil { return minPercent, maxPercent, fmt.Errorf("rolling update configuration require deploy.replicas to be set") } replicas := int(*service.Deploy.Replicas) if replicas < parallelism { return minPercent, maxPercent, fmt.Errorf("deploy.replicas (%d) must be greater than deploy.update_config.parallelism (%d)", replicas, parallelism) } if !okMin { minPercent = (replicas - parallelism) * 100 / replicas } if !okMax { maxPercent = (replicas + parallelism) * 100 / replicas } } return minPercent, maxPercent, nil } func (b *ecsAPIService) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancer awsResource, protocol string) string { listenerName := fmt.Sprintf( "%s%s%dListener", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), port.Target, ) //add listener to dependsOn //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ DefaultActions: []elasticloadbalancingv2.Listener_Action{ { ForwardConfig: &elasticloadbalancingv2.Listener_ForwardConfig{ TargetGroups: []elasticloadbalancingv2.Listener_TargetGroupTuple{ { TargetGroupArn: cloudformation.Ref(targetGroupName), }, }, }, Type: elbv2.ActionTypeEnumForward, }, }, LoadBalancerArn: loadBalancer.ARN(), Protocol: protocol, Port: int(port.Target), } return listenerName } func (b *ecsAPIService) createTargetGroup(project *types.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string, vpc string) string { targetGroupName := fmt.Sprintf( "%s%s%dTargetGroup", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), port.Published, ) template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ Port: int(port.Target), Protocol: protocol, Tags: projectTags(project), TargetType: elbv2.TargetTypeEnumIp, VpcId: vpc, } return targetGroupName } func (b *ecsAPIService) createServiceRegistry(service types.ServiceConfig, template *cloudformation.Template, healthCheck *cloudmap.Service_HealthCheckConfig) ecs.Service_ServiceRegistry { serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) serviceRegistry := ecs.Service_ServiceRegistry{ RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), } template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), HealthCheckConfig: healthCheck, HealthCheckCustomConfig: &cloudmap.Service_HealthCheckCustomConfig{ FailureThreshold: 1, }, Name: service.Name, NamespaceId: cloudformation.Ref("CloudMap"), DnsConfig: &cloudmap.Service_DnsConfig{ DnsRecords: []cloudmap.Service_DnsRecord{ { TTL: 60, Type: cloudmapapi.RecordTypeA, }, }, RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, }, } return serviceRegistry } func (b *ecsAPIService) createTaskExecutionRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template) string { taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) policies := b.createPolicies(project, service) template.Resources[taskExecutionRole] = &iam.Role{ AssumeRolePolicyDocument: ecsTaskAssumeRolePolicyDocument, Policies: policies, ManagedPolicyArns: []string{ ecsTaskExecutionPolicy, ecrReadOnlyPolicy, }, Tags: serviceTags(project, service), } return taskExecutionRole } func (b *ecsAPIService) createTaskRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template, resources awsResources) string { taskRole := fmt.Sprintf("%sTaskRole", normalizeResourceName(service.Name)) rolePolicies := []iam.Role_Policy{} if roles, ok := service.Extensions[extensionRole]; ok { rolePolicies = append(rolePolicies, iam.Role_Policy{ PolicyName: fmt.Sprintf("%sPolicy", normalizeResourceName(service.Name)), PolicyDocument: roles, }) } for _, vol := range service.Volumes { rolePolicies = append(rolePolicies, iam.Role_Policy{ PolicyName: fmt.Sprintf("%s%sVolumeMountPolicy", normalizeResourceName(service.Name), normalizeResourceName(vol.Source)), PolicyDocument: volumeMountPolicyDocument(vol.Source, resources.filesystems[vol.Source].ARN()), }) } managedPolicies := []string{} if v, ok := service.Extensions[extensionManagedPolicies]; ok { for _, s := range v.([]interface{}) { managedPolicies = append(managedPolicies, s.(string)) } } if len(rolePolicies) == 0 && len(managedPolicies) == 0 { return "" } template.Resources[taskRole] = &iam.Role{ AssumeRolePolicyDocument: ecsTaskAssumeRolePolicyDocument, Policies: rolePolicies, ManagedPolicyArns: managedPolicies, Tags: serviceTags(project, service), } return taskRole } func (b *ecsAPIService) createCloudMap(project *types.Project, template *cloudformation.Template, vpc string) { template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), Vpc: vpc, } } func (b *ecsAPIService) createPolicies(project *types.Project, service types.ServiceConfig) []iam.Role_Policy { var arns []string if value, ok := service.Extensions[extensionPullCredentials]; ok { arns = append(arns, value.(string)) } for _, secret := range service.Secrets { arns = append(arns, project.Secrets[secret.Source].Name) } if len(arns) > 0 { return []iam.Role_Policy{ { PolicyDocument: &PolicyDocument{ Statement: []PolicyStatement{ { Effect: "Allow", Action: []string{actionGetSecretValue, actionGetParameters, actionDecrypt}, Resource: arns, }, }, }, PolicyName: fmt.Sprintf("%sGrantAccessToSecrets", service.Name), }, } } return nil } func networkResourceName(network string) string { return fmt.Sprintf("%sNetwork", normalizeResourceName(network)) } func serviceResourceName(service string) string { return fmt.Sprintf("%sService", normalizeResourceName(service)) } func volumeResourceName(service string) string { return fmt.Sprintf("%sFilesystem", normalizeResourceName(service)) } func normalizeResourceName(s string) string { return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) }