From 757b9bb22168c1b26b86717efc1812c81ab3db83 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 30 Sep 2020 14:35:07 +0200 Subject: [PATCH 1/2] Introduce x-aws-autoscale to support CPU based autoscaling Signed-off-by: Nicolas De Loof --- ecs/architecture.md | 65 +++++++------ ecs/autoscaling.go | 93 +++++++++++++++++++ ecs/autoscaling_test.go | 41 ++++++++ ecs/cloudformation.go | 8 +- ecs/iam.go | 48 +++++----- .../simple-cloudformation-conversion.golden | 10 ++ ecs/x.go | 1 + 7 files changed, 213 insertions(+), 53 deletions(-) create mode 100644 ecs/autoscaling.go create mode 100644 ecs/autoscaling_test.go diff --git a/ecs/architecture.md b/ecs/architecture.md index f85616719..113c43501 100644 --- a/ecs/architecture.md +++ b/ecs/architecture.md @@ -8,38 +8,49 @@ This document describes the mapping between compose application model and AWS co This diagram shows compose model and on same line AWS components that get created as equivalent resources ``` -+----------+ +-------------+ +-------------------+ -| Project | | Cluster | | LoadBalancer | -+-+--------+ +-------------+ +-------------------+ ++----------+ +-------------+ +-------------------+ +| Project | . . . . . . . . . . . . . . | Cluster | . . . . . . . | LoadBalancer | ++-+--------+ +-------------+ +-------------------+ | - | +----------+ +-------------+ +----------------+ +-------------------+ - +----+ Service | | Service | | TaskDefinition | | TargetGroup | - | +--+-------+ +-------------+ +----------------+ +-------------------+ - | | +----------------+ - | | x-aws-role, x-aws-policies | TaskRole | - | | +----------------+ - | | +---------+ +-------------+ +-------------------+ - | +--+ Ports | | IngressRule | | Listener | - | | +---------+ +-------------+ +-------------------+ + | +----------+ +-------------++-------------------+ +-------------------+ + +----+ Service | . . . . . . . . . . | Service || TaskDefinition | | TargetGroup | + | +--+-------+ +-------------++-------------------+-+ +-------------------+ + | | | TaskRole | + | | +-------------------+-+ + | | x-aws-role, x-aws-policies . . . . . . . . | TaskExecutionRole | + | | +-------------------+ + | | +---------+ + | +--+ Deploy | + | | +---------+ +-------------------+ + | | x-aws-autoscale . . . . . . | ScalableTarget | + | | +-------------------+---+ + | | | ScalingPolicy | + | | +-------------------+-+ + | | | AutoScalingRole | + | | +-------------------+ | | + | | +---------+ +-------------+ +-------------------+ + | +--+ Ports | . . . . . . . | IngressRule +-----+ | Listener | + | | +---------+ +-------------+ | +-------------------+ + | | | | | +---------+ +---------------+ +------------------+ - | +--+ Secrets | | InitContainer | |TaskExecutionRole | + | +--+ Secrets | . . . . . . . | InitContainer | |TaskExecutionRole | | | +---------+ +---------------+ +------------+-----+ - | | | - | | +---------+ | - | +--+ Volumes | | - | | +---------+ | - | | | - | | +---------------+ | +------------------------------------------+ - | +--+ DeviceRequest | | | CapacityProvider || AutoscalingGroup | - | +---------------+ | +------------------------------------------+ - | | | LaunchConfiguration | - | +------------+ +---------------+ | +---------------------+ - +---+ Networks | | SecurityGroup | | - | +------------+ +---------------+ | + | | | | + | | +---------+ | | + | +--+ Volumes | | | + | | +---------+ | | + | | | | + | | +---------------+ | | +-------------------+ + | +--+ DeviceRequest | . . . . . . . . . . . . . . . | . . . . | . . . | CapacityProvider | + | +---------------+ | | +-------------------+--------+ + | | | | AutoscalingGroup | + | +------------+ +---------------+ | | +---------------------+ + +---+ Networks | . . . . . . . . . | SecurityGroup +---+ | | LaunchConfiguration | + | +------------+ +---------------+ | +---------------------+ | | | +------------+ +---------------+ | - +---+ Secret | | Secret +--------------+ + +---+ Secret | . . . . . . . . . | Secret +--------------+ +------------+ +---------------+ ``` @@ -63,6 +74,6 @@ A `TaskExecutionRole` is also created per service, and is updated to grant acces Services using a GPU (`DeviceRequest`) get the `Cluster` extended with an EC2 `CapacityProvider`, using an `AutoscalingGroup` to manage EC2 resources allocation based on a `LaunchConfiguration`. The latter uses ECS recommended AMI and machine type for GPU. - +Service to declare `deploy.x-aws-autoscaling` get a `ScalingPolicy` created targeting specified the configured CPU usage metric diff --git a/ecs/autoscaling.go b/ecs/autoscaling.go new file mode 100644 index 000000000..7a40938d7 --- /dev/null +++ b/ecs/autoscaling.go @@ -0,0 +1,93 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ecs + +import ( + "fmt" + + applicationautoscaling2 "github.com/aws/aws-sdk-go/service/applicationautoscaling" + "github.com/awslabs/goformation/v4/cloudformation" + "github.com/awslabs/goformation/v4/cloudformation/applicationautoscaling" + "github.com/awslabs/goformation/v4/cloudformation/iam" + "github.com/compose-spec/compose-go/types" +) + +func (b *ecsAPIService) createAutoscalingPolicy(project *types.Project, resources awsResources, template *cloudformation.Template, service types.ServiceConfig) { + if service.Deploy == nil { + return + } + v, ok := service.Deploy.Extensions[extensionAutoScaling] + if !ok { + return + } + + role := fmt.Sprintf("%sAutoScalingRole", normalizeResourceName(service.Name)) + template.Resources[role] = &iam.Role{ + AssumeRolePolicyDocument: ausocalingAssumeRolePolicyDocument, + Path: "/", + Policies: []iam.Role_Policy{ + { + PolicyDocument: &PolicyDocument{ + Statement: []PolicyStatement{ + { + Effect: "Allow", + Action: []string{ + actionAutoScaling, + actionDescribeService, + actionUpdateService, + actionGetMetrics, + }, + Resource: []string{cloudformation.Ref(serviceResourceName(service.Name))}, + }, + }, + }, + PolicyName: "service-autoscaling", + }, + }, + Tags: serviceTags(project, service), + } + + // Why isn't this just the service ARN ????? + resourceID := cloudformation.Join("/", []string{"service", resources.cluster, cloudformation.GetAtt(serviceResourceName(service.Name), "Name")}) + + target := fmt.Sprintf("%sScalableTarget", normalizeResourceName(service.Name)) + template.Resources[target] = &applicationautoscaling.ScalableTarget{ + MaxCapacity: 10, + MinCapacity: 0, + ResourceId: resourceID, + RoleARN: cloudformation.GetAtt(role, "Arn"), + ScalableDimension: applicationautoscaling2.ScalableDimensionEcsServiceDesiredCount, + ServiceNamespace: applicationautoscaling2.ServiceNamespaceEcs, + AWSCloudFormationDependsOn: []string{serviceResourceName(service.Name)}, + } + + policy := fmt.Sprintf("%sScalingPolicy", normalizeResourceName(service.Name)) + template.Resources[policy] = &applicationautoscaling.ScalingPolicy{ + PolicyType: "TargetTrackingScaling", + PolicyName: policy, + ScalingTargetId: cloudformation.Ref(target), + StepScalingPolicyConfiguration: nil, + TargetTrackingScalingPolicyConfiguration: &applicationautoscaling.ScalingPolicy_TargetTrackingScalingPolicyConfiguration{ + PredefinedMetricSpecification: &applicationautoscaling.ScalingPolicy_PredefinedMetricSpecification{ + PredefinedMetricType: applicationautoscaling2.MetricTypeEcsserviceAverageCpuutilization, + }, + ScaleOutCooldown: 60, + ScaleInCooldown: 60, + TargetValue: float64(v.(int)), + }, + } +} diff --git a/ecs/autoscaling_test.go b/ecs/autoscaling_test.go new file mode 100644 index 000000000..1a4116147 --- /dev/null +++ b/ecs/autoscaling_test.go @@ -0,0 +1,41 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ecs + +import ( + "testing" + + autoscaling "github.com/awslabs/goformation/v4/cloudformation/applicationautoscaling" + "gotest.tools/v3/assert" +) + +func TestAutoScaling(t *testing.T) { + template := convertYaml(t, ` +services: + foo: + image: hello_world + deploy: + x-aws-autoscaling: 75 +`) + target := template.Resources["FooScalableTarget"].(*autoscaling.ScalableTarget) + assert.Check(t, target != nil) + policy := template.Resources["FooScalingPolicy"].(*autoscaling.ScalingPolicy) + if policy == nil || policy.TargetTrackingScalingPolicyConfiguration == nil { + t.Fail() + } + assert.Check(t, policy.TargetTrackingScalingPolicyConfiguration.TargetValue == float64(75)) +} diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go index 0b9cb93e8..b9f7c61c0 100644 --- a/ecs/cloudformation.go +++ b/ecs/cloudformation.go @@ -91,7 +91,7 @@ func (b *ecsAPIService) convert(project *types.Project, resources awsResources) for _, service := range project.Services { taskExecutionRole := b.createTaskExecutionRole(project, service, template) - taskRole := b.createTaskRole(service, template) + taskRole := b.createTaskRole(project, service, template) definition, err := b.createTaskExecution(project, service) if err != nil { @@ -183,6 +183,8 @@ func (b *ecsAPIService) convert(project *types.Project, resources awsResources) Tags: serviceTags(project, service), TaskDefinition: cloudformation.Ref(normalizeResourceName(taskDefinition)), } + + b.createAutoscalingPolicy(project, resources, template, service) } return template, nil } @@ -363,11 +365,12 @@ func (b *ecsAPIService) createTaskExecutionRole(project *types.Project, service ecsTaskExecutionPolicy, ecrReadOnlyPolicy, }, + Tags: serviceTags(project, service), } return taskExecutionRole } -func (b *ecsAPIService) createTaskRole(service types.ServiceConfig, template *cloudformation.Template) string { +func (b *ecsAPIService) createTaskRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template) string { taskRole := fmt.Sprintf("%sTaskRole", normalizeResourceName(service.Name)) rolePolicies := []iam.Role_Policy{} if roles, ok := service.Extensions[extensionRole]; ok { @@ -388,6 +391,7 @@ func (b *ecsAPIService) createTaskRole(service types.ServiceConfig, template *cl AssumeRolePolicyDocument: ecsTaskAssumeRolePolicyDocument, Policies: rolePolicies, ManagedPolicyArns: managedPolicies, + Tags: serviceTags(project, service), } return taskRole } diff --git a/ecs/iam.go b/ecs/iam.go index 3a5174823..5b0611723 100644 --- a/ecs/iam.go +++ b/ecs/iam.go @@ -21,35 +21,35 @@ const ( ecrReadOnlyPolicy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" ecsEC2InstanceRole = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" - actionGetSecretValue = "secretsmanager:GetSecretValue" - actionGetParameters = "ssm:GetParameters" - actionDecrypt = "kms:Decrypt" + actionGetSecretValue = "secretsmanager:GetSecretValue" + actionGetParameters = "ssm:GetParameters" + actionDecrypt = "kms:Decrypt" + actionAutoScaling = "application-autoscaling:*" + actionGetMetrics = "cloudwatch:GetMetricStatistics" + actionDescribeService = "ecs:DescribeServices" + actionUpdateService = "ecs:UpdateService" ) -var ecsTaskAssumeRolePolicyDocument = PolicyDocument{ - Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html - Statement: []PolicyStatement{ - { - Effect: "Allow", - Principal: PolicyPrincipal{ - Service: "ecs-tasks.amazonaws.com", - }, - Action: []string{"sts:AssumeRole"}, - }, - }, -} +var ( + ecsTaskAssumeRolePolicyDocument = policyDocument("ecs-tasks.amazonaws.com") + ec2InstanceAssumeRolePolicyDocument = policyDocument("ec2.amazonaws.com") + ausocalingAssumeRolePolicyDocument = policyDocument("application-autoscaling.amazonaws.com") +) -var ec2InstanceAssumeRolePolicyDocument = PolicyDocument{ - Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html - Statement: []PolicyStatement{ - { - Effect: "Allow", - Principal: PolicyPrincipal{ - Service: "ec2.amazonaws.com", +func policyDocument(service string) PolicyDocument { + return PolicyDocument{ + Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html + Statement: []PolicyStatement{ + { + Effect: "Allow", + Principal: PolicyPrincipal{ + Service: service, + }, + Action: []string{"sts:AssumeRole"}, }, - Action: []string{"sts:AssumeRole"}, }, - }, + } + } // PolicyDocument describes an IAM policy document diff --git a/ecs/testdata/simple/simple-cloudformation-conversion.golden b/ecs/testdata/simple/simple-cloudformation-conversion.golden index 56ec12f06..5e1bc7798 100644 --- a/ecs/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/testdata/simple/simple-cloudformation-conversion.golden @@ -301,6 +301,16 @@ "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ], + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + }, + { + "Key": "com.docker.compose.service", + "Value": "simple" + } ] }, "Type": "AWS::IAM::Role" diff --git a/ecs/x.go b/ecs/x.go index 9b93436c2..6968ba997 100644 --- a/ecs/x.go +++ b/ecs/x.go @@ -29,4 +29,5 @@ const ( extensionRetention = "x-aws-logs_retention" extensionRole = "x-aws-role" extensionManagedPolicies = "x-aws-policies" + extensionAutoScaling = "x-aws-autoscaling" ) From 20a8f01269b7646e7be23b09959afc2c17747309 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 30 Sep 2020 15:17:45 +0200 Subject: [PATCH 2/2] Don't set securityGroup name, as all compose apps will create a `DefaultNetwork` Signed-off-by: Nicolas De Loof --- ecs/awsResources.go | 1 - ecs/testdata/simple/simple-cloudformation-conversion.golden | 1 - 2 files changed, 2 deletions(-) diff --git a/ecs/awsResources.go b/ecs/awsResources.go index 2ba44e5e5..829fc6bbc 100644 --- a/ecs/awsResources.go +++ b/ecs/awsResources.go @@ -189,7 +189,6 @@ func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project, securityGroup := networkResourceName(name) template.Resources[securityGroup] = &ec2.SecurityGroup{ GroupDescription: fmt.Sprintf("%s Security Group for %s network", project.Name, name), - GroupName: securityGroup, VpcId: r.vpc, Tags: networkTags(project, net), } diff --git a/ecs/testdata/simple/simple-cloudformation-conversion.golden b/ecs/testdata/simple/simple-cloudformation-conversion.golden index 5e1bc7798..07200659c 100644 --- a/ecs/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/testdata/simple/simple-cloudformation-conversion.golden @@ -37,7 +37,6 @@ "DefaultNetwork": { "Properties": { "GroupDescription": "TestSimpleConvert Security Group for default network", - "GroupName": "DefaultNetwork", "Tags": [ { "Key": "com.docker.compose.project",