From 934e7ab9ea2d31df33aa2859ef369703b5ffef9d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 25 Jun 2020 08:14:54 +0200 Subject: [PATCH] don't set service `Name` so they can be updated by CloudFormation Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 4 +- ecs/go.mod | 2 +- ecs/go.sum | 4 ++ ecs/pkg/amazon/backend/backend.go | 6 --- ecs/pkg/amazon/backend/cloudformation.go | 35 +++++++++++------ ecs/pkg/amazon/backend/down.go | 4 +- ecs/pkg/amazon/backend/list.go | 16 +++++--- ecs/pkg/amazon/backend/up.go | 7 +++- ecs/pkg/amazon/backend/wait.go | 5 +-- ecs/pkg/amazon/sdk/api.go | 2 +- ecs/pkg/amazon/sdk/api_mock.go | 14 +++---- ecs/pkg/amazon/sdk/sdk.go | 50 ++++++++++++++++++------ ecs/pkg/compose/tags.go | 7 ++++ 13 files changed, 103 insertions(+), 53 deletions(-) create mode 100644 ecs/pkg/compose/tags.go diff --git a/ecs/Dockerfile b/ecs/Dockerfile index dead5829d..2d2213bd2 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -9,7 +9,8 @@ ENV GO111MODULE=on ARG ALPINE_PKG_DOCKER_VERSION RUN apk add --no-cache \ docker=${ALPINE_PKG_DOCKER_VERSION} \ - make + make \ + build-base COPY go.* . RUN --mount=type=cache,target=/go/pkg/mod \ go mod download @@ -18,7 +19,6 @@ COPY . . FROM base AS make-plugin ARG TARGETOS ARG TARGETARCH -RUN apk add build-base RUN GO111MODULE=on go get github.com/golang/mock/mockgen@latest RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ diff --git a/ecs/go.mod b/ecs/go.mod index 438cf8b0c..16f9c9e3a 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a + github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 diff --git a/ecs/go.sum b/ecs/go.sum index 9edabc47c..5e211f537 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -60,6 +60,10 @@ github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc h1:jZfF+Hz github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc/go.mod h1:d3Vb4tH01Pr4YKD3RvfwguRcezDBUYJTVYgpCSRYSVg= github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a h1:FmEuebUePUA0Kd/NSiCmdPG/n6eKdZdBtIbfejVtRS8= github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= +github.com/compose-spec/compose-go v0.0.0-20200624090650-5d46d553c1e6 h1:9rsA2PlPOv50IOnzSiTqCWrWr3u2q7shPr76Y5hlxF0= +github.com/compose-spec/compose-go v0.0.0-20200624090650-5d46d553c1e6/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= +github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 h1:sVvKsoXizFOuJNc8dM91IeET2/zDNFj3hwHgk437iJ8= +github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go index 0d2f07f98..07764df35 100644 --- a/ecs/pkg/amazon/backend/backend.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -6,12 +6,6 @@ import ( "github.com/docker/ecs-plugin/pkg/amazon/sdk" ) -const ( - ProjectTag = "com.docker.compose.project" - NetworkTag = "com.docker.compose.network" - ServiceTag = "com.docker.compose.service" -) - func NewBackend(profile string, cluster string, region string) (*Backend, error) { sess, err := session.NewSessionWithOptions(session.Options{ Profile: profile, diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index f4111240b..98b9b00bb 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -31,13 +31,22 @@ const ( ) type FargateCompatibilityChecker struct { - *compatibility.AllowList + compatibility.AllowList +} + +func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortConfig) { + if p.Published == 0 { + p.Published = p.Target + } + if p.Published != p.Target { + c.Error("published port can't be set to a distinct value than container port") + } } // Convert a compose project into a CloudFormation template func (b Backend) Convert(project *types.Project) (*cloudformation.Template, error) { - var checker compatibility.Checker = FargateCompatibilityChecker{ - &compatibility.AllowList{ + var checker compatibility.Checker = &FargateCompatibilityChecker{ + compatibility.AllowList{ Supported: []string{ "services.command", "services.container_name", @@ -161,7 +170,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro dependsOn = append(dependsOn, listenerName) serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, - ContainerPort: int(port.Published), + ContainerPort: int(port.Target), TargetGroupArn: cloudformation.Ref(targetGroupName), }) } @@ -195,11 +204,11 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, { - Key: ServiceTag, + Key: compose.ServiceTag, Value: service.Name, }, }, @@ -252,7 +261,7 @@ func createLoadBalancer(project *types.Project, template *cloudformation.Templat }, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, }, @@ -267,7 +276,7 @@ func createListener(service types.ServiceConfig, port types.ServicePortConfig, t "%s%s%dListener", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), - port.Published, + port.Target, ) //add listener to dependsOn //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer @@ -286,7 +295,7 @@ func createListener(service types.ServiceConfig, port types.ServicePortConfig, t }, LoadBalancerArn: loadBalancerARN, Protocol: protocol, - Port: int(port.Published), + Port: int(port.Target), } return listenerName } @@ -304,7 +313,7 @@ func createTargetGroup(project *types.Project, service types.ServiceConfig, port Protocol: protocol, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, }, @@ -371,7 +380,7 @@ func createCluster(project *types.Project, template *cloudformation.Template) st ClusterName: project.Name, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, }, @@ -420,11 +429,11 @@ func convertNetwork(project *types.Project, net types.NetworkConfig, vpc string, VpcId: vpc, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, { - Key: NetworkTag, + Key: compose.NetworkTag, Value: net.Name, }, }, diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index 18b4cbf9a..fcfbc9acf 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -5,6 +5,7 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/console" ) func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { @@ -22,7 +23,8 @@ func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { return err } - err = b.WaitStackCompletion(ctx, name, compose.StackDelete) + w := console.NewProgressWriter() + err = b.WaitStackCompletion(ctx, name, compose.StackDelete, w) if err != nil { return err } diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index c13d2f903..1193d1fcb 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -14,18 +14,22 @@ func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.Ser cluster = project.Name } - status := []compose.ServiceStatus{} - for _, service := range project.Services { - desc, err := b.api.DescribeService(ctx, cluster, service.Name) + status, err := b.api.DescribeServices(ctx, cluster, project.Name) + if err != nil { + return nil, err + } + + for i, state := range status { + s, err := project.GetService(state.Name) if err != nil { return nil, err } ports := []string{} - for _, p := range service.Ports { + for _, p := range s.Ports { ports = append(ports, fmt.Sprintf("*:%d->%d/%s", p.Published, p.Target, p.Protocol)) } - desc.Ports = ports - status = append(status, desc) + state.Ports = ports + status[i] = state } return status, nil } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 24c75d5c6..fae330e62 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -7,6 +7,7 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/console" ) func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { @@ -67,7 +68,11 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { } fmt.Println() - return b.WaitStackCompletion(ctx, project.Name, compose.StackCreate) + w := console.NewProgressWriter() + for k := range template.Resources { + w.ResourceEvent(k, "PENDING", "") + } + return b.WaitStackCompletion(ctx, project.Name, compose.StackCreate, w) } func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, error) { diff --git a/ecs/pkg/amazon/backend/wait.go b/ecs/pkg/amazon/backend/wait.go index 77ad844af..babcaaf64 100644 --- a/ecs/pkg/amazon/backend/wait.go +++ b/ecs/pkg/amazon/backend/wait.go @@ -11,8 +11,7 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int) error { - w := console.NewProgressWriter() +func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int, w console.ProgressWriter) error { knownEvents := map[string]struct{}{} // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name @@ -53,7 +52,7 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio } knownEvents[*event.EventId] = struct{}{} - resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) + resource := aws.StringValue(event.LogicalResourceId) reason := aws.StringValue(event.ResourceStatusReason) status := aws.StringValue(event.ResourceStatus) w.ResourceEvent(resource, status, reason) diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 1ad68afa8..39e27d37d 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -49,7 +49,7 @@ type secretsAPI interface { } type listAPI interface { - DescribeService(ctx context.Context, cluster string, name string) (compose.ServiceStatus, error) + DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) } type waitAPI interface { diff --git a/ecs/pkg/amazon/sdk/api_mock.go b/ecs/pkg/amazon/sdk/api_mock.go index dda73b715..22294ff0c 100644 --- a/ecs/pkg/amazon/sdk/api_mock.go +++ b/ecs/pkg/amazon/sdk/api_mock.go @@ -122,19 +122,19 @@ func (mr *MockAPIMockRecorder) DeleteStack(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0, arg1) } -// DescribeService mocks base method -func (m *MockAPI) DescribeService(arg0 context.Context, arg1, arg2 string) (compose.ServiceStatus, error) { +// DescribeServices mocks base method +func (m *MockAPI) DescribeServices(arg0 context.Context, arg1, arg2 string) ([]compose.ServiceStatus, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeService", arg0, arg1, arg2) - ret0, _ := ret[0].(compose.ServiceStatus) + ret := m.ctrl.Call(m, "DescribeServices", arg0, arg1, arg2) + ret0, _ := ret[0].([]compose.ServiceStatus) ret1, _ := ret[1].(error) return ret0, ret1 } -// DescribeService indicates an expected call of DescribeService -func (mr *MockAPIMockRecorder) DescribeService(arg0, arg1, arg2 interface{}) *gomock.Call { +// DescribeServices indicates an expected call of DescribeServices +func (mr *MockAPIMockRecorder) DescribeServices(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeService", reflect.TypeOf((*MockAPI)(nil).DescribeService), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeServices", reflect.TypeOf((*MockAPI)(nil).DescribeServices), arg0, arg1, arg2) } // DescribeStackEvents mocks base method diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 3cc2c0fa0..1710c314b 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -175,7 +175,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template StackName: aws.String(name), TemplateBody: aws.String(string(json)), Parameters: param, - TimeoutInMinutes: aws.Int64(10), + TimeoutInMinutes: aws.Int64(15), Capabilities: []*string{ aws.String(cloudformation.CapabilityCapabilityIam), }, @@ -341,20 +341,46 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsu } } -func (s sdk) DescribeService(ctx context.Context, cluster string, name string) (compose.ServiceStatus, error) { - services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ - Cluster: aws.String(cluster), - Services: aws.StringSlice([]string{name}), +func (s sdk) DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) { + // TODO handle pagination + list, err := s.ECS.ListServicesWithContext(ctx, &ecs.ListServicesInput{ + Cluster: aws.String(cluster), }) if err != nil { - return compose.ServiceStatus{}, err + return nil, err } - return compose.ServiceStatus{ - ID: *services.Services[0].ServiceName, - Name: name, - Replicas: int(*services.Services[0].RunningCount), - Desired: int(*services.Services[0].DesiredCount), - }, nil + + services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ + Cluster: aws.String(cluster), + Services: list.ServiceArns, + }) + if err != nil { + return nil, err + } + status := []compose.ServiceStatus{} + for _, service := range services.Services { + var name string + var stack string + for _, t := range service.Tags { + switch *t.Key { + case compose.ProjectTag: + stack = *t.Value + case compose.ServiceTag: + name = *t.Value + } + } + if stack != project { + continue + } + status = append(status, compose.ServiceStatus{ + ID: *service.ServiceName, + Name: name, + Replicas: int(*services.Services[0].RunningCount), + Desired: int(*services.Services[0].DesiredCount), + }) + } + + return status, nil } func (s sdk) ListTasks(ctx context.Context, cluster string, family string) ([]string, error) { diff --git a/ecs/pkg/compose/tags.go b/ecs/pkg/compose/tags.go new file mode 100644 index 000000000..43236d45f --- /dev/null +++ b/ecs/pkg/compose/tags.go @@ -0,0 +1,7 @@ +package compose + +const ( + ProjectTag = "com.docker.compose.project" + NetworkTag = "com.docker.compose.network" + ServiceTag = "com.docker.compose.service" +)