diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go
index 51a37771f..83624f3dc 100644
--- a/pkg/compose/build_bake.go
+++ b/pkg/compose/build_bake.go
@@ -295,9 +295,8 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project
 
 	// Use docker/cli mechanism to propagate termination signal to child process
 	server, err := socket.NewPluginServer(nil)
-	if err != nil {
+	if err == nil {
 		defer server.Close() //nolint:errcheck
-		cmd.Cancel = server.Close
 		cmd.Env = replace(cmd.Env, socket.EnvKey, server.Addr().String())
 	}
 
diff --git a/pkg/compose/plugins.go b/pkg/compose/plugins.go
index 756a4b09e..66cfc53fc 100644
--- a/pkg/compose/plugins.go
+++ b/pkg/compose/plugins.go
@@ -207,7 +207,6 @@ func (s *composeService) setupPluginCommand(ctx context.Context, project *types.
 	server, err := socket.NewPluginServer(nil)
 	if err == nil {
 		defer server.Close() //nolint:errcheck
-		cmd.Cancel = server.Close
 		cmd.Env = replace(cmd.Env, socket.EnvKey, server.Addr().String())
 	}