diff --git a/azure/aci.go b/azure/aci.go
index 7404483d8..964a2ca17 100644
--- a/azure/aci.go
+++ b/azure/aci.go
@@ -98,11 +98,12 @@ func createACIContainers(ctx context.Context, aciContext store.AciContext, group
 	return err
 }
 
-func deleteACIContainerGroup(ctx context.Context, aciContext store.AciContext, containerGroupName string) (c containerinstance.ContainerGroup, err error) {
+func deleteACIContainerGroup(ctx context.Context, aciContext store.AciContext, containerGroupName string) (containerinstance.ContainerGroup, error) {
 	containerGroupsClient, err := getContainerGroupsClient(aciContext.SubscriptionID)
 	if err != nil {
-		return c, fmt.Errorf("cannot get container group client: %v", err)
+		return containerinstance.ContainerGroup{}, fmt.Errorf("cannot get container group client: %v", err)
 	}
+
 	return containerGroupsClient.Delete(ctx, aciContext.ResourceGroup, containerGroupName)
 }
 
@@ -271,7 +272,7 @@ func getSubscriptionsClient() subscription.SubscriptionsClient {
 	return subc
 }
 
-//GetGroupsClient ...
+// GetGroupsClient ...
 func GetGroupsClient(subscriptionID string) resources.GroupsClient {
 	groupsClient := resources.NewGroupsClient(subscriptionID)
 	authorizer, _ := auth.NewAuthorizerFromCLI()
@@ -279,7 +280,7 @@ func GetGroupsClient(subscriptionID string) resources.GroupsClient {
 	return groupsClient
 }
 
-//GetSubscriptionID ...
+// GetSubscriptionID ...
 func GetSubscriptionID(ctx context.Context) (string, error) {
 	c := getSubscriptionsClient()
 	res, err := c.List(ctx)
diff --git a/azure/backend.go b/azure/backend.go
index 1a6e7bdf4..f1fedddff 100644
--- a/azure/backend.go
+++ b/azure/backend.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"strconv"
 	"strings"
 
@@ -23,6 +24,9 @@ import (
 
 const singleContainerName = "single--container--aci"
 
+// ErrNoSuchContainer is returned when the mentioned container does not exist
+var ErrNoSuchContainer = errors.New("no such container")
+
 func init() {
 	backend.Register("aci", "aci", func(ctx context.Context) (backend.Service, error) {
 		return New(ctx)
@@ -214,6 +218,18 @@ func (cs *aciContainerService) Logs(ctx context.Context, containerName string, r
 	return err
 }
 
+func (cs *aciContainerService) Delete(ctx context.Context, containerID string, _ bool) error {
+	cg, err := deleteACIContainerGroup(ctx, cs.ctx, containerID)
+	if err != nil {
+		return err
+	}
+	if cg.StatusCode == http.StatusNoContent {
+		return ErrNoSuchContainer
+	}
+
+	return err
+}
+
 type aciComposeService struct {
 	containerGroupsClient containerinstance.ContainerGroupsClient
 	ctx                   store.AciContext
@@ -239,6 +255,14 @@ func (cs *aciComposeService) Down(ctx context.Context, opts compose.ProjectOptio
 		return err
 	}
 	logrus.Debugf("Down on project with name %q\n", project.Name)
-	_, err = deleteACIContainerGroup(ctx, cs.ctx, project.Name)
+
+	cg, err := deleteACIContainerGroup(ctx, cs.ctx, project.Name)
+	if err != nil {
+		return err
+	}
+	if cg.StatusCode == http.StatusNoContent {
+		return ErrNoSuchContainer
+	}
+
 	return err
 }
diff --git a/cli/cmd/rm.go b/cli/cmd/rm.go
new file mode 100644
index 000000000..6a962019d
--- /dev/null
+++ b/cli/cmd/rm.go
@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/docker/api/client"
+)
+
+type rmOpts struct {
+	force bool
+}
+
+// RmCommand deletes containers
+func RmCommand() *cobra.Command {
+	var opts rmOpts
+	cmd := &cobra.Command{
+		Use:     "rm",
+		Aliases: []string{"delete"},
+		Short:   "Remove containers",
+		Args:    cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var errs []string
+			c, err := client.New(cmd.Context())
+			if err != nil {
+				return errors.Wrap(err, "cannot connect to backend")
+			}
+
+			for _, id := range args {
+				err := c.ContainerService().Delete(cmd.Context(), id, opts.force)
+				if err != nil {
+					errs = append(errs, err.Error())
+					continue
+				}
+				println(id)
+			}
+
+			if len(errs) > 0 {
+				return errors.New(strings.Join(errs, "\n"))
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().BoolVarP(&opts.force, "force", "f", false, "Force removal")
+
+	return cmd
+}
diff --git a/cli/main.go b/cli/main.go
index ce569983f..22ae0cdce 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -108,6 +108,7 @@ func main() {
 		run.Command(),
 		cmd.ExecCommand(),
 		cmd.LogsCommand(),
+		cmd.RmCommand(),
 		compose.Command(),
 	)
 
diff --git a/containers/api.go b/containers/api.go
index f3ba59454..77d9aaead 100644
--- a/containers/api.go
+++ b/containers/api.go
@@ -54,4 +54,6 @@ type Service interface {
 	Exec(ctx context.Context, containerName string, command string, reader io.Reader, writer io.Writer) error
 	// Logs returns all the logs of a container
 	Logs(ctx context.Context, containerName string, request LogsRequest) error
+	// Delete removes containers
+	Delete(ctx context.Context, id string, force bool) error
 }
diff --git a/example/backend.go b/example/backend.go
index be00e0c19..e167191af 100644
--- a/example/backend.go
+++ b/example/backend.go
@@ -59,6 +59,11 @@ func (cs *containerService) Logs(ctx context.Context, containerName string, requ
 	return nil
 }
 
+func (cs *containerService) Delete(ctx context.Context, id string, force bool) error {
+	fmt.Printf("Deleting container %q with force = %t\n", id, force)
+	return nil
+}
+
 type composeService struct{}
 
 func (cs *composeService) Up(ctx context.Context, opts compose.ProjectOptions) error {
diff --git a/server/proxy/proxy.go b/server/proxy/proxy.go
index a30c38801..787e4501e 100644
--- a/server/proxy/proxy.go
+++ b/server/proxy/proxy.go
@@ -72,8 +72,13 @@ func (p *proxyContainerAPI) Kill(_ context.Context, _ *v1.KillRequest) (*v1.Kill
 	panic("not implemented") // TODO: Implement
 }
 
-func (p *proxyContainerAPI) Delete(_ context.Context, _ *v1.DeleteRequest) (*v1.DeleteResponse, error) {
-	panic("not implemented") // TODO: Implement
+func (p *proxyContainerAPI) Delete(ctx context.Context, request *v1.DeleteRequest) (*v1.DeleteResponse, error) {
+	err := Client(ctx).ContainerService().Delete(ctx, request.Id, request.Force)
+	if err != nil {
+		return &v1.DeleteResponse{}, err
+	}
+
+	return &v1.DeleteResponse{}, nil
 }
 
 func (p *proxyContainerAPI) Update(_ context.Context, _ *v1.UpdateRequest) (*v1.UpdateResponse, error) {