diff --git a/compose/project.py b/compose/project.py
index 92c352050..78b5a3569 100644
--- a/compose/project.py
+++ b/compose/project.py
@@ -627,7 +627,7 @@ class Project(object):
 
     def find_orphan_containers(self, remove_orphans):
         def _find():
-            containers = self._labeled_containers()
+            containers = set(self._labeled_containers() + self._labeled_containers(stopped=True))
             for ctnr in containers:
                 service_name = ctnr.labels.get(LABEL_SERVICE)
                 if service_name not in self.service_names:
@@ -638,7 +638,10 @@ class Project(object):
         if remove_orphans:
             for ctnr in orphans:
                 log.info('Removing orphan container "{0}"'.format(ctnr.name))
-                ctnr.kill()
+                try:
+                    ctnr.kill()
+                except APIError:
+                    pass
                 ctnr.remove(force=True)
         else:
             log.warning(
diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py
index 5b0a0e0fd..1844b757a 100644
--- a/tests/acceptance/cli_test.py
+++ b/tests/acceptance/cli_test.py
@@ -12,6 +12,7 @@ import subprocess
 import time
 from collections import Counter
 from collections import namedtuple
+from functools import reduce
 from operator import attrgetter
 
 import pytest
@@ -1099,6 +1100,22 @@ class CLITestCase(DockerClientTestCase):
         ]
         assert len(remote_volumes) > 0
 
+    @v2_only()
+    def test_up_no_start_remove_orphans(self):
+        self.base_dir = 'tests/fixtures/v2-simple'
+        self.dispatch(['up', '--no-start'], None)
+
+        services = self.project.get_services()
+
+        stopped = reduce((lambda prev, next: prev.containers(
+            stopped=True) + next.containers(stopped=True)), services)
+        assert len(stopped) == 2
+
+        self.dispatch(['-f', 'one-container.yml', 'up', '--no-start', '--remove-orphans'], None)
+        stopped2 = reduce((lambda prev, next: prev.containers(
+            stopped=True) + next.containers(stopped=True)), services)
+        assert len(stopped2) == 1
+
     @v2_only()
     def test_up_no_ansi(self):
         self.base_dir = 'tests/fixtures/v2-simple'
diff --git a/tests/fixtures/v2-simple/one-container.yml b/tests/fixtures/v2-simple/one-container.yml
new file mode 100644
index 000000000..22cd9863c
--- /dev/null
+++ b/tests/fixtures/v2-simple/one-container.yml
@@ -0,0 +1,5 @@
+version: "2"
+services:
+  simple:
+    image: busybox:latest
+    command: top