diff --git a/plum/tests/service_test.py b/plum/tests/service_test.py
index 6075965e9..0c03a2ab8 100644
--- a/plum/tests/service_test.py
+++ b/plum/tests/service_test.py
@@ -8,8 +8,9 @@ class ServiceTestCase(TestCase):
         self.client = Client('http://127.0.0.1:4243')
         self.client.pull('ubuntu')
 
-        for c in self.client.containers():
+        for c in self.client.containers(all=True):
             self.client.kill(c['Id'])
+            self.client.remove_container(c['Id'])
 
         self.service = Service(
             client=self.client,