diff --git a/compose/config.py b/compose/config.py
index c89f6de55..dea734d51 100644
--- a/compose/config.py
+++ b/compose/config.py
@@ -189,6 +189,12 @@ def merge_service_dicts(base, override):
             override.get('volumes'),
         )
 
+    if 'image' in override and 'build' in d:
+        del d['build']
+
+    if 'build' in override and 'image' in d:
+        del d['image']
+
     for k in ALLOWED_KEYS:
         if k not in ['environment', 'volumes']:
             if k in override:
diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py
index 8644d354b..280034449 100644
--- a/tests/unit/config_test.py
+++ b/tests/unit/config_test.py
@@ -80,6 +80,39 @@ class MergeTest(unittest.TestCase):
         )
         self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data']))
 
+    def test_merge_build_or_image_no_override(self):
+        self.assertEqual(
+            config.merge_service_dicts({'build': '.'}, {}),
+            {'build': '.'},
+        )
+
+        self.assertEqual(
+            config.merge_service_dicts({'image': 'redis'}, {}),
+            {'image': 'redis'},
+        )
+
+    def test_merge_build_or_image_override_with_same(self):
+        self.assertEqual(
+            config.merge_service_dicts({'build': '.'}, {'build': './web'}),
+            {'build': './web'},
+        )
+
+        self.assertEqual(
+            config.merge_service_dicts({'image': 'redis'}, {'image': 'postgres'}),
+            {'image': 'postgres'},
+        )
+
+    def test_merge_build_or_image_override_with_other(self):
+        self.assertEqual(
+            config.merge_service_dicts({'build': '.'}, {'image': 'redis'}),
+            {'image': 'redis'}
+        )
+
+        self.assertEqual(
+            config.merge_service_dicts({'image': 'redis'}, {'build': '.'}),
+            {'build': '.'}
+        )
+
 
 class EnvTest(unittest.TestCase):
     def test_parse_environment_as_list(self):