From 5691b8241dbf11e893d7d2b295b2dfb4ac7c6b1b Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 9 Nov 2017 17:53:27 -0600 Subject: [PATCH 1/4] Implement subnet config validation (fixes #4552) Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.5.json | 2 +- compose/config/validation.py | 30 +++++++++- tests/unit/config/config_test.py | 82 ++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index fa95d6a24..6ccecbfd4 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -418,7 +418,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/validation.py b/compose/config/validation.py index 8247cf150..a8061a5a4 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,6 +5,7 @@ import json import logging import os import re +import socket import sys import six @@ -43,6 +44,9 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' +VALID_IPV4_FORMAT = r'^(\d{1,3}.){3}\d{1,3}$' +VALID_IPV4_CIDR_FORMAT = r'^(\d|[1-2]\d|3[0-2])$' +VALID_IPV6_CIDR_FORMAT = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -64,6 +68,30 @@ def format_expose(instance): return True +@FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) +def format_subnet_ip_address(instance): + if isinstance(instance, six.string_types): + if '/' not in instance: + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + + ip_address, cidr = instance.split('/') + + if re.match(VALID_IPV4_FORMAT, ip_address): + if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and + all(0 <= int(component) <= 255 for component in ip_address.split("."))): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): + try: + if not (socket.inet_pton(socket.AF_INET6, ip_address)): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + except socket.error as e: + raise ValidationError(six.text_type(e)) + else: + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + + return True + + def match_named_volumes(service_dict, project_volumes): service_volumes = service_dict.get('volumes', []) for volume_spec in service_volumes: @@ -391,7 +419,7 @@ def process_config_schema_errors(error): def validate_against_config_schema(config_file): schema = load_jsonschema(config_file) - format_checker = FormatChecker(["ports", "expose"]) + format_checker = FormatChecker(["ports", "expose", "subnet_ip_address"]) validator = Draft4Validator( schema, resolver=RefResolver(get_resolver_path(), schema), diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a758154c0..819d8f5be 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2846,6 +2846,88 @@ class PortsTest(unittest.TestCase): ) +class SubnetTest(unittest.TestCase): + INVALID_SUBNET_TYPES = [ + None, + False, + 10, + ] + + INVALID_SUBNET_MAPPINGS = [ + "", + "192.168.0.1/sdfsdfs", + "192.168.0.1/", + "192.168.0.1/33", + "192.168.0.1/01", + "192.168.0.1", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/sdfsdfs", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/129", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156", + ] + + ILLEGAL_SUBNET_MAPPINGS = [ + "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128" + ] + + VALID_SUBNET_MAPPINGS = [ + "192.168.0.1/0", + "192.168.0.1/32", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/0", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/128", + ] + + def test_config_invalid_subnet_type_validation(self): + for invalid_subnet in self.INVALID_SUBNET_TYPES: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "contains an invalid type" in exc.value.msg + + def test_config_invalid_subnet_format_validation(self): + for invalid_subnet in self.INVALID_SUBNET_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg + + def test_config_illegal_subnet_type_validation(self): + for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "illegal IP address string" in exc.value.msg + + def test_config_valid_subnet_format_validation(self): + for valid_subnet in self.VALID_SUBNET_MAPPINGS: + self.check_config(valid_subnet) + + def check_config(self, subnet): + config.load( + build_config_details({ + 'version': '3.5', + 'services': { + 'web': { + 'image': 'busybox' + } + }, + 'networks': { + 'default': { + 'ipam': { + 'config': [ + { + 'subnet': subnet + } + ], + 'driver': 'default' + } + } + } + }) + ) + + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 68c636d728be2b9a2a2d0e7a464e2d5be8cfe651 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 9 Nov 2017 22:57:47 -0600 Subject: [PATCH 2/4] Fix subnet config test for windows Signed-off-by: Drew Romanyk --- compose/config/validation.py | 10 ++++++---- tests/unit/config/config_test.py | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index a8061a5a4..c2256804b 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -72,22 +72,24 @@ def format_expose(instance): def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): if '/' not in instance: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError("'{0}' 75 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) ip_address, cidr = instance.split('/') if re.match(VALID_IPV4_FORMAT, ip_address): if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and all(0 <= int(component) <= 255 for component in ip_address.split("."))): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError( + "'{0}' 83 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): try: if not (socket.inet_pton(socket.AF_INET6, ip_address)): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError( + "'{0}' 88 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) except socket.error as e: raise ValidationError(six.text_type(e)) else: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError("'{0}' 92 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 819d8f5be..51323cd32 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2896,8 +2896,11 @@ class SubnetTest(unittest.TestCase): for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: with pytest.raises(ConfigurationError) as exc: self.check_config(invalid_subnet) - - assert "illegal IP address string" in exc.value.msg + if IS_WINDOWS_PLATFORM: + assert "An invalid argument was supplied" in exc.value.msg or \ + "illegal IP address string" in exc.value.msg + else: + assert "illegal IP address string" in exc.value.msg def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: From 6c8184d0d02cda9e67aaedad2f37ae754729f416 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Fri, 10 Nov 2017 18:04:11 -0600 Subject: [PATCH 3/4] Add format to other v3 configs & remove unix dependency Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.0.json | 2 +- compose/config/config_schema_v3.1.json | 2 +- compose/config/config_schema_v3.2.json | 2 +- compose/config/config_schema_v3.3.json | 2 +- compose/config/config_schema_v3.4.json | 2 +- compose/config/validation.py | 52 +++++++++++++++++--------- tests/unit/config/config_test.py | 30 ++++++++------- 7 files changed, 55 insertions(+), 37 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index f39344cfb..fa601bed2 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -294,7 +294,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 719c0fa7a..41da89650 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -323,7 +323,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index 8d850d5d2..e4f8fe6d8 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -368,7 +368,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json index f1eb9a661..96dc1d7d0 100644 --- a/compose/config/config_schema_v3.3.json +++ b/compose/config/config_schema_v3.3.json @@ -412,7 +412,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json index dae7d7d23..8089c7e6d 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4.json @@ -420,7 +420,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/validation.py b/compose/config/validation.py index c2256804b..f97069935 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,7 +5,6 @@ import json import logging import os import re -import socket import sys import six @@ -44,9 +43,32 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' -VALID_IPV4_FORMAT = r'^(\d{1,3}.){3}\d{1,3}$' -VALID_IPV4_CIDR_FORMAT = r'^(\d|[1-2]\d|3[0-2])$' -VALID_IPV6_CIDR_FORMAT = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' + +VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' +VALID_REGEX_IPV4_CIDR = r'^(\d|[1-2]\d|3[0-2])$' +VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) +VALID_REGEX_IPV4_ADDR = "^{IPV4_ADDR}$".format(IPV4_ADDR=VALID_IPV4_ADDR) + +VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' +VALID_REGEX_IPV6_CIDR = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' +VALID_REGEX_IPV6_ADDR = "".join(""" +^ +( + (({IPV6_SEG}:){{7}}{IPV6_SEG})| + (({IPV6_SEG}:){{1,7}}:)| + (({IPV6_SEG}:){{1,6}}(:{IPV6_SEG}){{1,1}})| + (({IPV6_SEG}:){{1,5}}(:{IPV6_SEG}){{1,2}})| + (({IPV6_SEG}:){{1,4}}(:{IPV6_SEG}){{1,3}})| + (({IPV6_SEG}:){{1,3}}(:{IPV6_SEG}){{1,4}})| + (({IPV6_SEG}:){{1,2}}(:{IPV6_SEG}){{1,5}})| + (({IPV6_SEG}:){{1,1}}(:{IPV6_SEG}){{1,6}})| + (:((:{IPV6_SEG}){{1,7}}|:))| + (fe80:(:{IPV6_SEG}){{0,4}}%[0-9a-zA-Z]{{1,}})| + (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| + (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) +) +$ +""".format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -72,24 +94,18 @@ def format_expose(instance): def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): if '/' not in instance: - raise ValidationError("'{0}' 75 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") ip_address, cidr = instance.split('/') - if re.match(VALID_IPV4_FORMAT, ip_address): - if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and - all(0 <= int(component) <= 255 for component in ip_address.split("."))): - raise ValidationError( - "'{0}' 83 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) - elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): - try: - if not (socket.inet_pton(socket.AF_INET6, ip_address)): - raise ValidationError( - "'{0}' 88 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) - except socket.error as e: - raise ValidationError(six.text_type(e)) + if re.match(VALID_REGEX_IPV4_ADDR, ip_address): + if not re.match(VALID_REGEX_IPV4_CIDR, cidr): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + elif re.match(VALID_REGEX_IPV6_ADDR, ip_address): + if not re.match(VALID_REGEX_IPV6_CIDR, cidr): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") else: - raise ValidationError("'{0}' 92 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 51323cd32..1cf783c77 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2865,10 +2865,7 @@ class SubnetTest(unittest.TestCase): "fe80:0000:0000:0000:0204:61ff:fe9d:f156/129", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", "fe80:0000:0000:0000:0204:61ff:fe9d:f156", - ] - - ILLEGAL_SUBNET_MAPPINGS = [ - "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128" + "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128", ] VALID_SUBNET_MAPPINGS = [ @@ -2876,6 +2873,21 @@ class SubnetTest(unittest.TestCase): "192.168.0.1/32", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/0", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/128", + "1:2:3:4:5:6:7:8/0", + "1::/0", + "1:2:3:4:5:6:7::/0", + "1::8/0", + "1:2:3:4:5:6::8/0", + "::/0", + "::8/0", + "::2:3:4:5:6:7:8/0", + "fe80::7:8%eth0/0", + "fe80::7:8%1/0", + "::255.255.255.255/0", + "::ffff:255.255.255.255/0", + "::ffff:0:255.255.255.255/0", + "2001:db8:3:4::192.0.2.33/0", + "64:ff9b::192.0.2.33/0", ] def test_config_invalid_subnet_type_validation(self): @@ -2892,16 +2904,6 @@ class SubnetTest(unittest.TestCase): assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg - def test_config_illegal_subnet_type_validation(self): - for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: - with pytest.raises(ConfigurationError) as exc: - self.check_config(invalid_subnet) - if IS_WINDOWS_PLATFORM: - assert "An invalid argument was supplied" in exc.value.msg or \ - "illegal IP address string" in exc.value.msg - else: - assert "illegal IP address string" in exc.value.msg - def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: self.check_config(valid_subnet) From badd4d764a1ada326604f7bea3d806cb4eb3558e Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Mon, 13 Nov 2017 21:53:14 -0600 Subject: [PATCH 4/4] Refactor subnet cidr validator & add new test Signed-off-by: Drew Romanyk --- compose/config/validation.py | 23 ++++++----------------- tests/unit/config/config_test.py | 3 ++- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index f97069935..0fdcb37e7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -45,13 +45,11 @@ VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' -VALID_REGEX_IPV4_CIDR = r'^(\d|[1-2]\d|3[0-2])$' VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) -VALID_REGEX_IPV4_ADDR = "^{IPV4_ADDR}$".format(IPV4_ADDR=VALID_IPV4_ADDR) +VALID_REGEX_IPV4_CIDR = "^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' -VALID_REGEX_IPV6_CIDR = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' -VALID_REGEX_IPV6_ADDR = "".join(""" +VALID_REGEX_IPV6_CIDR = "".join(""" ^ ( (({IPV6_SEG}:){{7}}{IPV6_SEG})| @@ -67,6 +65,7 @@ VALID_REGEX_IPV6_ADDR = "".join(""" (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) ) +/(\d|[1-9]\d|1[0-1]\d|12[0-8]) $ """.format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) @@ -93,19 +92,9 @@ def format_expose(instance): @FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): - if '/' not in instance: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - - ip_address, cidr = instance.split('/') - - if re.match(VALID_REGEX_IPV4_ADDR, ip_address): - if not re.match(VALID_REGEX_IPV4_CIDR, cidr): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - elif re.match(VALID_REGEX_IPV6_ADDR, ip_address): - if not re.match(VALID_REGEX_IPV6_CIDR, cidr): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - else: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + if not re.match(VALID_REGEX_IPV4_CIDR, instance) and \ + not re.match(VALID_REGEX_IPV6_CIDR, instance): + raise ValidationError("should use the CIDR format") return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1cf783c77..32ccf1cec 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2866,6 +2866,7 @@ class SubnetTest(unittest.TestCase): "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", "fe80:0000:0000:0000:0204:61ff:fe9d:f156", "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128", + "192.168.0.1/31/31", ] VALID_SUBNET_MAPPINGS = [ @@ -2902,7 +2903,7 @@ class SubnetTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: self.check_config(invalid_subnet) - assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg + assert "should use the CIDR format" in exc.value.msg def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: