Make it possible to express test case values as a sequence of code points

tools/generation/lib/case.py

@@ -7,20 +7,21 @@ import re
 from .util.find_comments import find_comments
 from .util.parse_yaml import parse_yaml
 
-regionStartPattern = re.compile(r'-\s+(\S+)')
+regionStartPattern = re.compile(r'-\s+(\S+)(?P<options>\s*.*)?')
 metaPattern = r'\/\*---\n([\s]*)((?:\s|\S)*)[\n\s*]---\*\/'
 
 class Case:
     def __init__(self, file_name, encoding):
-        self.attribs = dict(meta=None, regions=dict())
+        self.attribs = dict(meta=None, regions=dict(), region_options=dict())
 
         with codecs.open(file_name, 'r', encoding) as handle:
             self.attribs = self._parse(handle.read())
 
     def _parse(self, source):
-        case = dict(meta=None, regions=dict())
+        case = dict(meta=None, regions=dict(), region_options=dict())
         region_name = None
         region_start = 0
+        region_options = None
         lines = source.split('\n')
         search = re.search(metaPattern, source, re.DOTALL|re.MULTILINE)
 
@@ -39,6 +40,9 @@ class Case:
 
                 region_name = match.group(1)
                 region_start = comment['lineno']
+                region_options = match.group('options').split()
+                if region_options:
+                    case['region_options'][region_name] = set(region_options)
                 continue
 
         if region_name:

tools/generation/lib/template.py

@@ -103,8 +103,9 @@ class Template:
         lines = source.split('\n')
 
         for region in self.regions:
+            region_name = region['name']
             whitespace = indentPattern.match(lines[region['lineno']]).group(1)
-            value = context['regions'].get(region['name'], '')
+            value = context['regions'].get(region_name, '')
 
             str_char = region.get('in_string')
             if str_char:
@@ -112,8 +113,19 @@ class Template:
                 value = value.replace(str_char, safe_char)
                 value = value.replace('\n', '\\\n')
 
+            # TODO: document region_options and "codepoints" (`//- <region_name> codepoints`)
+            if "codepoints" in context['region_options'].get(region_name, set()):
+                str_from_cp = chr
+                try:
+                    # In Python 2, explicitly work on code points
+                    str_from_cp = unichr
+                except NameError: pass
+                value = "".join(str_from_cp(int(cp, 16)) for cp in value.split())
+            else:
+                value = indent(value, whitespace, True).lstrip()
+
             source = source[:region['firstchar']] + \
-                indent(value, whitespace, True).lstrip() + \
+                value + \
                 source[region['lastchar']:]
 
         setup = context['regions'].get('setup')