Generation: support more complex globbing. Ref gh-1049

tools/generation/lib/case.py

@@ -8,6 +8,7 @@ from .util.find_comments import find_comments
 from .util.parse_yaml import parse_yaml
 
 regionStartPattern = re.compile(r'-\s+(\S+)')
+metaPattern = r'\/\*---\n([\s]*)((?:\s|\S)*)[\n\s*]---\*\/'
 
 class Case:
     def __init__(self, file_name, encoding):
@@ -21,13 +22,16 @@ class Case:
         region_name = None
         region_start = 0
         lines = source.split('\n')
+        matches = re.finditer(metaPattern, source, re.MULTILINE)
+
+        for matchNum, match in enumerate(matches, start=1):
+            meta = "{match}".format(matchNum = matchNum, start = match.start(), end = match.end(), match = match.group())
+            meta = parse_yaml(meta[2:-2])
+            if meta and not case['meta']:
+                case['meta'] = meta
+                break
 
         for comment in find_comments(source):
-            meta = parse_yaml(comment['source'])
-            if meta:
-                case['meta'] = meta
-                continue
-
             match = regionStartPattern.match(comment['source'])
             if match:
                 if region_name:

tools/generation/lib/expander.py

@@ -19,12 +19,15 @@ class Expander:
         file_names = []
 
         for expanded_directory in glob.glob(directory):
-            file_names.extend(
+            try:
-                map(
+                file_names.extend(
-                    lambda x: os.path.join(expanded_directory, x),
+                    map(
-                    filter(self.is_template_file, os.listdir(expanded_directory))
+                        lambda x: os.path.join(expanded_directory, x),
+                        filter(self.is_template_file, os.listdir(expanded_directory))
+                    )
                 )
-            )
+            except:
+                file_names.append(expanded_directory)
 
         self.templates[template_class] = [
             Template(x, encoding) for x in file_names
@@ -37,7 +40,7 @@ class Expander:
         return self.templates[template_class]
 
     def is_template_file(self, filename):
-      return re.match(templateFilenamePattern, filename)
+        return re.match(templateFilenamePattern, filename)
 
     def list_cases(self):
         for name in os.listdir(self.case_dir):

tools/generation/test/expected/glob/normal/a-nested/path2-glob-expr.js

@@ -0,0 +1,19 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob-expr.case
+// - tools/generation/test/fixtures/glob/a/normal2.template
+/*---
+description: foobar (Second template name)
+esid: sec-a-generic-id
+flags: [generated, a, b]
+includes: [foo.js, bar.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-Third value (Special characters like `≠` should be tolerated.)Second value-after
+
+/* Improperly-terminated comments should not break the tokenizer *
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/a-path1-glob-expr.js

@@ -0,0 +1,36 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob-expr.case
+// - tools/generation/test/fixtures/glob/a/normal.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b, c, d]
+includes: [foo.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-First value-between-Third value (Special characters like `≠` should be tolerated.)-after
+
+before*Second value*between*First value*after
+
+before/* " */Third value (Special characters like `≠` should be tolerated.)after
+
+// Special characters like `≠` should be tolerated.
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+Quote characters: " ' `
+"Quote characters: ' ' `"
+'Quote characters: " " `'
+`
+Quote characters: " ' '`
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-nested/path2-glob-expr.js

@@ -0,0 +1,19 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob-expr.case
+// - tools/generation/test/fixtures/glob/b/normal2.template
+/*---
+description: foobar (Second template name)
+esid: sec-a-generic-id
+flags: [generated, a, b]
+includes: [foo.js, bar.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-Third value (Special characters like `≠` should be tolerated.)Second value-after
+
+/* Improperly-terminated comments should not break the tokenizer *
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-path1-glob-expr.js

@@ -0,0 +1,36 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob-expr.case
+// - tools/generation/test/fixtures/glob/b/normal.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b, c, d]
+includes: [foo.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-First value-between-Third value (Special characters like `≠` should be tolerated.)-after
+
+before*Second value*between*First value*after
+
+before/* " */Third value (Special characters like `≠` should be tolerated.)after
+
+// Special characters like `≠` should be tolerated.
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+Quote characters: " ' `
+"Quote characters: ' ' `"
+'Quote characters: " " `'
+`
+Quote characters: " ' '`
+
+'This is "teardown" code.';

tools/generation/test/fixtures/glob-expr.case

@@ -0,0 +1,29 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+template: glob/**/normal*.template
+desc: foobar
+info: case info
+flags: [a, b]
+includes: [foo.js]
+---*/
+
+Because this test appears before any "region" delimiters, it should not appear
+in the generated files.
+
+// - first
+this is not a valid region delimiter
+
+/* *//- first
+this is also not a valid region delimiter
+
+//- first
+First value
+//- second
+Second value
+//- third
+Third value (Special characters like `≠` should be tolerated.)
+//- fourth
+Quote characters: " ' `
+//- teardown
+'This is "teardown" code.';