Add globbing support to test generator

tools/generation/lib/expander.py

@@ -1,7 +1,7 @@
 # Copyright (C) 2016 the V8 project authors. All rights reserved.
 # This code is governed by the BSD license found in the LICENSE file.
 
-import re, os
+import glob, os, re
 
 from .case import Case
 from .template import Template
@@ -16,10 +16,15 @@ class Expander:
 
     def _load_templates(self, template_class, encoding):
         directory = os.path.join(self.case_dir, template_class)
-        file_names = map(
-            lambda x: os.path.join(directory, x),
-            filter(self.is_template_file, os.listdir(directory))
-        )
+        file_names = []
+
+        for expanded_directory in glob.glob(directory):
+            file_names.extend(
+                map(
+                    lambda x: os.path.join(expanded_directory, x),
+                    filter(self.is_template_file, os.listdir(expanded_directory))
+                )
+            )
 
         self.templates[template_class] = [
             Template(x, encoding) for x in file_names

tools/generation/test/expected/glob/normal/a-features-glob.js

@@ -0,0 +1,14 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/a/features.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+features: [f1]
+flags: [generated, a, b]
+includes: [foo.js]
+info: |
+    case info
+---*/
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/a-info-multiline-folding-glob.js

@@ -0,0 +1,30 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/a/info-multiline-folding.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b]
+includes: [foo.js]
+info: |
+    This is an "info" field with an
+
+    empty line
+    Trailing white space in the template:
+    should be preserved:
+
+
+    case info
+---*/
+
+// Trailing white space in the test body:
+// should be preserved:
+
+
+
+
+
+
+
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/a-nested/path2-glob.js

@@ -0,0 +1,19 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/a/normal2.template
+/*---
+description: foobar (Second template name)
+esid: sec-a-generic-id
+flags: [generated, a, b]
+includes: [foo.js, bar.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-Third value (Special characters like `≠` should be tolerated.)Second value-after
+
+/* Improperly-terminated comments should not break the tokenizer *
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/a-no-info-glob.js

@@ -0,0 +1,15 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/a/no-info.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b]
+includes: [foo.js]
+info: |
+    case info
+---*/
+
+First value
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/a-path1-glob.js

@@ -0,0 +1,36 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/a/normal.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b, c, d]
+includes: [foo.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-First value-between-Third value (Special characters like `≠` should be tolerated.)-after
+
+before*Second value*between*First value*after
+
+before/* " */Third value (Special characters like `≠` should be tolerated.)after
+
+// Special characters like `≠` should be tolerated.
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+Quote characters: " ' `
+"Quote characters: ' ' `"
+'Quote characters: " " `'
+`
+Quote characters: " ' '`
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-features-glob.js

@@ -0,0 +1,14 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/b/features.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+features: [f1]
+flags: [generated, a, b]
+includes: [foo.js]
+info: |
+    case info
+---*/
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-info-multiline-folding-glob.js

@@ -0,0 +1,30 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/b/info-multiline-folding.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b]
+includes: [foo.js]
+info: |
+    This is an "info" field with an
+
+    empty line
+    Trailing white space in the template:
+    should be preserved:
+
+
+    case info
+---*/
+
+// Trailing white space in the test body:
+// should be preserved:
+
+
+
+
+
+
+
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-nested/path2-glob.js

@@ -0,0 +1,19 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/b/normal2.template
+/*---
+description: foobar (Second template name)
+esid: sec-a-generic-id
+flags: [generated, a, b]
+includes: [foo.js, bar.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-Third value (Special characters like `≠` should be tolerated.)Second value-after
+
+/* Improperly-terminated comments should not break the tokenizer *
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-no-info-glob.js

@@ -0,0 +1,15 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/b/no-info.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b]
+includes: [foo.js]
+info: |
+    case info
+---*/
+
+First value
+
+'This is "teardown" code.';

tools/generation/test/expected/glob/normal/b-path1-glob.js

@@ -0,0 +1,36 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/glob.case
+// - tools/generation/test/fixtures/glob/b/normal.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b, c, d]
+includes: [foo.js]
+info: |
+    template info
+
+    case info
+---*/
+
+before-First value-between-Third value (Special characters like `≠` should be tolerated.)-after
+
+before*Second value*between*First value*after
+
+before/* " */Third value (Special characters like `≠` should be tolerated.)after
+
+// Special characters like `≠` should be tolerated.
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+Quote characters: " ' `
+"Quote characters: ' ' `"
+'Quote characters: " " `'
+`
+Quote characters: " ' '`
+
+'This is "teardown" code.';

tools/generation/test/fixtures/glob.case

@@ -0,0 +1,29 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+template: glob/*
+desc: foobar
+info: case info
+flags: [a, b]
+includes: [foo.js]
+---*/
+
+Because this test appears before any "region" delimiters, it should not appear
+in the generated files.
+
+// - first
+this is not a valid region delimiter
+
+/* *//- first
+this is also not a valid region delimiter
+
+//- first
+First value
+//- second
+Second value
+//- third
+Third value (Special characters like `≠` should be tolerated.)
+//- fourth
+Quote characters: " ' `
+//- teardown
+'This is "teardown" code.';

tools/generation/test/fixtures/glob/a/features.template

@@ -0,0 +1,8 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/a-features-
+es6id: 1.2.3
+features: [f1]
+---*/

tools/generation/test/fixtures/glob/a/info-multiline-folding.template

@@ -0,0 +1,30 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/a-info-multiline-folding-
+es6id: 1.2.3
+info: |
+    This is an "info" field with an
+
+    empty line
+    Trailing white space in the template:
+    should be preserved:
+
+
+
+
+
+
+
+---*/
+
+// Trailing white space in the test body:
+// should be preserved:
+
+
+
+
+
+
+

tools/generation/test/fixtures/glob/a/no-info.template

@@ -0,0 +1,9 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/a-no-info-
+es6id: 1.2.3
+---*/
+
+/*{ first }*/

tools/generation/test/fixtures/glob/a/normal.template

@@ -0,0 +1,30 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/a-path1-
+es6id: 1.2.3
+info: template info
+flags: [c, d]
+---*/
+
+before-/*{ first }*/-between-/*{ third }*/-after
+
+before*/*{ second }*/*between*/*{ first }*/*after
+
+before/* " *//*{ third }*/after
+
+// Special characters like `≠` should be tolerated.
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+/*{ fourth }*/
+"/*{ fourth }*/"
+'/*{ fourth }*/'
+`
+/*{ fourth }*/`

tools/generation/test/fixtures/glob/a/normal2.template

@@ -0,0 +1,13 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: Second template name
+path: glob/normal/a-nested/path2-
+esid: sec-a-generic-id
+includes: [bar.js]
+info: template info
+---*/
+
+before-/*{ third }*//*{ second }*/-after
+
+/* Improperly-terminated comments should not break the tokenizer *

tools/generation/test/fixtures/glob/b/features.template

@@ -0,0 +1,8 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/b-features-
+es6id: 1.2.3
+features: [f1]
+---*/

tools/generation/test/fixtures/glob/b/info-multiline-folding.template

@@ -0,0 +1,30 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/b-info-multiline-folding-
+es6id: 1.2.3
+info: |
+    This is an "info" field with an
+
+    empty line
+    Trailing white space in the template:
+    should be preserved:
+
+
+
+
+
+
+
+---*/
+
+// Trailing white space in the test body:
+// should be preserved:
+
+
+
+
+
+
+

tools/generation/test/fixtures/glob/b/no-info.template

@@ -0,0 +1,9 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/b-no-info-
+es6id: 1.2.3
+---*/
+
+/*{ first }*/

tools/generation/test/fixtures/glob/b/normal.template

@@ -0,0 +1,30 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: glob/normal/b-path1-
+es6id: 1.2.3
+info: template info
+flags: [c, d]
+---*/
+
+before-/*{ first }*/-between-/*{ third }*/-after
+
+before*/*{ second }*/*between*/*{ first }*/*after
+
+before/* " *//*{ third }*/after
+
+// Special characters like `≠` should be tolerated.
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+/*{ fourth }*/
+"/*{ fourth }*/"
+'/*{ fourth }*/'
+`
+/*{ fourth }*/`

tools/generation/test/fixtures/glob/b/normal2.template

@@ -0,0 +1,13 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: Second template name
+path: glob/normal/b-nested/path2-
+esid: sec-a-generic-id
+includes: [bar.js]
+info: template info
+---*/
+
+before-/*{ third }*//*{ second }*/-after
+
+/* Improperly-terminated comments should not break the tokenizer *

tools/generation/test/run.py

@@ -49,6 +49,11 @@ class TestGeneration(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(OUT_DIR, ignore_errors=True)
 
+    def test_glob(self):
+        result = self.fixture('glob.case')
+        self.assertEqual(result['returncode'], 0)
+        self.compareTrees('glob')
+
     def test_normal(self):
         result = self.fixture('normal.case')
         self.assertEqual(result['returncode'], 0)