Introduce test generation tool

tools/generation/generator.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+from __future__ import print_function
+import argparse
+import os, sys
+
+from lib.expander import Expander
+from lib.test import Test
+
+def print_error(*values):
+    print('ERROR:', *values, file=sys.stderr)
+
+def find_cases(location):
+    # When a file is specified, return the file name and its containing
+    # directory
+    if os.path.isfile(location):
+        return location, [os.path.dirname(location)]
+
+    # When a directory is specified, if that directory contains a sub-directory
+    # names "default" interpret it as a "case directory"
+    if (os.path.isdir(os.path.join(location, 'default'))):
+        return None, [location]
+    else:
+        return None, map(
+            lambda x: os.path.join(args.cases, x), os.listdir(args.cases))
+
+def clean(args):
+    for (subdir, _, fileNames) in os.walk(args.directory):
+        for fileName in map(lambda x: os.path.join(subdir, x), fileNames):
+            test = Test(fileName)
+            test.load()
+            if test.is_generated():
+                print('Deleting file "' + fileName + '"...')
+                os.remove(fileName)
+
+def create(args):
+    caseFile, caseDirs = find_cases(args.cases)
+
+    for caseDir in caseDirs:
+        exp = Expander(caseDir)
+        for test in exp.expand('utf-8', caseFile):
+            if args.out:
+                try:
+                    test.load(args.out)
+
+                    if args.no_clobber:
+                        print_error(
+                            'Refusing to overwrite file: ' + test.file_name)
+                        exit(1)
+
+                    if not test.is_generated():
+                        print_error(
+                            'Refusing to overwrite non-generated file: ' +
+                            test.file_name)
+                        exit(1)
+                except IOError:
+                    pass
+
+                test.write(args.out, parents=args.parents)
+            else:
+                print(test.to_string())
+
+parser = argparse.ArgumentParser(description='Test262 test generator tool')
+subparsers = parser.add_subparsers()
+
+create_parser = subparsers.add_parser('create',
+    help='''Generate test material''')
+create_parser.add_argument('-o', '--out', help='''The directory to write the
+    compiled tests. If unspecified, tests will be written to standard out.''')
+create_parser.add_argument('-p', '--parents', action='store_true',
+    help='''Create non-existent directories as necessary.''')
+create_parser.add_argument('-n', '--no-clobber', action='store_true',
+    help='''Do not produce test if a corresponding file exists within this
+        directory.''')
+create_parser.add_argument('cases',
+    help='''Test cases to generate. May be a file or a directory.''')
+create_parser.set_defaults(func=create)
+
+clean_parser = subparsers.add_parser('clean',
+    help='''Remove previously-generated files''')
+clean_parser.add_argument('directory',
+    help='''Remove any generated tests from this directory''')
+clean_parser.set_defaults(func=clean)
+
+args = parser.parse_args()
+args.func(args)

tools/generation/lib/__init__.py

@@ -0,0 +1 @@
+pass

tools/generation/lib/case.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import re
+
+from util.find_comments import find_comments
+from util.parse_yaml import parse_yaml
+
+regionStartPattern = re.compile(r'-\s+(\S+)')
+
+class Case:
+    def __init__(self, file_name):
+        self.attribs = dict(meta=None, regions=dict())
+
+        with open(file_name) as handle:
+            self.attribs = self._parse(handle.read())
+
+    def _parse(self, source):
+        case = dict(meta=None, regions=dict())
+        region_name = None
+        region_start = 0
+        lines = source.split('\n')
+
+        for comment in find_comments(source):
+            meta = parse_yaml(comment['source'])
+            if meta:
+                case['meta'] = meta
+                continue
+
+            match = regionStartPattern.match(comment['source'])
+            if match:
+                if region_name:
+                    case['regions'][region_name] = \
+                        '\n'.join(lines[region_start:comment['lineno'] - 1])
+
+                region_name = match.group(1)
+                region_start = comment['lineno']
+                continue
+
+        if region_name:
+            case['regions'][region_name] = \
+                '\n'.join(lines[region_start:-1])
+
+        return case

tools/generation/lib/expander.py

@@ -0,0 +1,58 @@
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import re, os
+
+from case import Case
+from template import Template
+
+caseFilenamePattern = re.compile(r'^[^\.].*\.case$')
+templateFilenamePattern = re.compile(r'^[^\.].*\.template$')
+
+class Expander:
+    def __init__(self, case_dir):
+        self.templates = dict()
+        self.case_dir = case_dir
+
+    def _load_templates(self, template_class):
+        directory = os.path.join(self.case_dir, template_class)
+        file_names = map(
+            lambda x: os.path.join(directory, x),
+            filter(self.is_template_file, os.listdir(directory))
+        )
+
+        self.templates[template_class] = [Template(x) for x in file_names]
+
+    def _get_templates(self, template_class):
+        if not template_class in self.templates:
+            self._load_templates(template_class)
+
+        return self.templates[template_class]
+
+    def is_template_file(self, filename):
+      return re.match(templateFilenamePattern, filename)
+
+    def list_cases(self):
+        for name in os.listdir(self.case_dir):
+            full = os.path.join(self.case_dir, name)
+            if os.path.isfile(full) and caseFilenamePattern.match(name):
+                yield full
+
+    def expand(self, encoding, case_file = None):
+        if case_file:
+            case_files = [case_file]
+        else:
+            case_files = self.list_cases()
+
+        for case_file in case_files:
+            for test in self.expand_case(case_file, encoding):
+                yield test
+
+    def expand_case(self, file_name, encoding):
+        case = Case(file_name)
+
+        template_class = case.attribs['meta']['template']
+        templates = self.templates.get(template_class)
+
+        for template in self._get_templates(template_class):
+            yield template.expand(file_name, os.path.basename(file_name[:-5]), case.attribs, encoding)

tools/generation/lib/template.py

@@ -0,0 +1,160 @@
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import os, re
+import codecs, yaml
+
+from util.find_comments import find_comments
+from util.parse_yaml import parse_yaml
+from test import Test
+
+indentPattern = re.compile(r'^(\s*)')
+interpolatePattern = re.compile(r'\{\s*(\S+)\s*\}')
+
+def indent(text, prefix = '    '):
+    '''Prefix a block of text (as defined by the "line break" control
+    character) with some character sequence.'''
+
+    if isinstance(text, list):
+        lines = text
+    else:
+        lines = text.split('\n')
+
+    return prefix + ('\n' + prefix).join(lines)
+
+class Template:
+    def __init__(self, filename):
+        self.filename = filename
+
+        with open(filename) as template_file:
+            self.source = template_file.read()
+
+        self.attribs = dict()
+        self.regions = []
+
+        self._parse()
+
+    def _remove_comment(self, comment):
+        '''Create a region that is not intended to be referenced by any case,
+        ensuring that the comment is not emitted in the rendered file.'''
+        name = '__remove_comment_' + str(comment['firstchar']) + '__'
+
+        # When a removed comment ends the line, the following newline character
+        # should also be removed from the generated file.
+        lastchar = comment['lastchar']
+        if self.source[lastchar] == '\n':
+            comment['lastchar'] = comment['lastchar'] + 1
+
+        self.regions.insert(0, dict(name=name, **comment))
+
+    def _parse(self):
+        for comment in find_comments(self.source):
+            meta = parse_yaml(comment['source'])
+
+            # Do not emit the template's frontmatter in generated files
+            # (file-specific frontmatter is generated as part of the rendering
+            # process)
+            if meta:
+                self.attribs['meta'] = meta
+                self._remove_comment(comment)
+                continue
+
+            # Do not emit license information in generated files (recognized as
+            # comments preceeding the YAML frontmatter)
+            if not self.attribs.get('meta'):
+                self._remove_comment(comment)
+                continue
+
+            match = interpolatePattern.match(comment['source'])
+
+            if match == None:
+                continue
+
+            self.regions.insert(0, dict(name=match.group(1), **comment))
+
+    def expand_regions(self, source, context):
+        lines = source.split('\n')
+
+        for region in self.regions:
+            whitespace = indentPattern.match(lines[region['lineno']]).group(1)
+            value = context['regions'].get(region['name'], '')
+            source = source[:region['firstchar']] + \
+                indent(value, whitespace).lstrip() + \
+                source[region['lastchar']:]
+
+        setup = context['regions'].get('setup')
+
+        if setup:
+            source = setup + '\n' + source
+
+        teardown = context['regions'].get('teardown')
+
+        if teardown:
+            source += '\n' + teardown + '\n'
+
+        return source
+
+    def _frontmatter(self, case_filename, case_values):
+        description = case_values['meta']['desc'].strip() + \
+            ' (' + self.attribs['meta']['name'].strip() + ')'
+        lines = []
+
+        lines += [
+            '// This file was procedurally generated from the following sources:',
+            '// - ' + case_filename,
+            '// - ' + self.filename,
+            '/*---',
+            'description: ' + description,
+        ]
+
+        esid = self.attribs['meta'].get('esid')
+        if esid:
+            lines.append('esid: ' + esid)
+
+        es6id = self.attribs['meta'].get('es6id')
+        if es6id:
+            lines.append('es6id: ' + es6id)
+
+        features = []
+        features += case_values['meta'].get('features', [])
+        features += self.attribs['meta'].get('features', [])
+        if len(features):
+            lines += ['features: ' + yaml.dump(features)]
+
+        flags = ['generated']
+        flags += case_values['meta'].get('flags', [])
+        flags += self.attribs['meta'].get('flags', [])
+        lines += ['flags: ' + yaml.dump(flags).strip()]
+
+        includes = []
+        includes += case_values['meta'].get('includes', [])
+        includes += self.attribs['meta'].get('includes', [])
+        if len(includes):
+            lines += ['includes: ' + yaml.dump(includes).strip()]
+
+        if case_values['meta'].get('negative'):
+            lines += ['negative: ' + case_values['meta'].get('negative')]
+
+        info = []
+
+        if 'info' in self.attribs['meta']:
+            info.append(indent(self.attribs['meta']['info']))
+        if 'info' in case_values['meta']:
+            if len(info):
+                info.append('')
+            info.append(indent(case_values['meta']['info']))
+
+        if len(info):
+            lines.append('info: >')
+            lines += info
+
+        lines.append('---*/')
+
+        return '\n'.join(lines)
+
+    def expand(self, case_filename, case_name, case_values, encoding):
+        frontmatter = self._frontmatter(case_filename, case_values)
+        body = self.expand_regions(self.source, case_values)
+
+        return Test(self.attribs['meta']['path'] + case_name + '.js',
+            source=codecs.encode(frontmatter + '\n' + body, encoding))

tools/generation/lib/test.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import os, re
+
+from util.find_comments import find_comments
+from util.parse_yaml import parse_yaml
+
+class Test:
+    """Representation of a generated test. Specifies a file location which may
+    or may not exist."""
+    def __init__(self, file_name, source=None):
+        self.file_name = file_name
+        self.source = source
+        self.attribs = dict(meta=None)
+
+        if self.source:
+            self._parse()
+
+    def load(self, prefix = None):
+        location = os.path.join(prefix or '', self.file_name)
+        with open(location) as handle:
+            self.source = handle.read()
+        self._parse()
+
+    def _parse(self):
+        for comment in find_comments(self.source):
+            meta = parse_yaml(comment['source'])
+            if meta:
+                self.attribs['meta'] = meta
+                break
+
+    def is_generated(self):
+        if not self.attribs['meta']:
+            return False
+        flags = self.attribs['meta'].get('flags')
+
+        if not flags:
+            return False
+
+        return 'generated' in flags
+
+    def to_string(self):
+        return '\n'.join([
+            '/**',
+            ' * ----------------------------------------------------------------',
+            ' * ' + self.file_name,
+            ' * ----------------------------------------------------------------',
+            ' */',
+            self.source,
+            '\n'])
+
+    def write(self, prefix, parents=False):
+        location = os.path.join(prefix, self.file_name)
+        path = os.path.dirname(location)
+        if not os.path.exists(path):
+            if parents:
+                os.makedirs(path)
+            else:
+                raise Exception('Directory does not exist: ' + path)
+
+        with open(location, 'w') as handle:
+            handle.write(self.source)

tools/generation/lib/util/__init__.py

@@ -0,0 +1 @@
+pass

tools/generation/lib/util/find_comments.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+def find_comments(source):
+    '''Parse input string describing JavaScript source and yield dictionaries
+    describing the JavaScript comments in the order they appear in the source.
+
+    Each dictionary defines the following attributes:
+
+    - source: the source text of the comment
+    - firstchar: the zero-indexed position of the token that begins the comment
+    - lastchar: the zero-indexed position of the token that closes the comment
+    - lineno: the zero-indexed offset of the line on which the comment appears
+    '''
+    in_string = False
+    in_s_comment = False
+    in_m_comment = False
+    follows_escape = False
+    comment = ''
+    lineno = 0
+
+    for idx in xrange(len(source)):
+        if source[idx] == '\n':
+            lineno += 1
+
+        # Within comments and strings, any odd number of back-slashes begins an
+        # escape sequence.
+        if source[idx - 1] == '\\':
+            follows_escape = not follows_escape
+        else:
+            follows_escape = False
+
+        if in_s_comment:
+            if source[idx] == '\n':
+                in_s_comment = False
+                yield dict(
+                    source=comment[1:],
+                    firstchar=idx - len(comment) - 1,
+                    lastchar=idx,
+                    lineno=lineno)
+                continue
+        elif in_m_comment:
+            if source[idx - 1] == '*' and source[idx] == '/':
+                in_m_comment = False
+                yield dict(
+                    source=comment[1:-1],
+                    firstchar=idx - len(comment) - 1,
+                    lastchar=idx + 1,
+                    lineno=lineno)
+                continue
+        elif in_string:
+            if source[idx] == in_string and not follows_escape:
+                in_string = False
+            elif source[idx] == '\n' and in_string != '`' and not follows_escape:
+                in_string = False
+            continue
+
+        if in_m_comment or in_s_comment:
+            comment += source[idx]
+            continue
+
+        in_m_comment = source[idx] == '/' and source[idx + 1] == '*'
+        in_s_comment = source[idx] == '/' and source[idx + 1] == '/'
+
+        if in_m_comment or in_s_comment:
+            comment = ''
+        elif source[idx] == '\'' or source[idx] == '"' or source[idx] == '`':
+            in_string = source[idx]

tools/generation/lib/util/parse_yaml.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import yaml, re
+
+yamlPattern = re.compile(r'\---\n([\s]*)((?:\s|\S)*)[\n\s*]---',
+                         flags=re.DOTALL|re.MULTILINE)
+
+def parse_yaml(string):
+    match = yamlPattern.match(string)
+    if not match:
+        return False
+
+    unindented = re.sub('^' + match.group(1), '',
+        match.group(2), flags=re.MULTILINE)
+
+    return yaml.safe_load(unindented)

tools/generation/requirements.txt

@@ -0,0 +1 @@
+PyYAML==3.11

tools/generation/test/expected/normal/nested/path2-normal.js

@@ -0,0 +1,19 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/normal.case
+// - tools/generation/test/fixtures/normal/normal2.template
+/*---
+description: foobar (Second template name)
+esid: sec-a-generic-id
+flags: [generated, a, b]
+includes: [foo.js, bar.js]
+info: >
+    template info
+
+    case info
+---*/
+
+before-Third valueSecond value-after
+
+/* Improperly-terminated comments should not break the tokenizer *
+
+'This is "teardown" code.';

tools/generation/test/expected/normal/no-info-normal.js

@@ -0,0 +1,15 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/normal.case
+// - tools/generation/test/fixtures/normal/no-info.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b]
+includes: [foo.js]
+info: >
+    case info
+---*/
+
+First value
+
+'This is "teardown" code.';

tools/generation/test/expected/normal/path1-normal.js

@@ -0,0 +1,33 @@
+// This file was procedurally generated from the following sources:
+// - tools/generation/test/fixtures/normal.case
+// - tools/generation/test/fixtures/normal/normal.template
+/*---
+description: foobar (First template name)
+es6id: 1.2.3
+flags: [generated, a, b, c, d]
+includes: [foo.js]
+info: >
+    template info
+
+    case info
+---*/
+
+before-First value-between-Third value-after
+
+before*Second value*between*First value*after
+
+before/* " */Third valueafter
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+"/*{ first }*/"
+'/*{ first }*/'
+`
+/*{ first }*/`
+
+'This is "teardown" code.';

tools/generation/test/fixtures/normal.case

@@ -0,0 +1,27 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+template: normal
+desc: foobar
+info: case info
+flags: [a, b]
+includes: [foo.js]
+---*/
+
+Because this test appears before any "region" delimiters, it should not appear
+in the generated files.
+
+// - first
+this is not a valid region delimiter
+
+/* *//- first
+this is also not a valid region delimiter
+
+//- first
+First value
+//- second
+Second value
+//- third
+Third value
+//- teardown
+'This is "teardown" code.';

tools/generation/test/fixtures/normal/no-info.template

@@ -0,0 +1,9 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: normal/no-info-
+es6id: 1.2.3
+---*/
+
+/*{ first }*/

tools/generation/test/fixtures/normal/normal.template

@@ -0,0 +1,27 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: First template name
+path: normal/path1-
+es6id: 1.2.3
+info: template info
+flags: [c, d]
+---*/
+
+before-/*{ first }*/-between-/*{ third }*/-after
+
+before*/*{ second }*/*between*/*{ first }*/*after
+
+before/* " *//*{ third }*/after
+
+The following should not be expanded:
+
+/* */*{ first }*/
+/*
+*/*{ first }*/
+//*{ first }*/
+// /*{ first }*/
+"/*{ first }*/"
+'/*{ first }*/'
+`
+/*{ first }*/`

tools/generation/test/fixtures/normal/normal2.template

@@ -0,0 +1,13 @@
+// Copyright (C) 2016 the V8 project authors. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+name: Second template name
+path: normal/nested/path2-
+esid: sec-a-generic-id
+includes: [bar.js]
+info: template info
+---*/
+
+before-/*{ third }*//*{ second }*/-after
+
+/* Improperly-terminated comments should not break the tokenizer *

tools/generation/test/run.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Copyright (C) 2016 the V8 project authors. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import shutil, subprocess, sys, os, unittest
+
+testDir = os.path.dirname(os.path.relpath(__file__))
+OUT_DIR = os.path.join(testDir, 'out')
+EXPECTED_DIR = os.path.join(testDir, 'expected')
+ex = os.path.join(testDir, '..', 'generator.py')
+
+class TestGeneration(unittest.TestCase):
+    maxDiff = None
+
+    def fixture(self, name):
+        relpath = os.path.relpath(os.path.join(testDir, 'fixtures', name))
+        sp = subprocess.Popen(
+            [ex, 'create', '-o', OUT_DIR, '-p', relpath],
+            stdout=subprocess.PIPE)
+        stdout, stderr = sp.communicate()
+        return dict(stdout=stdout, stderr=stderr, returncode=sp.returncode)
+
+    def getFiles(self, path):
+        names = []
+        for root, _, fileNames in os.walk(path):
+            for fileName in filter(lambda x: x[0] != '.', fileNames):
+                names.append(os.path.join(root, fileName))
+        names.sort()
+        return names
+
+    def compareTrees(self, targetName):
+        expectedPath = os.path.join(EXPECTED_DIR, targetName)
+        actualPath = os.path.join(OUT_DIR, targetName)
+
+        expectedFiles = self.getFiles(expectedPath)
+        actualFiles = self.getFiles(actualPath)
+
+        self.assertListEqual(
+            map(lambda x: os.path.relpath(x, expectedPath), expectedFiles),
+            map(lambda x: os.path.relpath(x, actualPath), actualFiles))
+
+        for expectedFile, actualFile in zip(expectedFiles, actualFiles):
+            with open(expectedFile) as expectedHandle:
+                with open(actualFile) as actualHandle:
+                    self.assertMultiLineEqual(
+                        expectedHandle.read(),
+                        actualHandle.read())
+
+    def tearDown(self):
+        shutil.rmtree(OUT_DIR, ignore_errors=True)
+
+    def test_normal(self):
+        result = self.fixture('normal.case')
+        self.assertEqual(result['returncode'], 0)
+        self.compareTrees('normal')
+
+if __name__ == '__main__':
+    unittest.main()