tyler.durden
/
test262
mirror of https://github.com/tc39/test262.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove old test format parser and simplify yaml frontmatter parsing

This commit is contained in:
André Bargull 2019-02-28 06:39:59 -08:00
parent 59b89a1c83
commit 41edfcebce
4 changed files with 144 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tools/lint/lib/checks/license.py
View File

@ -6,18 +6,16 @@ _MIN_YEAR = 2009
_MAX_YEAR = 2030
_LICENSE_PATTERN = re.compile(
r'\/\/ Copyright( \([cC]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
r'// Copyright( \([C]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
r'(' +
r'\/\/ (' +
r'This code is governed by the( BSD)? license found in the LICENSE file\.' +
r'|' +
r'See LICENSE for details' +
r')' +
r'// This code is governed by the( BSD)? license found in the LICENSE file\.' +
r'|' +
r'\/\/ Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
r'\/\/ found in the LICENSE file\.' +
r'// See LICENSE for details.' +
r'|' +
r'\/\/ See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
r'// Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
r'// found in the LICENSE file\.' +
r'|' +
r'// See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
r')', re.IGNORECASE)
class CheckLicense(Check):

182
tools/packaging/parseTestRecord.py
View File

@ -3,84 +3,40 @@
# Copyright 2011 by Google, Inc. All rights reserved.
# This code is governed by the BSD license found in the LICENSE file.
# TODO: resolve differences with common.py and unify into one file.
from __future__ import print_function
import logging
import optparse
import os
from os import path
import platform
import re
import subprocess
import sys
import tempfile
import time
import imp
# from TestCasePackagerConfig import *
# Matches trailing whitespace and any following blank lines.
_BLANK_LINES = r"([ \t]*[\r\n]{1,2})*"
headerPatternStr = r"(?:(?:\s*\/\/.*)?\s*\n)*"
captureCommentPatternStr = r"\/\*\*?((?:\s|\S)*?)\*\/\s*\n"
anyPatternStr = r"(?:\s|\S)*"
# Matches the YAML frontmatter block.
_YAML_PATTERN = re.compile(r"/\*---(.*)---\*/" + _BLANK_LINES, re.DOTALL)
headerPattern = re.compile("^" + headerPatternStr)
# Should match anything
testRecordPattern = re.compile(r"^(" + headerPatternStr +
r")(?:" + captureCommentPatternStr +
r")?(" + anyPatternStr +
r")$")
stars = re.compile(r"\s*\n\s*\*\s?")
atattrs = re.compile(r"\s*\n\s*\*\s*@")
yamlPattern = re.compile(r"---((?:\s|\S)*)---")
newlinePattern = re.compile(r"\n")
# Matches all known variants for the license block.
_LICENSE_PATTERN = re.compile(
r'// Copyright( \(C\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
r'(' +
r'// This code is governed by the( BSD)? license found in the LICENSE file\.' +
r'|' +
r'// See LICENSE for details\.' +
r'|' +
r'// Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
r'// found in the LICENSE file\.' +
r'|' +
r'// See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
r')[\r\n]{1,2}' + _BLANK_LINES, re.IGNORECASE)
yamlLoad = None
def stripStars(text):
return stars.sub('\n', text).strip()
def stripHeader(src):
header = headerPattern.match(src).group(0)
return src[len(header):]
def matchParts(src, name):
match = testRecordPattern.match(src)
if match == None:
raise Exception('unrecognized: ' + name)
return match
def hasYAML(text):
match = yamlPattern.match(text)
if match == None:
return False
return True
def oldAttrParser(testRecord, body, name):
propTexts = atattrs.split(body)
testRecord['commentary'] = stripStars(propTexts[0])
del propTexts[0]
for propText in propTexts:
propMatch = re.match(r"^\w+", propText)
if propMatch == None:
raise Exception('Malformed "@" attribute: ' + name)
propName = propMatch.group(0)
propVal = stripStars(propText[len(propName):])
if propName in testRecord:
raise Exception('duplicate: ' + propName)
testRecord[propName] = propVal;
def yamlAttrParser(testRecord, attrs, name):
match = yamlPattern.match(attrs)
body = match.group(1)
def yamlAttrParser(testRecord, attrs, name, onerror):
importYamlLoad()
parsed = yamlLoad(body)
if (parsed is None):
print("Failed to parse yaml in name %s"%(name))
parsed = yamlLoad(attrs)
if parsed is None:
onerror("Failed to parse yaml in name %s" % name)
return
for key in parsed:
@ -93,65 +49,61 @@ def yamlAttrParser(testRecord, attrs, name):
for flag in testRecord['flags']:
testRecord[flag] = ""
def findAttrs(src):
match = re.search(r'\/\*---(?:[\s]*)((?:[\s\S])*)(?:[\s]*)---\*\/', src, re.DOTALL)
if not match:
return (None, None)
return (match.group(0), match.group(1).strip())
def findLicense(src):
_LICENSE_PATTERN = re.compile(
r'\/\/ Copyright( \([cC]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
r'(' +
r'\/\/ (' +
r'This code is governed by the( BSD)? license found in the LICENSE file\.' +
r'|' +
r'See LICENSE for details' +
r')' +
r'|' +
r'\/\/ Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
r'\/\/ found in the LICENSE file\.' +
r'|' +
r'\/\/ See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
r')', re.IGNORECASE)
match = _LICENSE_PATTERN.search(src)
if not match:
return None
return match.group(0).strip()
return match.group(0)
def findAttrs(src):
match = _YAML_PATTERN.search(src)
if not match:
return (None, None)
return (match.group(0), match.group(1).strip())
def parseTestRecord(src, name, onerror = print):
# Find the license block.
header = findLicense(src)
# Find the YAML frontmatter.
(frontmatter, attrs) = findAttrs(src)
# YAML frontmatter is required for all tests.
if frontmatter is None:
onerror("Missing frontmatter: %s" % name)
# The license should be placed before the frontmatter and there shouldn't be
# be any extra content between the license and the frontmatter.
if header is not None and frontmatter is not None:
headerIdx = src.index(header)
frontmatterIdx = src.index(frontmatter)
if headerIdx > frontmatterIdx:
onerror("Unexpected license after frontmatter: %s" % name)
# Search for any extra test content, but ignore whitespace only or comment lines.
extra = src[headerIdx + len(header) : frontmatterIdx]
if extra and any(line.strip() and not line.lstrip().startswith("//") for line in extra.split("\n")):
onerror("Unexpected test content between license and frontmatter: %s" % name)
# Remove the license and YAML parts from the actual test content.
test = src
if frontmatter is not None:
test = test.replace(frontmatter, '')
if header is not None:
test = test.replace(header, '')
def parseTestRecord(src, name):
testRecord = {}
header = ""
test = ""
attrs = ""
try:
match = matchParts(src, name)
header = match.group(1).strip()
attrs = match.group(2)
test = match.group(3)
except:
# match = something else that works without copyright
header = findLicense(src)
[frontmatter, attrs] = findAttrs(src)
test = src
if frontmatter:
test = test.replace(frontmatter, '')
if header:
test = test.replace(header, '')
testRecord['header'] = header
testRecord['header'] = header.strip() if header else ''
testRecord['test'] = test
if attrs:
if hasYAML(attrs):
yamlAttrParser(testRecord, attrs, name)
else:
oldAttrParser(testRecord, attrs, name)
yamlAttrParser(testRecord, attrs, name, onerror)
# Report if the license block is missing in non-generated tests.
if header is None and "generated" not in testRecord:
onerror("No license found in: %s" % name)
return testRecord

19
tools/packaging/test/fixtures/test262-old-headers.js vendored
View File

@ -1,19 +0,0 @@
// Copyright 2009 the Sputnik authors. All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.
/**
* The production Block { } in strict code can't contain function
* declaration;
*
* @path bestPractice/Sbp_A1_T1.js
* @description Trying to declare function at the Block statement
* @onlyStrict
* @negative SyntaxError
* @bestPractice http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls
*/
"use strict";
{
function __func(){}
}

199
tools/packaging/test/test_parseTestRecord.py
View File

@ -3,121 +3,61 @@
# Copyright 2014 by Sam Mikes. All rights reserved.
# This code is governed by the BSD license found in the LICENSE file.
from __future__ import print_function
import unittest
import os
import yaml
from textwrap import dedent
# add parent dir to search path
import sys
sys.path.insert(0, "..")
# Temporarily add parent dir to search path to be able to load "parseTestRecord".
try:
import sys
sys.path.insert(0, "..")
from parseTestRecord import *
from parseTestRecord import *
finally:
del sys.path[0]
def slurpFile(name):
with open(name) as f:
contents = f.read()
return contents
class TestOldParsing(unittest.TestCase):
def dedent_strip(content):
return dedent(content).strip("\n")
def test_test(self):
self.assertTrue(True)
def test_overview(self):
name = 'fixtures/test262-old-headers.js'
contents = slurpFile(name)
record = parseTestRecord(contents, name)
self.assertEqual("""// Copyright 2009 the Sputnik authors. All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.""",
record['header'])
self.assertEqual("""The production Block { } in strict code can't contain function
declaration;""", record['commentary'])
self.assertEqual("bestPractice/Sbp_A1_T1.js", record['path'])
self.assertEqual("Trying to declare function at the Block statement",
record['description'])
self.assertEqual("", record['onlyStrict'])
self.assertEqual("SyntaxError", record['negative'])
self.assertEqual("http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls",
record['bestPractice'])
self.assertEqual(""""use strict";
{
function __func(){}
}
""", record['test'])
@unittest.expectedFailure
def test_nomatch(self):
with self.assertRaisesRegexp(Exception, "unrecognized"):
parseTestRecord("#!/usr/bin/env python", "random.py")
def test_duplicate(self):
with self.assertRaisesRegexp(Exception, "duplicate: foo"):
parseTestRecord("""
// Copyright
/**
* @foo bar
* @foo bar
*/
1;
"""
, "name")
def test_malformed(self):
with self.assertRaisesRegexp(Exception, 'Malformed "@" attribute: name'):
parseTestRecord("""
// Copyright
/**
* @ baz
* @foo bar
*/
1;
"""
, "name")
def test_stripStars(self):
self.assertEqual("", stripStars(""))
self.assertEqual("foo", stripStars("\n* foo"))
self.assertEqual("@foo bar", stripStars("\n* @foo bar"))
self.assertEqual("@foo bar", stripStars("\n *@foo bar"))
def dedent_lstrip(content):
return dedent(content).lstrip("\n")
def raiseExceptionOnError(message):
raise Exception(message)
class TestYAMLParsing(unittest.TestCase):
def test_test(self):
self.assertTrue(True)
def test_split(self):
def test_findAttrs(self):
name = 'fixtures/test262-yaml-headers.js'
contents = slurpFile(name)
self.assertTrue('---' in contents)
match = matchParts(contents, name)
self.assertEqual("""---
info: >
The production Block { } in strict code can't contain function
declaration;
description: Trying to declare function at the Block statement
negative: SyntaxError
bestPractice: "http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls"
flags: [onlyStrict]
---""", match.group(2))
(frontmatter, attrs) = findAttrs(contents)
self.assertIsNotNone(frontmatter)
self.assertIsNotNone(attrs)
self.assertEqual(dedent_strip(
"""
info: >
The production Block { } in strict code can't contain function
declaration;
description: Trying to declare function at the Block statement
negative: SyntaxError
bestPractice: "http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls"
flags: [onlyStrict]
"""),
attrs)
def test_yamlParse(self):
text = """
info: >
The production Block { } in strict code can't contain function
declaration;
description: Trying to declare function at the Block statement
negative: SyntaxError
bestPractice: "http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls"
flags: [onlyStrict]"""
name = 'fixtures/test262-yaml-headers.js'
contents = slurpFile(name)
(_, text) = findAttrs(contents)
parsed = yaml.load(text)
self.assertEqual("Trying to declare function at the Block statement",
@ -127,32 +67,31 @@ flags: [onlyStrict]"""
self.assertEqual(["onlyStrict"], parsed['flags'])
self.assertEqual("The production Block { } in strict code can't contain function declaration;\n", parsed['info'])
def test_hasYAML(self):
self.assertTrue(hasYAML("---\n some: yaml\n\n---"))
self.assertFalse(hasYAML("\n* Test description\n *\n * @foo bar\n* @noStrict\n"))
def test_fixturehasYAML(self):
name = 'fixtures/test262-yaml-headers.js'
contents = slurpFile(name)
self.assertTrue('---' in contents)
match = matchParts(contents, name)
self.assertTrue(hasYAML(match.group(2)))
def test_missingKeys(self):
result = {}
yamlAttrParser(result, """---
info: some info (note no flags or includes)
---""", "")
yamlAttrParser(
result,
dedent_strip(
"""
info: some info (note no flags or includes)
"""
),
"",
raiseExceptionOnError
)
self.assertEqual("some info (note no flags or includes)", result['commentary'])
def test_overview(self):
name = 'fixtures/test262-yaml-headers.js'
contents = slurpFile(name)
record = parseTestRecord(contents, name)
record = parseTestRecord(contents, name, raiseExceptionOnError)
self.assertEqual("""// Copyright 2009 the Sputnik authors. All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.""",
record['header'])
self.assertEqual(dedent_strip(
"""
// Copyright 2009 the Sputnik authors. All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.
"""),
record['header'])
self.assertEqual("The production Block { } in strict code can't contain function declaration;\n", record['commentary'])
self.assertEqual("Trying to declare function at the Block statement",
@ -163,21 +102,20 @@ flags: [onlyStrict]"""
self.assertEqual('"http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls"',
record['bestPractice'])
self.assertEqual(""""use strict";
{
function __func(){}
}
self.assertEqual(dedent_lstrip(
"""
"use strict";
{
function __func(){}
}
""", record['test'])
"""),
record['test'])
class TestYAML2Parsing(unittest.TestCase):
def test_test(self):
self.assertTrue(True)
def test_overview(self):
def test_overview_no_copyright(self):
name = 'fixtures/test262-yaml-headers-no-cr.js'
contents = slurpFile(name)
record = parseTestRecord(contents, name)
record = parseTestRecord(contents, name, print)
self.assertEqual('',
record['header'])
@ -191,12 +129,15 @@ class TestYAML2Parsing(unittest.TestCase):
self.assertEqual('"http://wiki.ecmascript.org/doku.php?id=conventions:no_non_standard_strict_decls"',
record['bestPractice'])
self.assertEqual(""""use strict";
{
function __func(){}
}
self.assertEqual(dedent_lstrip(
"""
"use strict";
{
function __func(){}
}
""", record['test'])
"""),
record['test'])
if __name__ == '__main__':