Lint test features tag (#1060)

A recent commit introduced a document that enumerated acceptable values
for the test "features" metadata tag. However, this list was incomplete,
and maintaining it placed extra burden on the project owners.

Restructure the document into a machine-readable format. Add entries for
all previously-omitted values. Add in-line documentation with
recommendations for maintenance of the file. Extend the project's
linting tool to validate tests according to the document's contents.

CONTRIBUTING.md

@@ -145,7 +145,7 @@ This tag is for boolean properties associated with the test.
 #### features
 **features**: [list]
 
-Some tests require the use of language features that are not directly described by the test file's location in the directory structure. These features should be formally listed here.
+Some tests require the use of language features that are not directly described by the test file's location in the directory structure. These features should be specified with this tag. See the `features.txt` file for a complete list of available values.
 
 ## Test Environment
 

FEATURES.md

@@ -1,16 +0,0 @@
-# Feature Flags
-
-Current, post-ES2015, flags used to identify new features:
-
-- `async-functions`: Async Functions
-- `async-iteration`: [Async Iteration and Generators](https://github.com/tc39/proposal-async-iteration)
-- `object-rest`: [Object rest/spread properties](https://github.com/tc39/proposal-object-rest-spread)
-- `object-spread`: [Object rest/spread properties](https://github.com/tc39/proposal-object-rest-spread)
-- `regexp-dotall`: [RegExp s (dotAll) flag](https://github.com/tc39/proposal-regexp-dotall-flag)
-- `regexp-lookbehind`: [RegExp lookBehind](https://github.com/tc39/proposal-regexp-lookbehind)
-- `regexp-named-groups`: [RegExp named groups capturing]()
-- `regexp-unicode-property-escapes`: [RegExp Unicode Property Escapes](https://github.com/tc39/proposal-regexp-unicode-property-escapes)
-- `SharedArrayBuffer`
-- `Symbol.asyncIterator`
-
-While it's mostly optional when already in a current released specification, it's highly recommended to reuse the features flags for any matching case in new tests.

features.txt

@@ -0,0 +1,99 @@
+# Proposed language features
+#
+# This project accepts tests for language proposals that have reached stage 3
+# in TC39's standardization process. Those tests should be annotated with a
+# dedicated feature flag so that consumers may more easily omit them as
+# necessary.
+#
+# https://github.com/tc39/process-document
+
+# Async Iteration and Generators
+# https://github.com/tc39/proposal-async-iteration
+async-iteration
+Symbol.asyncIterator
+
+# Object rest/spread properties
+# https://github.com/tc39/proposal-object-rest-spread
+object-rest
+object-spread
+
+# RegExp s (dotAll) flag
+# https://github.com/tc39/proposal-regexp-dotall-flag
+regexp-dotall
+
+# RegExp lookBehind
+# https://github.com/tc39/proposal-regexp-lookbehind
+regexp-lookbehind
+
+# RegExp named groups capturing
+# https://github.com/tc39/proposal-regexp-named-groups
+regexp-named-groups
+
+# RegExp Unicode Property Escapes
+# https://github.com/tc39/proposal-regexp-unicode-property-escapes
+regexp-unicode-property-escapes
+
+# Shared Memory and atomics
+# https://github.com/tc39/ecmascript_sharedmem
+SharedArrayBuffer
+
+# Standard language features
+#
+# Language features that have been included in a published version of the
+# ECMA-262 specification. These flags are largely maintained for historical
+# reasons, though their use for relatively new features (i.e. prior to
+# availability across major implementations) is appreciated.
+
+ArrayBuffer
+Array.prototype.values
+arrow-function
+async-functions
+caller
+class
+const
+DataView
+DataView.prototype.getFloat32
+DataView.prototype.getFloat64
+DataView.prototype.getInt16
+DataView.prototype.getInt32
+DataView.prototype.getInt8
+DataView.prototype.getUint16
+DataView.prototype.getUint32
+DataView.prototype.setUint8
+default-arg
+default-parameters
+destructuring-binding
+Float64Array
+generator
+generators
+Int8Array
+let
+Map
+new.target
+Proxy
+Reflect
+Reflect.construct
+Reflect.set
+Reflect.setPrototypeOf
+Set
+String#endsWith
+String#includes
+super
+Symbol
+Symbol.hasInstance
+Symbol.isConcatSpreadable
+Symbol.iterator
+Symbol.match
+Symbol.replace
+Symbol.search
+Symbol.species
+Symbol.split
+Symbol.toPrimitive
+Symbol.toStringTag
+Symbol.unscopables
+tail-call-optimization
+template
+TypedArray
+Uint8Array
+WeakMap
+WeakSet

tools/lint/lib/checks/features.py

@@ -0,0 +1,38 @@
+from ..check import Check
+
+_REQUIRED_FIELDS = set(['description'])
+_OPTIONAL_FIELDS = set([
+    'author', 'es5id', 'es6id', 'esid', 'features', 'flags', 'includes',
+    'info', 'negative', 'timeout'
+])
+_VALID_FIELDS = _REQUIRED_FIELDS | _OPTIONAL_FIELDS
+
+class CheckFeatures(Check):
+    '''Ensure tests specify only `features` from a list of valid values.'''
+    ID = 'FEATURES'
+
+    def __init__(self, filename):
+        with open(filename, 'r') as f:
+            self.valid_features = self._parse(f.read())
+
+    @staticmethod
+    def _parse(content):
+        features = []
+        for line in content.split():
+            if not line or line.startswith('#'):
+                continue
+            features.append(line)
+        return features
+
+    def run(self, name, meta, source):
+        if not meta or 'features' not in meta:
+            return
+
+        features = meta['features']
+
+        if len(features) == 0:
+            return 'If present, the `features` tag must have at least one member'
+
+        for feature in features:
+            if feature not in self.valid_features:
+                return 'Unrecognized feature: "%s"' % feature

tools/lint/lint.py

@@ -6,6 +6,7 @@ import argparse
 import sys
 
 from lib.collect_files import collect_files
+from lib.checks.features import CheckFeatures
 from lib.checks.frontmatter import CheckFrontmatter
 from lib.checks.license import CheckLicense
 from lib.eprint import eprint
@@ -20,7 +21,7 @@ parser.add_argument('path',
         nargs='+',
         help='file name or directory of files to lint')
 
-checks = [CheckFrontmatter(), CheckLicense()]
+checks = [CheckFrontmatter(), CheckFeatures('features.txt'), CheckLicense()]
 
 def lint(file_names):
     errors = dict()

tools/lint/test/fixtures/features_empty.js

@@ -0,0 +1,11 @@
+FEATURES
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+features: []
+---*/
+
+// empty

tools/lint/test/fixtures/features_unrecognized.js

@@ -0,0 +1,11 @@
+FEATURES
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+features: [not-a-valid-feature]
+---*/
+
+// empty

tools/lint/test/fixtures/features_valid.js

@@ -0,0 +1,10 @@
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+features: [async-functions, object-spread]
+---*/
+
+async function f({ ...a }) {}