tools: enforce restriction on YAML negative key

The phase field must precede the type field for negative tests to have a consistent style and be able to parse easier. Related to the goal of https://github.com/tc39/test262/issues/1997 Added this check to the linting script and updated tests accordingly.
2022-01-06 13:01:51 +01:00 · 2022-01-06 13:01:51 +01:00 · 00f682e746
parent 9aedfd146d
commit 00f682e746
41 changed files with 66 additions and 40 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -159,10 +159,10 @@ single line comment syntax.
 #### negative
 `negative: [dictionary containing "phase" and "type" keys]`

-This means the test is expected to throw an error of the given type.  If no error is thrown, a test failure is reported.
+This means the test is expected to throw an error of the given type.  If no error is thrown, a test failure is reported. The **phase** field must precede the **type** field.

- **type** - If an error is thrown, it is implicitly converted to a string. In order for the test to pass, this value must match the name of the error constructor.
 - **phase** - Negative tests whose **phase** value is "parse" must produce the specified error prior to executing code. The value "resolution" indicates that the error is expected to result while performing ES2015 module resolution. The value "runtime" dictates that the error is expected to be produced as a result of executing the test code.
+- **type** - If an error is thrown, it is implicitly converted to a string. In order for the test to pass, this value must match the name of the error constructor.

 For best practices on how to use the negative key please see [Handling Errors and Negative Test Cases](#handling-errors-and-negative-test-cases), below.

--- a/test/language/expressions/object/getter-body-strict-inside.js
+++ b/test/language/expressions/object/getter-body-strict-inside.js
@ -7,8 +7,8 @@ description: >
    reserved word or a future reserved word is made inside a strict
    mode FunctionBody of a PropertyAssignment
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [noStrict]
 ---*/

--- a/test/language/expressions/object/getter-body-strict-outside.js
+++ b/test/language/expressions/object/getter-body-strict-outside.js
@ -6,8 +6,8 @@ description: >
    Strict Mode - SyntaxError is thrown when an assignment to a
    reserved word or a future reserved word is contained in strict code
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [onlyStrict]
 ---*/

--- a/test/language/expressions/object/setter-body-strict-inside.js
+++ b/test/language/expressions/object/setter-body-strict-inside.js
@ -7,8 +7,8 @@ description: >
    reserved word is made in  a strict FunctionBody of a
    PropertyAssignment
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [noStrict]
 ---*/

--- a/test/language/expressions/object/setter-body-strict-outside.js
+++ b/test/language/expressions/object/setter-body-strict-outside.js
@ -6,8 +6,8 @@ description: >
    Strict Mode - SyntaxError is thrown when an assignment to a
    reserved word is contained in strict code
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [onlyStrict]
 ---*/

--- a/test/language/expressions/object/setter-param-arguments-strict-inside.js
+++ b/test/language/expressions/object/setter-param-arguments-strict-inside.js
@ -7,8 +7,8 @@ description: >
    the Identifier in a PropertySetParameterList of a
    PropertyAssignment  if its FunctionBody is strict code
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [noStrict]
 ---*/

--- a/test/language/expressions/object/setter-param-arguments-strict-outside.js
+++ b/test/language/expressions/object/setter-param-arguments-strict-outside.js
@ -7,8 +7,8 @@ description: >
    Identifier in a PropertySetParameterList of a PropertyAssignment
    that is contained in strict code
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [onlyStrict]
 ---*/

--- a/test/language/expressions/object/setter-param-eval-strict-inside.js
+++ b/test/language/expressions/object/setter-param-eval-strict-inside.js
@ -7,8 +7,8 @@ description: >
    Identifier in a PropertySetParameterList of a PropertyAssignment
    if its FunctionBody is strict code
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [noStrict]
 ---*/

--- a/test/language/expressions/object/setter-param-eval-strict-outside.js
+++ b/test/language/expressions/object/setter-param-eval-strict-outside.js
@ -7,8 +7,8 @@ description: >
    Identifier in a PropertySetParameterList of a PropertyAssignment
    that is contained in strict code
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [onlyStrict]
 ---*/

--- a/test/language/expressions/optional-chaining/call-expression-super-no-base.js
+++ b/test/language/expressions/optional-chaining/call-expression-super-no-base.js
@ -10,8 +10,8 @@ info: |
      SuperCall OptionalChain
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-null-op-template-string-esi.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-null-op-template-string-esi.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-null-op-template-string.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-null-op-template-string.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-null-optchain-template-string-esi.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-null-optchain-template-string-esi.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-null-optchain-template-string.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-null-optchain-template-string.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-op-template-string-esi.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-op-template-string-esi.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-op-template-string.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-op-template-string.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-optchain-template-string-esi.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-optchain-template-string-esi.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/early-errors-tail-position-optchain-template-string.js
+++ b/test/language/expressions/optional-chaining/early-errors-tail-position-optchain-template-string.js
@ -13,8 +13,8 @@ info: |
  It is a Syntax Error if any code matches this production.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/static-semantics-simple-assignment.js
+++ b/test/language/expressions/optional-chaining/static-semantics-simple-assignment.js
@ -12,8 +12,8 @@ info: |
    Return false.
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/update-expression-postfix.js
+++ b/test/language/expressions/optional-chaining/update-expression-postfix.js
@ -12,8 +12,8 @@ info: |
    --UnaryExpression
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/optional-chaining/update-expression-prefix.js
+++ b/test/language/expressions/optional-chaining/update-expression-prefix.js
@ -12,8 +12,8 @@ info: |
    --UnaryExpression
 features: [optional-chaining]
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/expressions/property-accessors/non-identifier-name.js
+++ b/test/language/expressions/property-accessors/non-identifier-name.js
@ -11,8 +11,8 @@ description: >
      MemberExpression[?Yield, ?Await] . IdentifierName

 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/identifiers/vertical-tilde-continue-escaped.js
+++ b/test/language/identifiers/vertical-tilde-continue-escaped.js
@ -7,8 +7,8 @@ description: Test VERTICAL TILDE (U+2E2F) is not recognized as ID_Continue chara
 info: |
  VERTICAL TILDE is in General Category 'Lm' and [:Pattern_Syntax:].
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/identifiers/vertical-tilde-continue.js
+++ b/test/language/identifiers/vertical-tilde-continue.js
@ -7,8 +7,8 @@ description: Test VERTICAL TILDE (U+2E2F) is not recognized as ID_Continue chara
 info: |
  VERTICAL TILDE is in General Category 'Lm' and [:Pattern_Syntax:].
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/identifiers/vertical-tilde-start-escaped.js
+++ b/test/language/identifiers/vertical-tilde-start-escaped.js
@ -7,8 +7,8 @@ description: Test VERTICAL TILDE (U+2E2F) is not recognized as ID_Start characte
 info: |
  VERTICAL TILDE is in General Category 'Lm' and [:Pattern_Syntax:].
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/test/language/identifiers/vertical-tilde-start.js
+++ b/test/language/identifiers/vertical-tilde-start.js
@ -7,8 +7,8 @@ description: Test VERTICAL TILDE (U+2E2F) is not recognized as ID_Start characte
 info: |
  VERTICAL TILDE is in General Category 'Lm' and [:Pattern_Syntax:].
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/tools/lint/lib/checks/negative.py
+++ b/tools/lint/lib/checks/negative.py
@ -1,4 +1,5 @@
 import re
+import yaml
 from ..check import Check

 _THROW_STMT = re.compile(
@ -39,3 +40,13 @@ class CheckNegative(Check):
                    return 'Negative tests of type "early" must include a `throw` statement'
            elif not _THROW_STMT.search(source):
                return 'Negative tests of type "early" must include a $DONOTEVALUATE() call'
+
+        for event in meta.parsing_events:
+            if isinstance(event, yaml.ScalarEvent):
+                if event.value == 'type':
+                    line_type = event.start_mark.line
+                elif event.value == 'phase':
+                    line_phase = event.start_mark.line
+
+        if line_phase > line_type:
+            return '"phase" field must precede "type" field'
--- a/tools/lint/test/fixtures/test/negative_extra_fields.js
+++ b/tools/lint/test/fixtures/test/negative_extra_fields.js
@ -7,8 +7,8 @@ esid: sec-assignment-operators-static-semantics-early-errors
 description: Extraneous field in "negative" frontmatter
 negative:
  flags: strict
-  type: ReferenceError
  phase: runtime
+  type: ReferenceError
 ---*/

 x;
--- a/tools/lint/test/fixtures/test/negative_invalid_phase.js
+++ b/tools/lint/test/fixtures/test/negative_invalid_phase.js
@ -6,8 +6,8 @@ NEGATIVE
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: early
+  type: SyntaxError
 flags: [raw]
 ---*/

--- a/tools/lint/test/fixtures/test/negative_invalid_phase_type_order.js
+++ b/tools/lint/test/fixtures/test/negative_invalid_phase_type_order.js
@ -0,0 +1,15 @@
+NEGATIVE
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+description: Minimal test
+negative:
+  type: SyntaxError
+  phase: parse
+---*/
+
+$DONOTEVALUATE();
+
+!!!
--- a/tools/lint/test/fixtures/test/negative_no_raw_stmt.js
+++ b/tools/lint/test/fixtures/test/negative_no_raw_stmt.js
@ -5,8 +5,8 @@
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/tools/lint/test/fixtures/test/negative_no_raw_wrong_stmt.js
+++ b/tools/lint/test/fixtures/test/negative_no_raw_wrong_stmt.js
@ -6,8 +6,8 @@ NEGATIVE
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 throw "Test262: This statement should not be evaluated!";
--- a/tools/lint/test/fixtures/test/negative_parse_throw_bad_value.js
+++ b/tools/lint/test/fixtures/test/negative_parse_throw_bad_value.js
@ -6,8 +6,8 @@ NEGATIVE
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE(1);
--- a/tools/lint/test/fixtures/test/negative_parse_throw_missing.js
+++ b/tools/lint/test/fixtures/test/negative_parse_throw_missing.js
@ -6,8 +6,8 @@ NEGATIVE
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 !!!
--- a/tools/lint/test/fixtures/test/negative_raw_call.js
+++ b/tools/lint/test/fixtures/test/negative_raw_call.js
@ -5,8 +5,8 @@
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [raw]
 ---*/

--- a/tools/lint/test/fixtures/test/negative_raw_wrong_call.js
+++ b/tools/lint/test/fixtures/test/negative_raw_wrong_call.js
@ -6,8 +6,8 @@ NEGATIVE
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 flags: [raw]
 ---*/

--- a/tools/lint/test/fixtures/test/negative_resolution_throw_bad_value.js
+++ b/tools/lint/test/fixtures/test/negative_resolution_throw_bad_value.js
@ -7,8 +7,8 @@ esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 flags: [module]
 negative:
-  type: SyntaxError
  phase: resolution
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE(1);
--- a/tools/lint/test/fixtures/test/negative_resolution_throw_missing.js
+++ b/tools/lint/test/fixtures/test/negative_resolution_throw_missing.js
@ -7,8 +7,8 @@ esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 flags: [module]
 negative:
-  type: SyntaxError
  phase: resolution
+  type: SyntaxError
 ---*/

 import 'non-existent-module.js';
--- a/tools/lint/test/fixtures/test/negative_valid_parse.js
+++ b/tools/lint/test/fixtures/test/negative_valid_parse.js
@ -5,8 +5,8 @@
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: SyntaxError
  phase: parse
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/tools/lint/test/fixtures/test/negative_valid_resolution.js
+++ b/tools/lint/test/fixtures/test/negative_valid_resolution.js
@ -6,8 +6,8 @@ esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 flags: [module]
 negative:
-  type: SyntaxError
  phase: resolution
+  type: SyntaxError
 ---*/

 $DONOTEVALUATE();
--- a/tools/lint/test/fixtures/test/negative_valid_runtime.js
+++ b/tools/lint/test/fixtures/test/negative_valid_runtime.js
@ -5,8 +5,8 @@
 esid: sec-assignment-operators-static-semantics-early-errors
 description: Minimal test
 negative:
-  type: ReferenceError
  phase: runtime
+  type: ReferenceError
 ---*/

 x;