Implement `raw` flag

Some tests involving the directive prologue are invalidated by source
text transformations that insert executable code in the beginning of the
script. Implement a `raw` flag that allows these tests to opt-out of
this transformation. Update the relevant tests to use this flag (and
remove references to globals only available when code is injected).

Update the Python runner accordingly:

- Do not run tests marked as "raw" in strict mode
- Reject invalid test configurations

Update the browser runner accordingly:

- Do not modify the script body of tests marked as "raw"

CONTRIBUTING.md

@@ -117,6 +117,9 @@ This tag is for boolean properties associated with the test.
 - **`noStrict`** - only run the test in "sloppy" mode
 - **`module`** - interpret the source text as [module
   code](http://www.ecma-international.org/ecma-262/6.0/#sec-modules)
+- **`raw`** - execute the test without any modification (no helpers will be
+  available); necessary to test the behavior of directive prologue; implies
+  `noStrict`
 
 #### features
 **features**: [list]

test/language/directive-prologue/10.1.1-2gs.js

@@ -9,10 +9,10 @@ es5id: 10.1.1-2gs
 description: >
     Strict Mode - Use Strict Directive Prologue is ''use strict''
     which lost the last character ';'
-negative: NotEarlyError
+negative: SyntaxError
-flags: [noStrict]
+flags: [raw]
 ---*/
 
 "use strict"
-throw NotEarlyError;
+throw new Error("This code should not execute");
 var public = 1;

test/language/directive-prologue/10.1.1-5gs.js

@@ -9,10 +9,10 @@ es5id: 10.1.1-5gs
 description: >
     Strict Mode - Use Strict Directive Prologue is ''use strict';'
     which appears at the start of the code
-negative: NotEarlyError
+negative: SyntaxError
-flags: [noStrict]
+flags: [raw]
 ---*/
 
 "use strict";
-throw NotEarlyError;
+throw new Error("This code should not execute");
 var public = 1;

test/language/directive-prologue/10.1.1-8gs.js

@@ -9,8 +9,8 @@ es5id: 10.1.1-8gs
 description: >
     Strict Mode - Use Strict Directive Prologue is ''use strict';'
     which appears twice in the code
-negative: NotEarlyError
+negative: SyntaxError
-flags: [noStrict]
+flags: [raw]
 ---*/
 
 "use strict";

test/language/directive-prologue/14.1-4gs.js

@@ -10,9 +10,9 @@ description: >
     StrictMode - a Use Strict Directive followed by a strict mode
     violation
 negative: SyntaxError
-flags: [onlyStrict]
+flags: [raw]
 ---*/
 
 "use strict";
-throw NotEarlyError;
+throw new Error("This code should not execute");
 eval = 42;

test/language/directive-prologue/14.1-5gs.js

@@ -10,11 +10,11 @@ description: >
     StrictMode - a Use Strict Directive embedded in a directive
     prologue followed by a strict mode violation
 negative: SyntaxError
-flags: [onlyStrict]
+flags: [raw]
 ---*/
 
 "a";
 "use strict";
 "c";
-throw NotEarlyError;
+throw new Error("This code should not execute");
 eval = 42;

tools/packaging/test262.py

@@ -243,6 +243,8 @@ class TestCase(object):
     testRecord.pop("commentary", None)    # do not throw if missing
     self.testRecord = testRecord;
 
+    self.validate()
+
   def NegativeMatch(self, stderr):
     neg = re.compile(self.GetNegative())
     return re.search(neg, stderr)
@@ -269,7 +271,10 @@ class TestCase(object):
     return 'onlyStrict' in self.testRecord
 
   def IsNoStrict(self):
-    return 'noStrict' in self.testRecord
+    return 'noStrict' in self.testRecord or self.IsRaw()
+
+  def IsRaw(self):
+    return 'raw' in self.testRecord
 
   def IsAsyncTest(self):
     return '$DONE' in self.test
@@ -282,20 +287,10 @@ class TestCase(object):
   def GetAdditionalIncludes(self):
     return '\n'.join([self.suite.GetInclude(include) for include in self.GetIncludeList()])
 
-  def WrapTest(self, command):
+  def GetSource(self):
-    if "cscript" not in command:
+    if self.IsRaw():
-      return self.test
+        return self.test
 
-    return """
-try {
-""" + self.test + """
-} catch(e) {
-    $ERROR(e.message);
-}
-"""
-
-  def GetSource(self, command_template):
-    # "var testDescrip = " + str(self.testRecord) + ';\n\n' + \
     source = self.suite.GetInclude("sta.js") + \
         self.suite.GetInclude("cth.js") + \
         self.suite.GetInclude("assert.js")
@@ -307,7 +302,7 @@ try {
 
     source = source + \
         self.GetAdditionalIncludes() + \
-        self.WrapTest(command_template) + '\n'
+        self.test + '\n'
 
     if self.strict_mode:
       source = '"use strict";\nvar strict_mode = true;\n' + source
@@ -347,7 +342,7 @@ try {
     return (code, out, err)
 
   def RunTestIn(self, command_template, tmp):
-    tmp.Write(self.GetSource(command_template))
+    tmp.Write(self.GetSource())
     tmp.Close()
     command = self.InstantiateTemplate(command_template, {
       'path': tmp.name
@@ -364,8 +359,23 @@ try {
     return result
 
   def Print(self):
-    print self.GetSource("")
+    print self.GetSource()
 
+  def validate(self):
+    flags = self.testRecord.get("flags")
+
+    if not flags:
+        return
+
+    if 'raw' in flags:
+        if 'noStrict' in flags:
+            raise TypeError("The `raw` flag implies the `noStrict` flag")
+        elif 'onlyStrict' in flags:
+            raise TypeError(
+                "The `raw` flag is incompatible with the `onlyStrict` flag")
+        elif len(self.GetIncludeList()) > 0:
+            raise TypeError(
+                "The `raw` flag is incompatible with the `includes` tag")
 
 class ProgressIndicator(object):
 

website/scripts/sth.js

@@ -272,7 +272,8 @@ function BrowserRunner() {
 BrowserRunner.prototype.compileSource = function(test, code) {
     var flags = test.flags;
 
-    if (flags && flags.indexOf("onlyStrict") > -1) {
+    if (flags && flags.indexOf("raw") === -1 &&
+        flags.indexOf("onlyStrict") > -1) {
         code = "'use strict';\n" + code;
     }