Adding Support for Async Tests

console/harness/doneprintHandle.js

@@ -0,0 +1,10 @@
+function __consolePrintHandle__(msg){
+	print(msg);
+}
+
+function $DONE(){
+	if(arguments.length === 0)
+		__consolePrintHandle__('Test262:AsyncTestComplete');
+	else
+		__consolePrintHandle__('Error: ' + arguments[0]);
+}

console/harness/ed.js

@@ -8,6 +8,7 @@
 if (this.window!==undefined) {  //for console support
     this.window.onerror = function(errorMsg, url, lineNumber) {
         this.window.iframeError = errorMsg;
+        if(typeof $DONE === 'function') $DONE();
     };
 }
 

console/harness/gs.js

@@ -5,9 +5,10 @@
 /// copyright and this notice and otherwise comply with the Use Terms.
 
 //Global Scope Test Case Validator
+function $DONE() {
 
-//An exception is expected
+    //An exception is expected
-if (testDescrip.negative !== undefined) {
+    if (testDescrip.negative !== undefined) {
         //TODO - come up with a generic way of catching the error type
         //from this.onerror
         testDescrip.negative = testDescrip.negative === "NotEarlyError" ?
@@ -44,10 +45,10 @@ if (testDescrip.negative !== undefined) {
                     'pass',
                     undefined);
         }
-}
+    }
 
-//Exception was not expected to be thrown
+    //Exception was not expected to be thrown
-else if (this.iframeError !== undefined) {
+    else if (this.iframeError !== undefined) {
         testRun(testDescrip.id,
                 testDescrip.path,
                 testDescrip.description,
@@ -55,15 +56,17 @@ else if (this.iframeError !== undefined) {
                 'fail',
                 Error('Unexpected exception, "' +
                       this.iframeError + '" was thrown.'));
-}
+    } 
 
-else {
+    else {
         testRun(testDescrip.id,
                 testDescrip.path,
                 testDescrip.description,
                 testDescrip.code,
                 'pass',
                 undefined);
-}
+    }
 
-testFinished();
+    //teardown
+    testFinished();
+}

console/harness/sth.js

@@ -27,6 +27,8 @@ function BrowserRunner() {
         errorDetectorFileContents,
         simpleTestAPIContents,
         globalScopeContents,
+        timerContents,
+        startTime,
         harnessDir = "harness/";
 
     $.ajax({async: false,
@@ -44,6 +46,11 @@ function BrowserRunner() {
             success: function(data){globalScopeContents = data;},
             url:harnessDir+"gs.js"});
 
+	$.ajax({async: false,
+		dataType: "text",
+		success: function(data){timerContents = data;},
+		url:harnessDir+"timer.js"});
+
     /* Called by the child window to notify that the test has
      * finished. This function call is put in a separate script block
      * at the end of the page so errors in the test script block
@@ -70,6 +77,8 @@ function BrowserRunner() {
         document.body.removeChild(iframe);
 
         instance.onComplete(currentTest);
+        //update elapsed time
+        controller.testElapsedTime(new Date() - startTime);
     }
 
     /* Called from the child window after the test has run. */
@@ -85,7 +94,7 @@ function BrowserRunner() {
 
     /* Run the test. */
     this.run = function (test, code) {
-        var start = new Date();
+        startTime = new Date();
 
         //--Detect proper window.onerror support
         if (instance.supportsWindowOnerror===undefined) {
@@ -133,8 +142,6 @@ function BrowserRunner() {
         }
         currentTest.code = code;
 
-
-
         iframe = document.createElement("iframe");
         iframe.setAttribute("id", "runnerIframe");
         //FireFox has a defect where it doesn't fire window.onerror for an iframe if the iframe
@@ -188,11 +195,25 @@ function BrowserRunner() {
         iwin.testDescrip = currentTest;
 
         //Add an error handler capable of catching so-called early errors
-        //idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>");
+		//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>")
         idoc.writeln("<script type='text/javascript'>");
         idoc.writeln(errorDetectorFileContents);
         idoc.writeln("</script>");
 
+        //Validate the results
+        //idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
+        idoc.writeln("<script type='text/javascript'>");
+        idoc.writeln(globalScopeContents);
+        idoc.writeln("</script>");
+
+        //this is mainly applicable for consoles that do not have setTimeout support
+		//idoc.writeln("<script type='text/javascript' src='harness/timer.js' defer>" + "</script>");
+        if(setTimeout === undefined && /\$DONE()/.test(code)){
+         idoc.writeln("<script type='text/javascript'>");
+         idoc.writeln(timerContents);
+         idoc.writeln("</script>");
+        }
+
         //Run the code
         idoc.writeln("<script type='text/javascript'>");
         if (! instance.supportsWindowOnerror) {
@@ -202,16 +223,21 @@ function BrowserRunner() {
         }
         idoc.writeln("</script>");
 		
-        //Validate the results
-        //idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
         idoc.writeln("<script type='text/javascript'>");
-        idoc.writeln(globalScopeContents);
+		
+
+        if(!/\$DONE()/.test(code))
+        //if the test is synchronous - call $DONE immediately
+            idoc.writeln("if(typeof $DONE === 'function') $DONE()");
+        else{
+        //in case the test does not call $DONE asynchronously then
+        //bailout after 1 min or given bailout time by calling $DONE
+            var asyncval = parseInt(test.timeout);
+            var testTimeout = asyncval !== asyncval ? 2000 : asyncval;
+			idoc.writeln("setTimeout(function() {$ERROR(\" Test Timed Out at " + testTimeout +"\" )} ," + testTimeout + ")");
+        }
         idoc.writeln("</script>");
         idoc.close();
-        
-        var elapsed = new Date() - start;
-
-        return elapsed;
     };
 
     //--Helper functions-------------------------------------------------------
@@ -559,7 +585,7 @@ function Controller() {
     /* Executes when a test is ready to run. */
     loader.onTestReady = function(testObj, testSrc) {
         presenter.updateStatus("Running Test: " + testObj.id);
-        elapsed += runner.run(testObj, testSrc);
+        runner.run(testObj, testSrc);
     };
 
     /* Executes when there are no more tests to run. */
@@ -629,6 +655,10 @@ function Controller() {
     this.toggleSelection = function(index) {
         loader.toggleSelection(index);
     }
+
+    this.testElapsedTime = function(time){
+        elapsed += time;
+    }
 }
 
 var controller = new Controller();

console/harness/timer.js

@@ -0,0 +1,22 @@
+//setTimeout is not available, hence this script was loaded
+if(Promise === undefined && this.setTimeout === undefined){
+    if(/\$DONE()/.test(code))
+        $ERROR("Async test capability is not supported in your test environment");
+}
+
+if(Promise !== undefined && this.setTimeout === undefined) 
+    (function(that){
+       that.setTimeout = function(callback, delay) {
+            var p = Promise.resolve();
+            var start = Date.now();
+            var end = start + delay;
+            function check(){
+                var timeLeft = end - Date.now();        
+                if(timeLeft)
+                    p.then(check);
+                else
+                    callback();
+            }        
+            p.then(check);
+        }
+    })(this);

test/harness/doneprintHandle.js

@@ -0,0 +1,10 @@
+function __consolePrintHandle__(msg){
+	print(msg);
+}
+
+function $DONE(){
+	if(arguments.length === 0)
+		__consolePrintHandle__('Test262:AsyncTestComplete');
+	else
+		__consolePrintHandle__('Error: ' + arguments[0]);
+}

test/harness/ed.js

@@ -8,6 +8,7 @@
 if (this.window!==undefined) {  //for console support
     this.window.onerror = function(errorMsg, url, lineNumber) {
         this.window.iframeError = errorMsg;
+        if(typeof $DONE === 'function') $DONE();
     };
 }
 

test/harness/gs.js

@@ -5,9 +5,10 @@
 /// copyright and this notice and otherwise comply with the Use Terms.
 
 //Global Scope Test Case Validator
+function $DONE() {
 
-//An exception is expected
+    //An exception is expected
-if (testDescrip.negative !== undefined) {
+    if (testDescrip.negative !== undefined) {
         //TODO - come up with a generic way of catching the error type
         //from this.onerror
         testDescrip.negative = testDescrip.negative === "NotEarlyError" ?
@@ -44,10 +45,10 @@ if (testDescrip.negative !== undefined) {
                     'pass',
                     undefined);
         }
-}
+    }
 
-//Exception was not expected to be thrown
+    //Exception was not expected to be thrown
-else if (this.iframeError !== undefined) {
+    else if (this.iframeError !== undefined) {
         testRun(testDescrip.id,
                 testDescrip.path,
                 testDescrip.description,
@@ -55,15 +56,17 @@ else if (this.iframeError !== undefined) {
                 'fail',
                 Error('Unexpected exception, "' +
                       this.iframeError + '" was thrown.'));
-}
+    } 
 
-else {
+    else {
         testRun(testDescrip.id,
                 testDescrip.path,
                 testDescrip.description,
                 testDescrip.code,
                 'pass',
                 undefined);
-}
+    }
 
-testFinished();
+    //teardown
+    testFinished();
+}

test/harness/sth.js

@@ -27,6 +27,8 @@ function BrowserRunner() {
         errorDetectorFileContents,
         simpleTestAPIContents,
         globalScopeContents,
+        timerContents,
+        startTime,
         harnessDir = "harness/";
 
     $.ajax({async: false,
@@ -44,6 +46,11 @@ function BrowserRunner() {
             success: function(data){globalScopeContents = data;},
             url:harnessDir+"gs.js"});
     
+	$.ajax({async: false,
+		dataType: "text",
+		success: function(data){timerContents = data;},
+		url:harnessDir+"timer.js"});
+
     /* Called by the child window to notify that the test has
      * finished. This function call is put in a separate script block
      * at the end of the page so errors in the test script block
@@ -70,6 +77,8 @@ function BrowserRunner() {
         document.body.removeChild(iframe);
 
         instance.onComplete(currentTest);
+        //update elapsed time
+        controller.testElapsedTime(new Date() - startTime);
     }
 
     /* Called from the child window after the test has run. */
@@ -85,7 +94,7 @@ function BrowserRunner() {
 
     /* Run the test. */
     this.run = function (test, code) {
-        var start = new Date();
+        startTime = new Date();
 
         //--Detect proper window.onerror support
         if (instance.supportsWindowOnerror===undefined) {
@@ -133,8 +142,6 @@ function BrowserRunner() {
         }
         currentTest.code = code;
 
-
-
         iframe = document.createElement("iframe");
         iframe.setAttribute("id", "runnerIframe");
         //FireFox has a defect where it doesn't fire window.onerror for an iframe if the iframe
@@ -188,11 +195,25 @@ function BrowserRunner() {
         iwin.testDescrip = currentTest;
 
         //Add an error handler capable of catching so-called early errors
-        //idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>");
+		//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>")
         idoc.writeln("<script type='text/javascript'>");
         idoc.writeln(errorDetectorFileContents);
         idoc.writeln("</script>");
 
+        //Validate the results
+        //idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
+        idoc.writeln("<script type='text/javascript'>");
+        idoc.writeln(globalScopeContents);
+        idoc.writeln("</script>");
+
+        //this is mainly applicable for consoles that do not have setTimeout support
+		//idoc.writeln("<script type='text/javascript' src='harness/timer.js' defer>" + "</script>");
+        if(setTimeout === undefined && /\$DONE()/.test(code)){
+         idoc.writeln("<script type='text/javascript'>");
+         idoc.writeln(timerContents);
+         idoc.writeln("</script>");
+        }
+
         //Run the code
         idoc.writeln("<script type='text/javascript'>");
         if (! instance.supportsWindowOnerror) {
@@ -202,16 +223,20 @@ function BrowserRunner() {
         }
         idoc.writeln("</script>");
 		
-        //Validate the results
-        //idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
         idoc.writeln("<script type='text/javascript'>");
-        idoc.writeln(globalScopeContents);
+		
+        if(!/\$DONE()/.test(code))
+        //if the test is synchronous - call $DONE immediately
+            idoc.writeln("if(typeof $DONE === 'function') $DONE()");
+        else{
+        //in case the test does not call $DONE asynchronously then
+        //bailout after 1 min or given bailout time by calling $DONE
+            var asyncval = parseInt(test.timeout);
+            var testTimeout = asyncval !== asyncval ? 2000 : asyncval;
+			idoc.writeln("setTimeout(function() {$ERROR(\" Test Timed Out at " + testTimeout +"\" )} ," + testTimeout + ")");
+        }
         idoc.writeln("</script>");
         idoc.close();
-        
-        var elapsed = new Date() - start;
-
-        return elapsed;
     };
 
     //--Helper functions-------------------------------------------------------
@@ -559,7 +584,7 @@ function Controller() {
     /* Executes when a test is ready to run. */
     loader.onTestReady = function(testObj, testSrc) {
         presenter.updateStatus("Running Test: " + testObj.id);
-        elapsed += runner.run(testObj, testSrc);
+        runner.run(testObj, testSrc);
     };
 
     /* Executes when there are no more tests to run. */
@@ -629,6 +654,10 @@ function Controller() {
     this.toggleSelection = function(index) {
         loader.toggleSelection(index);
     }
+
+    this.testElapsedTime = function(time){
+        elapsed += time;
+    }
 }
 
 var controller = new Controller();

test/harness/timer.js

@@ -0,0 +1,22 @@
+//setTimeout is not available, hence this script was loaded
+if(Promise === undefined && this.setTimeout === undefined){
+    if(/\$DONE()/.test(code))
+        $ERROR("Async test capability is not supported in your test environment");
+}
+
+if(Promise !== undefined && this.setTimeout === undefined) 
+    (function(that){
+       that.setTimeout = function(callback, delay) {
+            var p = Promise.resolve();
+            var start = Date.now();
+            var end = start + delay;
+            function check(){
+                var timeLeft = end - Date.now();        
+                if(timeLeft)
+                    p.then(check);
+                else
+                    callback();
+            }        
+            p.then(check);
+        }
+    })(this);

tools/packaging/test262.py

@@ -73,6 +73,7 @@ def BuildOptions():
   result.add_option("--junitname", help="Filename to save test results in JUnit XML format")
   result.add_option("--loglevel", default="warning",
                     help="sets log level to debug, info, warning, error, or critical") 
+  result.add_option("--print-handle", default="", help="Command to print from console")
   return result
 
 
@@ -196,13 +197,17 @@ class TestResult(object):
   def HasFailed(self):    
     return self.exit_code != 0
 
+  def AsyncHasFailed(self):   
+    return 'Test262:AsyncTestComplete' not in self.stdout
+
   def HasUnexpectedOutcome(self):
-    if self.case.IsNegative():
+    if self.case.IsAsyncTest():		
+	return self.AsyncHasFailed() or self.HasFailed()
+    elif self.case.IsNegative():      
        return not self.HasFailed()
     else:      
        return self.HasFailed()
 
-
 class TestCase(object):
 
   def __init__(self, suite, name, full_path, strict_mode):
@@ -242,6 +247,9 @@ class TestCase(object):
   def IsNoStrict(self):
     return 'noStrict' in self.testRecord
 
+  def IsAsyncTest(self):	
+	return '$DONE' in self.test
+
   def GetSource(self):
     # "var testDescrip = " + str(self.testRecord) + ';\n\n' + \
     source = self.suite.GetInclude("cth.js") + \
@@ -249,6 +257,8 @@ class TestCase(object):
         self.suite.GetInclude("ed.js") + \
         self.suite.GetInclude("testBuiltInObject.js") + \
         self.suite.GetInclude("testIntl.js") + \
+	self.suite.GetInclude("timer.js") + \
+	self.suite.GetInclude("doneprintHandle.js").replace('print', self.suite.print_handle) + \
         self.test + '\n'
 
     if self.strict_mode:
@@ -332,15 +342,17 @@ def MakePlural(n):
 
 class TestSuite(object):
 
-  def __init__(self, root, strict_only, non_strict_only, unmarked_default):
+  def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
     # TODO: derive from packagerConfig.py
     self.test_root = path.join(root, 'test', 'suite')
     self.lib_root = path.join(root, 'test', 'harness')
     self.strict_only = strict_only
     self.non_strict_only = non_strict_only
     self.unmarked_default = unmarked_default
+    self.print_handle = print_handle
     self.include_cache = { }
 	
+
   def Validate(self):
     if not path.exists(self.test_root):
       ReportError("No test repository found")
@@ -536,7 +548,8 @@ def Main():
   test_suite = TestSuite(options.tests, 
                          options.strict_only, 
                          options.non_strict_only,
-                         options.unmarked_default)
+                         options.unmarked_default,
+			 options.print_handle)
   test_suite.Validate()
   if options.loglevel == 'debug':
     logging.basicConfig(level=logging.DEBUG)