tyler.durden
/
test262
mirror of https://github.com/tc39/test262.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding Support for Async Tests

This commit is contained in:
NikSurya 2014-07-15 13:47:59 -07:00
parent 9b669da66c
commit 9aa6b917fa
11 changed files with 370 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
console/harness/doneprintHandle.js Normal file
View File

@ -0,0 +1,10 @@
function __consolePrintHandle__(msg){
print(msg);
}
function $DONE(){
if(arguments.length === 0)
__consolePrintHandle__('Test262:AsyncTestComplete');
else
__consolePrintHandle__('Error: ' + arguments[0]);
}

1
console/harness/ed.js
View File

@ -8,6 +8,7 @@
if (this.window!==undefined) { //for console support
this.window.onerror = function(errorMsg, url, lineNumber) {
this.window.iframeError = errorMsg;
if(typeof $DONE === 'function') $DONE();
};
}

21
console/harness/gs.js
View File

@ -5,9 +5,10 @@
/// copyright and this notice and otherwise comply with the Use Terms.
//Global Scope Test Case Validator
function $DONE() {
//An exception is expected
if (testDescrip.negative !== undefined) {
//An exception is expected
if (testDescrip.negative !== undefined) {
//TODO - come up with a generic way of catching the error type
//from this.onerror
testDescrip.negative = testDescrip.negative === "NotEarlyError" ?
@ -44,10 +45,10 @@ if (testDescrip.negative !== undefined) {
'pass',
undefined);
}
}
}
//Exception was not expected to be thrown
else if (this.iframeError !== undefined) {
//Exception was not expected to be thrown
else if (this.iframeError !== undefined) {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
@ -55,15 +56,17 @@ else if (this.iframeError !== undefined) {
'fail',
Error('Unexpected exception, "' +
this.iframeError + '" was thrown.'));
}
}
else {
else {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'pass',
undefined);
}
}
testFinished();
//teardown
testFinished();
}

54
console/harness/sth.js
View File

@ -27,6 +27,8 @@ function BrowserRunner() {
errorDetectorFileContents,
simpleTestAPIContents,
globalScopeContents,
timerContents,
startTime,
harnessDir = "harness/";
$.ajax({async: false,
@ -44,6 +46,11 @@ function BrowserRunner() {
success: function(data){globalScopeContents = data;},
url:harnessDir+"gs.js"});
$.ajax({async: false,
dataType: "text",
success: function(data){timerContents = data;},
url:harnessDir+"timer.js"});
/* Called by the child window to notify that the test has
* finished. This function call is put in a separate script block
* at the end of the page so errors in the test script block
@ -70,6 +77,8 @@ function BrowserRunner() {
document.body.removeChild(iframe);
instance.onComplete(currentTest);
//update elapsed time
controller.testElapsedTime(new Date() - startTime);
}
/* Called from the child window after the test has run. */
@ -85,7 +94,7 @@ function BrowserRunner() {
/* Run the test. */
this.run = function (test, code) {
var start = new Date();
startTime = new Date();
//--Detect proper window.onerror support
if (instance.supportsWindowOnerror===undefined) {
@ -133,8 +142,6 @@ function BrowserRunner() {
}
currentTest.code = code;
iframe = document.createElement("iframe");
iframe.setAttribute("id", "runnerIframe");
//FireFox has a defect where it doesn't fire window.onerror for an iframe if the iframe
@ -188,11 +195,25 @@ function BrowserRunner() {
iwin.testDescrip = currentTest;
//Add an error handler capable of catching so-called early errors
//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>");
//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>")
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(errorDetectorFileContents);
idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
idoc.writeln("</script>");
//this is mainly applicable for consoles that do not have setTimeout support
//idoc.writeln("<script type='text/javascript' src='harness/timer.js' defer>" + "</script>");
if(setTimeout === undefined && /\$DONE()/.test(code)){
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(timerContents);
idoc.writeln("</script>");
}
//Run the code
idoc.writeln("<script type='text/javascript'>");
if (! instance.supportsWindowOnerror) {
@ -202,16 +223,21 @@ function BrowserRunner() {
}
idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
if(!/\$DONE()/.test(code))
//if the test is synchronous - call $DONE immediately
idoc.writeln("if(typeof $DONE === 'function') $DONE()");
else{
//in case the test does not call $DONE asynchronously then
//bailout after 1 min or given bailout time by calling $DONE
var asyncval = parseInt(test.timeout);
var testTimeout = asyncval !== asyncval ? 2000 : asyncval;
idoc.writeln("setTimeout(function() {$ERROR(\" Test Timed Out at " + testTimeout +"\" )} ," + testTimeout + ")");
}
idoc.writeln("</script>");
idoc.close();
var elapsed = new Date() - start;
return elapsed;
};
//--Helper functions-------------------------------------------------------
@ -559,7 +585,7 @@ function Controller() {
/* Executes when a test is ready to run. */
loader.onTestReady = function(testObj, testSrc) {
presenter.updateStatus("Running Test: " + testObj.id);
elapsed += runner.run(testObj, testSrc);
runner.run(testObj, testSrc);
};
/* Executes when there are no more tests to run. */
@ -629,6 +655,10 @@ function Controller() {
this.toggleSelection = function(index) {
loader.toggleSelection(index);
}
this.testElapsedTime = function(time){
elapsed += time;
}
}
var controller = new Controller();

22
console/harness/timer.js Normal file
View File

@ -0,0 +1,22 @@
//setTimeout is not available, hence this script was loaded
if(Promise === undefined && this.setTimeout === undefined){
if(/\$DONE()/.test(code))
$ERROR("Async test capability is not supported in your test environment");
}
if(Promise !== undefined && this.setTimeout === undefined)
(function(that){
that.setTimeout = function(callback, delay) {
var p = Promise.resolve();
var start = Date.now();
var end = start + delay;
function check(){
var timeLeft = end - Date.now();
if(timeLeft)
p.then(check);
else
callback();
}
p.then(check);
}
})(this);

10
test/harness/doneprintHandle.js Normal file
View File

@ -0,0 +1,10 @@
function __consolePrintHandle__(msg){
print(msg);
}
function $DONE(){
if(arguments.length === 0)
__consolePrintHandle__('Test262:AsyncTestComplete');
else
__consolePrintHandle__('Error: ' + arguments[0]);
}

1
test/harness/ed.js
View File

@ -8,6 +8,7 @@
if (this.window!==undefined) { //for console support
this.window.onerror = function(errorMsg, url, lineNumber) {
this.window.iframeError = errorMsg;
if(typeof $DONE === 'function') $DONE();
};
}

21
test/harness/gs.js
View File

@ -5,9 +5,10 @@
/// copyright and this notice and otherwise comply with the Use Terms.
//Global Scope Test Case Validator
function $DONE() {
//An exception is expected
if (testDescrip.negative !== undefined) {
//An exception is expected
if (testDescrip.negative !== undefined) {
//TODO - come up with a generic way of catching the error type
//from this.onerror
testDescrip.negative = testDescrip.negative === "NotEarlyError" ?
@ -44,10 +45,10 @@ if (testDescrip.negative !== undefined) {
'pass',
undefined);
}
}
}
//Exception was not expected to be thrown
else if (this.iframeError !== undefined) {
//Exception was not expected to be thrown
else if (this.iframeError !== undefined) {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
@ -55,15 +56,17 @@ else if (this.iframeError !== undefined) {
'fail',
Error('Unexpected exception, "' +
this.iframeError + '" was thrown.'));
}
}
else {
else {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'pass',
undefined);
}
}
testFinished();
//teardown
testFinished();
}

53
test/harness/sth.js
View File

@ -27,6 +27,8 @@ function BrowserRunner() {
errorDetectorFileContents,
simpleTestAPIContents,
globalScopeContents,
timerContents,
startTime,
harnessDir = "harness/";
$.ajax({async: false,
@ -44,6 +46,11 @@ function BrowserRunner() {
success: function(data){globalScopeContents = data;},
url:harnessDir+"gs.js"});
$.ajax({async: false,
dataType: "text",
success: function(data){timerContents = data;},
url:harnessDir+"timer.js"});
/* Called by the child window to notify that the test has
* finished. This function call is put in a separate script block
* at the end of the page so errors in the test script block
@ -70,6 +77,8 @@ function BrowserRunner() {
document.body.removeChild(iframe);
instance.onComplete(currentTest);
//update elapsed time
controller.testElapsedTime(new Date() - startTime);
}
/* Called from the child window after the test has run. */
@ -85,7 +94,7 @@ function BrowserRunner() {
/* Run the test. */
this.run = function (test, code) {
var start = new Date();
startTime = new Date();
//--Detect proper window.onerror support
if (instance.supportsWindowOnerror===undefined) {
@ -133,8 +142,6 @@ function BrowserRunner() {
}
currentTest.code = code;
iframe = document.createElement("iframe");
iframe.setAttribute("id", "runnerIframe");
//FireFox has a defect where it doesn't fire window.onerror for an iframe if the iframe
@ -188,11 +195,25 @@ function BrowserRunner() {
iwin.testDescrip = currentTest;
//Add an error handler capable of catching so-called early errors
//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>");
//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>")
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(errorDetectorFileContents);
idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
idoc.writeln("</script>");
//this is mainly applicable for consoles that do not have setTimeout support
//idoc.writeln("<script type='text/javascript' src='harness/timer.js' defer>" + "</script>");
if(setTimeout === undefined && /\$DONE()/.test(code)){
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(timerContents);
idoc.writeln("</script>");
}
//Run the code
idoc.writeln("<script type='text/javascript'>");
if (! instance.supportsWindowOnerror) {
@ -202,16 +223,20 @@ function BrowserRunner() {
}
idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
if(!/\$DONE()/.test(code))
//if the test is synchronous - call $DONE immediately
idoc.writeln("if(typeof $DONE === 'function') $DONE()");
else{
//in case the test does not call $DONE asynchronously then
//bailout after 1 min or given bailout time by calling $DONE
var asyncval = parseInt(test.timeout);
var testTimeout = asyncval !== asyncval ? 2000 : asyncval;
idoc.writeln("setTimeout(function() {$ERROR(\" Test Timed Out at " + testTimeout +"\" )} ," + testTimeout + ")");
}
idoc.writeln("</script>");
idoc.close();
var elapsed = new Date() - start;
return elapsed;
};
//--Helper functions-------------------------------------------------------
@ -559,7 +584,7 @@ function Controller() {
/* Executes when a test is ready to run. */
loader.onTestReady = function(testObj, testSrc) {
presenter.updateStatus("Running Test: " + testObj.id);
elapsed += runner.run(testObj, testSrc);
runner.run(testObj, testSrc);
};
/* Executes when there are no more tests to run. */
@ -629,6 +654,10 @@ function Controller() {
this.toggleSelection = function(index) {
loader.toggleSelection(index);
}
this.testElapsedTime = function(time){
elapsed += time;
}
}
var controller = new Controller();

22
test/harness/timer.js Normal file
View File

@ -0,0 +1,22 @@
//setTimeout is not available, hence this script was loaded
if(Promise === undefined && this.setTimeout === undefined){
if(/\$DONE()/.test(code))
$ERROR("Async test capability is not supported in your test environment");
}
if(Promise !== undefined && this.setTimeout === undefined)
(function(that){
that.setTimeout = function(callback, delay) {
var p = Promise.resolve();
var start = Date.now();
var end = start + delay;
function check(){
var timeLeft = end - Date.now();
if(timeLeft)
p.then(check);
else
callback();
}
p.then(check);
}
})(this);

21
tools/packaging/test262.py
View File

@ -73,6 +73,7 @@ def BuildOptions():
result.add_option("--junitname", help="Filename to save test results in JUnit XML format")
result.add_option("--loglevel", default="warning",
help="sets log level to debug, info, warning, error, or critical")
result.add_option("--print-handle", default="", help="Command to print from console")
return result
@ -196,13 +197,17 @@ class TestResult(object):
def HasFailed(self):
return self.exit_code != 0
def AsyncHasFailed(self):
return 'Test262:AsyncTestComplete' not in self.stdout
def HasUnexpectedOutcome(self):
if self.case.IsNegative():
if self.case.IsAsyncTest():
return self.AsyncHasFailed() or self.HasFailed()
elif self.case.IsNegative():
return not self.HasFailed()
else:
return self.HasFailed()
class TestCase(object):
def __init__(self, suite, name, full_path, strict_mode):
@ -242,6 +247,9 @@ class TestCase(object):
def IsNoStrict(self):
return 'noStrict' in self.testRecord
def IsAsyncTest(self):
return '$DONE' in self.test
def GetSource(self):
# "var testDescrip = " + str(self.testRecord) + ';\n\n' + \
source = self.suite.GetInclude("cth.js") + \
@ -249,6 +257,8 @@ class TestCase(object):
self.suite.GetInclude("ed.js") + \
self.suite.GetInclude("testBuiltInObject.js") + \
self.suite.GetInclude("testIntl.js") + \
self.suite.GetInclude("timer.js") + \
self.suite.GetInclude("doneprintHandle.js").replace('print', self.suite.print_handle) + \
self.test + '\n'
if self.strict_mode:
@ -332,15 +342,17 @@ def MakePlural(n):
class TestSuite(object):
def __init__(self, root, strict_only, non_strict_only, unmarked_default):
def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
# TODO: derive from packagerConfig.py
self.test_root = path.join(root, 'test', 'suite')
self.lib_root = path.join(root, 'test', 'harness')
self.strict_only = strict_only
self.non_strict_only = non_strict_only
self.unmarked_default = unmarked_default
self.print_handle = print_handle
self.include_cache = { }
def Validate(self):
if not path.exists(self.test_root):
ReportError("No test repository found")
@ -536,7 +548,8 @@ def Main():
test_suite = TestSuite(options.tests,
options.strict_only,
options.non_strict_only,
options.unmarked_default)
options.unmarked_default,
options.print_handle)
test_suite.Validate()
if options.loglevel == 'debug':
logging.basicConfig(level=logging.DEBUG)