tyler.durden/test262
1
0
Fork 0
mirror of https://github.com/tc39/test262.git synced 2025-07-01 11:14:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Adding Support for Async Tests

Browse Source
This commit is contained in:
NikSurya 2014-07-15 13:47:59 -07:00
parent 9b669da66c
commit 9aa6b917fa
11 changed files with 370 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
console/harness/doneprintHandle.js Normal file
View File

@ -0,0 +1,10 @@
function __consolePrintHandle__(msg){
print(msg);
}
function $DONE(){
if(arguments.length === 0)
__consolePrintHandle__('Test262:AsyncTestComplete');
else
__consolePrintHandle__('Error: ' + arguments[0]);
}

1
console/harness/ed.js
View File

@ -8,6 +8,7 @@
if (this.window!==undefined) { //for console support if (this.window!==undefined) { //for console support
this.window.onerror = function(errorMsg, url, lineNumber) { this.window.onerror = function(errorMsg, url, lineNumber) {
this.window.iframeError = errorMsg; this.window.iframeError = errorMsg;
if(typeof $DONE === 'function') $DONE();
}; };
} }

99
console/harness/gs.js
View File

@ -5,38 +5,60 @@
/// copyright and this notice and otherwise comply with the Use Terms. /// copyright and this notice and otherwise comply with the Use Terms.
//Global Scope Test Case Validator //Global Scope Test Case Validator
function $DONE() {
//An exception is expected //An exception is expected
if (testDescrip.negative !== undefined) { if (testDescrip.negative !== undefined) {
//TODO - come up with a generic way of catching the error type //TODO - come up with a generic way of catching the error type
//from this.onerror //from this.onerror
testDescrip.negative = testDescrip.negative === "NotEarlyError" ? testDescrip.negative = testDescrip.negative === "NotEarlyError" ?
testDescrip.negative : testDescrip.negative :
(testDescrip.negative === "^((?!NotEarlyError).)*$" ? (testDescrip.negative === "^((?!NotEarlyError).)*$" ?
testDescrip.negative : "."); testDescrip.negative : ".");
if (this.iframeError === undefined) { //no exception was thrown if (this.iframeError === undefined) { //no exception was thrown
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('No exception was thrown; expected an error "message"' +
' property matching the regular expression "' +
testDescrip.negative + '".'));
} else if (!(new RegExp(testDescrip.negative,
"i").test(this.iframeError))) {
//wrong type of exception thrown
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('Expected an exception with a "message"' +
' property matching the regular expression "' +
testDescrip.negative +
'" to be thrown; actual was "' +
this.iframeError + '".'));
} else {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'pass',
undefined);
}
}
//Exception was not expected to be thrown
else if (this.iframeError !== undefined) {
testRun(testDescrip.id, testRun(testDescrip.id,
testDescrip.path, testDescrip.path,
testDescrip.description, testDescrip.description,
testDescrip.code, testDescrip.code,
'fail', 'fail',
Error('No exception was thrown; expected an error "message"' + Error('Unexpected exception, "' +
' property matching the regular expression "' + this.iframeError + '" was thrown.'));
testDescrip.negative + '".')); }
} else if (!(new RegExp(testDescrip.negative,
"i").test(this.iframeError))) { else {
//wrong type of exception thrown
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('Expected an exception with a "message"' +
' property matching the regular expression "' +
testDescrip.negative +
'" to be thrown; actual was "' +
this.iframeError + '".'));
} else {
testRun(testDescrip.id, testRun(testDescrip.id,
testDescrip.path, testDescrip.path,
testDescrip.description, testDescrip.description,
@ -44,26 +66,7 @@ if (testDescrip.negative !== undefined) {
'pass', 'pass',
undefined); undefined);
} }
}
//Exception was not expected to be thrown //teardown
else if (this.iframeError !== undefined) { testFinished();
testRun(testDescrip.id, }
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('Unexpected exception, "' +
this.iframeError + '" was thrown.'));
}
else {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'pass',
undefined);
}
testFinished();

152
console/harness/sth.js
View File

@ -1,7 +1,7 @@
/// Copyright (c) 2012 Ecma International. All rights reserved. /// Copyright (c) 2012 Ecma International. All rights reserved.
/// Ecma International makes this code available under the terms and conditions set /// Ecma International makes this code available under the terms and conditions set
/// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the /// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the
/// "Use Terms"). Any redistribution of this code must retain the above /// "Use Terms"). Any redistribution of this code must retain the above
/// copyright and this notice and otherwise comply with the Use Terms. /// copyright and this notice and otherwise comply with the Use Terms.
// Do not cache any JSON files - see // Do not cache any JSON files - see
@ -27,6 +27,8 @@ function BrowserRunner() {
errorDetectorFileContents, errorDetectorFileContents,
simpleTestAPIContents, simpleTestAPIContents,
globalScopeContents, globalScopeContents,
timerContents,
startTime,
harnessDir = "harness/"; harnessDir = "harness/";
$.ajax({async: false, $.ajax({async: false,
@ -44,6 +46,11 @@ function BrowserRunner() {
success: function(data){globalScopeContents = data;}, success: function(data){globalScopeContents = data;},
url:harnessDir+"gs.js"}); url:harnessDir+"gs.js"});
$.ajax({async: false,
dataType: "text",
success: function(data){timerContents = data;},
url:harnessDir+"timer.js"});
/* Called by the child window to notify that the test has /* Called by the child window to notify that the test has
* finished. This function call is put in a separate script block * finished. This function call is put in a separate script block
* at the end of the page so errors in the test script block * at the end of the page so errors in the test script block
@ -70,6 +77,8 @@ function BrowserRunner() {
document.body.removeChild(iframe); document.body.removeChild(iframe);
instance.onComplete(currentTest); instance.onComplete(currentTest);
//update elapsed time
controller.testElapsedTime(new Date() - startTime);
} }
/* Called from the child window after the test has run. */ /* Called from the child window after the test has run. */
@ -85,8 +94,8 @@ function BrowserRunner() {
/* Run the test. */ /* Run the test. */
this.run = function (test, code) { this.run = function (test, code) {
var start = new Date(); startTime = new Date();
//--Detect proper window.onerror support //--Detect proper window.onerror support
if (instance.supportsWindowOnerror===undefined) { if (instance.supportsWindowOnerror===undefined) {
var iframePrereqs = document.createElement("iframe"); var iframePrereqs = document.createElement("iframe");
@ -96,25 +105,25 @@ function BrowserRunner() {
} }
document.body.appendChild(iframePrereqs); document.body.appendChild(iframePrereqs);
var iwinPrereqs = iframePrereqs.contentWindow; var iwinPrereqs = iframePrereqs.contentWindow;
var idocPrereqs = iwinPrereqs.document; var idocPrereqs = iwinPrereqs.document;
idocPrereqs.open(); idocPrereqs.open();
iwinPrereqs.failCount = 0; iwinPrereqs.failCount = 0;
var stuff = [ var stuff = [
"window.onerror = function(a, b, c) { this.failCount++; }", "window.onerror = function(a, b, c) { this.failCount++; }",
"va xyz =", "va xyz =",
"throw Error();" "throw Error();"
]; ];
for(var i in stuff) { for(var i in stuff) {
idocPrereqs.writeln("<script type='text/javascript'>"); idocPrereqs.writeln("<script type='text/javascript'>");
idocPrereqs.writeln(stuff[i]); idocPrereqs.writeln(stuff[i]);
idocPrereqs.writeln("</script>"); idocPrereqs.writeln("</script>");
} }
idocPrereqs.close(); idocPrereqs.close();
//TODO - 500ms *should* be a sufficient delay //TODO - 500ms *should* be a sufficient delay
setTimeout(function() { setTimeout(function() {
instance.supportsWindowOnerror = iwinPrereqs.failCount === 2; instance.supportsWindowOnerror = iwinPrereqs.failCount === 2;
@ -124,7 +133,7 @@ function BrowserRunner() {
}, 500); }, 500);
return 0; // initial config, ignore this timing. return 0; // initial config, ignore this timing.
} }
currentTest = {}; currentTest = {};
for (var tempIndex in test) { for (var tempIndex in test) {
if (test.hasOwnProperty(tempIndex)) { if (test.hasOwnProperty(tempIndex)) {
@ -132,8 +141,6 @@ function BrowserRunner() {
} }
} }
currentTest.code = code; currentTest.code = code;
iframe = document.createElement("iframe"); iframe = document.createElement("iframe");
iframe.setAttribute("id", "runnerIframe"); iframe.setAttribute("id", "runnerIframe");
@ -188,11 +195,25 @@ function BrowserRunner() {
iwin.testDescrip = currentTest; iwin.testDescrip = currentTest;
//Add an error handler capable of catching so-called early errors //Add an error handler capable of catching so-called early errors
//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>"); //idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>")
idoc.writeln("<script type='text/javascript'>"); idoc.writeln("<script type='text/javascript'>");
idoc.writeln(errorDetectorFileContents); idoc.writeln(errorDetectorFileContents);
idoc.writeln("</script>"); idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
idoc.writeln("</script>");
//this is mainly applicable for consoles that do not have setTimeout support
//idoc.writeln("<script type='text/javascript' src='harness/timer.js' defer>" + "</script>");
if(setTimeout === undefined && /\$DONE()/.test(code)){
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(timerContents);
idoc.writeln("</script>");
}
//Run the code //Run the code
idoc.writeln("<script type='text/javascript'>"); idoc.writeln("<script type='text/javascript'>");
if (! instance.supportsWindowOnerror) { if (! instance.supportsWindowOnerror) {
@ -201,17 +222,22 @@ function BrowserRunner() {
idoc.writeln(code); idoc.writeln(code);
} }
idoc.writeln("</script>"); idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>"); idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
if(!/\$DONE()/.test(code))
//if the test is synchronous - call $DONE immediately
idoc.writeln("if(typeof $DONE === 'function') $DONE()");
else{
//in case the test does not call $DONE asynchronously then
//bailout after 1 min or given bailout time by calling $DONE
var asyncval = parseInt(test.timeout);
var testTimeout = asyncval !== asyncval ? 2000 : asyncval;
idoc.writeln("setTimeout(function() {$ERROR(\" Test Timed Out at " + testTimeout +"\" )} ," + testTimeout + ")");
}
idoc.writeln("</script>"); idoc.writeln("</script>");
idoc.close(); idoc.close();
var elapsed = new Date() - start;
return elapsed;
}; };
//--Helper functions------------------------------------------------------- //--Helper functions-------------------------------------------------------
@ -264,10 +290,10 @@ function TestLoader() {
else { else {
presenter.updateStatus("Loading file: " + testGroups[testGroupIndex].path); presenter.updateStatus("Loading file: " + testGroups[testGroupIndex].path);
testGroups[testGroupIndex].onLoaded = getNextXML; testGroups[testGroupIndex].onLoaded = getNextXML;
} }
} }
/* Get the test list xml */ /* Get the test list xml */
function loadTestXML() { function loadTestXML() {
var testsListLoader = new XMLHttpRequest(); var testsListLoader = new XMLHttpRequest();
@ -288,7 +314,7 @@ function TestLoader() {
onLoaded: function(){} onLoaded: function(){}
}; };
presenter.setTestWaiting(i, testSuite[i]); presenter.setTestWaiting(i, testSuite[i]);
var tr = $('#chapterSelector table tr').filter(':nth-child(' + (i+1) + ')'); var tr = $('#chapterSelector table tr').filter(':nth-child(' + (i+1) + ')');
tr.find('img').filter('[alt="Run"]').bind('click', {index:i}, function(event){ tr.find('img').filter('[alt="Run"]').bind('click', {index:i}, function(event){
controller.runIndividualTest(event.data.index); controller.runIndividualTest(event.data.index);
@ -298,25 +324,25 @@ function TestLoader() {
getFile(); getFile();
}}); }});
} }
/* Get the test file. Handles all the logic of figuring out the next file to load. */ /* Get the test file. Handles all the logic of figuring out the next file to load. */
function getFile(index) { function getFile(index) {
index = (arguments.length == 0) ? -1 : index; index = (arguments.length == 0) ? -1 : index;
// Look for selected waiting chapters (priority because you'll probably want to run them soon) // Look for selected waiting chapters (priority because you'll probably want to run them soon)
for(var i = 0; index == -1 && i < testGroups.length; i++) { for(var i = 0; index == -1 && i < testGroups.length; i++) {
if(testGroups[i].status == 'waiting' && testGroups[i].selected) { if(testGroups[i].status == 'waiting' && testGroups[i].selected) {
index = i; index = i;
} }
} }
// Look for just chapters waiting to be loaded. // Look for just chapters waiting to be loaded.
for(var i = 0; index == -1 && i < testGroups.length; i++) { for(var i = 0; index == -1 && i < testGroups.length; i++) {
if(testGroups[i].status == 'waiting') { if(testGroups[i].status == 'waiting') {
index = i; index = i;
} }
} }
if(index == -1) { if(index == -1) {
// Still -1? No more tests are waiting to be loaded then. // Still -1? No more tests are waiting to be loaded then.
if(controller.state == 'loading') { if(controller.state == 'loading') {
@ -324,13 +350,13 @@ function TestLoader() {
} }
return; return;
} }
presenter.setTestLoading(index, testGroups[index].path); presenter.setTestLoading(index, testGroups[index].path);
// the only other status that should be set when we get here is 'priorityloading' // the only other status that should be set when we get here is 'priorityloading'
if(testGroups[index].status == 'waiting') { if(testGroups[index].status == 'waiting') {
testGroups[index].status = 'loading'; testGroups[index].status = 'loading';
} }
loader.onTestStartLoading(index, testGroups[index].path); loader.onTestStartLoading(index, testGroups[index].path);
// Create the AJAX call to grab the file. // Create the AJAX call to grab the file.
$.ajax({ $.ajax({
@ -344,11 +370,11 @@ function TestLoader() {
}, },
error: function(xhr, textStatus, errorThrown) { error: function(xhr, textStatus, errorThrown) {
// TODO: Catch this error and update UI accordingly. Unlikely to happen, but errors would be 404 or 5-- errors. // TODO: Catch this error and update UI accordingly. Unlikely to happen, but errors would be 404 or 5-- errors.
} }
}); });
} }
/* Executes when a test file finishes loading. */ /* Executes when a test file finishes loading. */
function onTestLoaded(index, name, numTests) { function onTestLoaded(index, name, numTests) {
presenter.setTestLoaded(index, name, numTests); presenter.setTestLoaded(index, name, numTests);
@ -357,7 +383,7 @@ function TestLoader() {
loader.runningTests += numTests; loader.runningTests += numTests;
loader.onInitialized( loader.runningTests ); loader.onInitialized( loader.runningTests );
} }
// The loading status is only assigned when downloading files in sequence, otherwise it // The loading status is only assigned when downloading files in sequence, otherwise it
// gets the status of priorityloading. When loading out of order, we only want to download // gets the status of priorityloading. When loading out of order, we only want to download
// the single file, so we'll only tell it to get the next file when we see a status of // the single file, so we'll only tell it to get the next file when we see a status of
@ -371,10 +397,10 @@ function TestLoader() {
testGroups[index].status = 'loaded'; testGroups[index].status = 'loaded';
loader.setChapter(index); loader.setChapter(index);
} }
testGroups[index].onLoaded(); testGroups[index].onLoaded();
}; };
function getIdFromPath (path) { function getIdFromPath (path) {
//path is of the form "a/b/c.js" //path is of the form "a/b/c.js"
@ -420,7 +446,7 @@ function TestLoader() {
return; return;
} }
} }
// And if // And if
else { else {
// We don't have tests left in this test group, so move on // We don't have tests left in this test group, so move on
// to the next. // to the next.
@ -428,7 +454,7 @@ function TestLoader() {
} }
getNextXML(); getNextXML();
} }
// //
else { else {
// We're done. // We're done.
loader.onTestsExhausted(); loader.onTestsExhausted();
@ -446,18 +472,18 @@ function TestLoader() {
currentTestIndex = 0; currentTestIndex = 0;
testGroupIndex = 0; testGroupIndex = 0;
}; };
/* Begin downloading test files. */ /* Begin downloading test files. */
this.startLoadingTests = function() { this.startLoadingTests = function() {
loadTestXML(); loadTestXML();
}; };
/* Prepare for testing a single chapter. */ /* Prepare for testing a single chapter. */
this.setChapter = function(index) { this.setChapter = function(index) {
currentTestIndex = 0; currentTestIndex = 0;
testGroupIndex = index; testGroupIndex = index;
mode = "one"; mode = "one";
if(testGroups[index].status == 'loaded') { if(testGroups[index].status == 'loaded') {
loader.onInitialized(testGroups[index].tests.length); loader.onInitialized(testGroups[index].tests.length);
} }
@ -467,7 +493,7 @@ function TestLoader() {
loader.onInitialized(0); loader.onInitialized(0);
} }
}; };
/* Prepare for testing multiple chapters. Returns true if at least one chapter was selected. */ /* Prepare for testing multiple chapters. Returns true if at least one chapter was selected. */
this.setMultiple = function() { this.setMultiple = function() {
// Find the index of the first selection // Find the index of the first selection
@ -481,16 +507,16 @@ function TestLoader() {
if(firstSelectedIndex == -1) { if(firstSelectedIndex == -1) {
return false; return false;
} }
// Begin loading the file immediately, if necessary // Begin loading the file immediately, if necessary
if(testGroups[firstSelectedIndex].status == 'waiting') { if(testGroups[firstSelectedIndex].status == 'waiting') {
getFile(firstSelectedIndex); getFile(firstSelectedIndex);
} }
mode = "multiple"; mode = "multiple";
testGroupIndex = firstSelectedIndex; // start at this chapter testGroupIndex = firstSelectedIndex; // start at this chapter
currentTestIndex = 0; // start at test 0 currentTestIndex = 0; // start at test 0
// Count the number of tests // Count the number of tests
runningTests = 0; runningTests = 0;
for(var i = 0; i < testGroups.length; i++) { for(var i = 0; i < testGroups.length; i++) {
@ -499,16 +525,16 @@ function TestLoader() {
loader.onInitialized(runningTests); loader.onInitialized(runningTests);
return true; return true;
}; };
this.getNumTestFiles = function() { this.getNumTestFiles = function() {
return testGroups.length; return testGroups.length;
}; };
/* Toggle the selection of a file. */ /* Toggle the selection of a file. */
this.toggleSelection = function(index) { this.toggleSelection = function(index) {
testGroups[index].selected = !testGroups[index].selected; testGroups[index].selected = !testGroups[index].selected;
} }
} }
/* Controls test generation and running, and sends results to the presenter. */ /* Controls test generation and running, and sends results to the presenter. */
@ -523,8 +549,8 @@ function Controller() {
//into this test framework to handle test case failures and passes in their //into this test framework to handle test case failures and passes in their
//own way (e.g., logging failures to the filesystem) //own way (e.g., logging failures to the filesystem)
this.implementerHook = { this.implementerHook = {
//Adds a test result //Adds a test result
addTestResult: function (test) { }, addTestResult: function (test) { },
//Called whenever all tests have finished running. Provided with the //Called whenever all tests have finished running. Provided with the
//elapsed time in milliseconds. //elapsed time in milliseconds.
@ -550,7 +576,7 @@ function Controller() {
} }
presenter.setTotalTests(totalTests); presenter.setTotalTests(totalTests);
}; };
/* Executes when a test file starts loading. */ /* Executes when a test file starts loading. */
loader.onTestStartLoading = function(index, path) { loader.onTestStartLoading = function(index, path) {
presenter.setTestLoading(index, path); presenter.setTestLoading(index, path);
@ -559,14 +585,14 @@ function Controller() {
/* Executes when a test is ready to run. */ /* Executes when a test is ready to run. */
loader.onTestReady = function(testObj, testSrc) { loader.onTestReady = function(testObj, testSrc) {
presenter.updateStatus("Running Test: " + testObj.id); presenter.updateStatus("Running Test: " + testObj.id);
elapsed += runner.run(testObj, testSrc); runner.run(testObj, testSrc);
}; };
/* Executes when there are no more tests to run. */ /* Executes when there are no more tests to run. */
loader.onTestsExhausted = function() { loader.onTestsExhausted = function() {
elapsed = elapsed/(1000*60); //minutes elapsed = elapsed/(1000*60); //minutes
elapsed = elapsed.toFixed(3); elapsed = elapsed.toFixed(3);
state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading';
presenter.setState(state); presenter.setState(state);
presenter.finished(elapsed); presenter.finished(elapsed);
@ -574,7 +600,7 @@ function Controller() {
controller.implementerHook.finished(elapsed); controller.implementerHook.finished(elapsed);
} catch(e) { /*no-op*/} } catch(e) { /*no-op*/}
}; };
/* Start the test execution. */ /* Start the test execution. */
this.start = function() { this.start = function() {
elapsed = 0; elapsed = 0;
@ -594,25 +620,25 @@ function Controller() {
loader.onInitialized(); loader.onInitialized();
loader.reset(); loader.reset();
presenter.reset(); presenter.reset();
state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading';
presenter.setState(state); presenter.setState(state);
}; };
/* Start loading tests. */ /* Start loading tests. */
this.startLoadingTests = function() { this.startLoadingTests = function() {
state = 'loading'; state = 'loading';
presenter.setState(state); presenter.setState(state);
loader.startLoadingTests(); loader.startLoadingTests();
} }
/* Set the individual chapter in the laoder and start the controller. */ /* Set the individual chapter in the laoder and start the controller. */
this.runIndividualTest = function(index) { this.runIndividualTest = function(index) {
controller.reset(); controller.reset();
loader.setChapter(index); loader.setChapter(index);
controller.start(); controller.start();
} }
/* Compile a list of the selected tests and start the controller. */ /* Compile a list of the selected tests and start the controller. */
this.runSelected = function() { this.runSelected = function() {
controller.reset(); controller.reset();
@ -620,15 +646,19 @@ function Controller() {
controller.start(); controller.start();
} }
} }
this.runAll = function() { this.runAll = function() {
controller.reset(); controller.reset();
controller.start(); controller.start();
} }
this.toggleSelection = function(index) { this.toggleSelection = function(index) {
loader.toggleSelection(index); loader.toggleSelection(index);
} }
this.testElapsedTime = function(time){
elapsed += time;
}
} }
var controller = new Controller(); var controller = new Controller();
@ -698,7 +728,7 @@ $(function () {
presenter.setDate(data.date); presenter.setDate(data.date);
} }
}); });
// Start loading the files right away. // Start loading the files right away.
controller.startLoadingTests(); controller.startLoadingTests();

22
console/harness/timer.js Normal file
View File

@ -0,0 +1,22 @@
//setTimeout is not available, hence this script was loaded
if(Promise === undefined && this.setTimeout === undefined){
if(/\$DONE()/.test(code))
$ERROR("Async test capability is not supported in your test environment");
}
if(Promise !== undefined && this.setTimeout === undefined)
(function(that){
that.setTimeout = function(callback, delay) {
var p = Promise.resolve();
var start = Date.now();
var end = start + delay;
function check(){
var timeLeft = end - Date.now();
if(timeLeft)
p.then(check);
else
callback();
}
p.then(check);
}
})(this);

10
test/harness/doneprintHandle.js Normal file
View File

@ -0,0 +1,10 @@
function __consolePrintHandle__(msg){
print(msg);
}
function $DONE(){
if(arguments.length === 0)
__consolePrintHandle__('Test262:AsyncTestComplete');
else
__consolePrintHandle__('Error: ' + arguments[0]);
}

1
test/harness/ed.js
View File

@ -8,6 +8,7 @@
if (this.window!==undefined) { //for console support if (this.window!==undefined) { //for console support
this.window.onerror = function(errorMsg, url, lineNumber) { this.window.onerror = function(errorMsg, url, lineNumber) {
this.window.iframeError = errorMsg; this.window.iframeError = errorMsg;
if(typeof $DONE === 'function') $DONE();
}; };
} }

99
test/harness/gs.js
View File

@ -5,38 +5,60 @@
/// copyright and this notice and otherwise comply with the Use Terms. /// copyright and this notice and otherwise comply with the Use Terms.
//Global Scope Test Case Validator //Global Scope Test Case Validator
function $DONE() {
//An exception is expected //An exception is expected
if (testDescrip.negative !== undefined) { if (testDescrip.negative !== undefined) {
//TODO - come up with a generic way of catching the error type //TODO - come up with a generic way of catching the error type
//from this.onerror //from this.onerror
testDescrip.negative = testDescrip.negative === "NotEarlyError" ? testDescrip.negative = testDescrip.negative === "NotEarlyError" ?
testDescrip.negative : testDescrip.negative :
(testDescrip.negative === "^((?!NotEarlyError).)*$" ? (testDescrip.negative === "^((?!NotEarlyError).)*$" ?
testDescrip.negative : "."); testDescrip.negative : ".");
if (this.iframeError === undefined) { //no exception was thrown if (this.iframeError === undefined) { //no exception was thrown
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('No exception was thrown; expected an error "message"' +
' property matching the regular expression "' +
testDescrip.negative + '".'));
} else if (!(new RegExp(testDescrip.negative,
"i").test(this.iframeError))) {
//wrong type of exception thrown
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('Expected an exception with a "message"' +
' property matching the regular expression "' +
testDescrip.negative +
'" to be thrown; actual was "' +
this.iframeError + '".'));
} else {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'pass',
undefined);
}
}
//Exception was not expected to be thrown
else if (this.iframeError !== undefined) {
testRun(testDescrip.id, testRun(testDescrip.id,
testDescrip.path, testDescrip.path,
testDescrip.description, testDescrip.description,
testDescrip.code, testDescrip.code,
'fail', 'fail',
Error('No exception was thrown; expected an error "message"' + Error('Unexpected exception, "' +
' property matching the regular expression "' + this.iframeError + '" was thrown.'));
testDescrip.negative + '".')); }
} else if (!(new RegExp(testDescrip.negative,
"i").test(this.iframeError))) { else {
//wrong type of exception thrown
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('Expected an exception with a "message"' +
' property matching the regular expression "' +
testDescrip.negative +
'" to be thrown; actual was "' +
this.iframeError + '".'));
} else {
testRun(testDescrip.id, testRun(testDescrip.id,
testDescrip.path, testDescrip.path,
testDescrip.description, testDescrip.description,
@ -44,26 +66,7 @@ if (testDescrip.negative !== undefined) {
'pass', 'pass',
undefined); undefined);
} }
}
//Exception was not expected to be thrown //teardown
else if (this.iframeError !== undefined) { testFinished();
testRun(testDescrip.id, }
testDescrip.path,
testDescrip.description,
testDescrip.code,
'fail',
Error('Unexpected exception, "' +
this.iframeError + '" was thrown.'));
}
else {
testRun(testDescrip.id,
testDescrip.path,
testDescrip.description,
testDescrip.code,
'pass',
undefined);
}
testFinished();

151
test/harness/sth.js
View File

@ -1,7 +1,7 @@
/// Copyright (c) 2012 Ecma International. All rights reserved. /// Copyright (c) 2012 Ecma International. All rights reserved.
/// Ecma International makes this code available under the terms and conditions set /// Ecma International makes this code available under the terms and conditions set
/// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the /// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the
/// "Use Terms"). Any redistribution of this code must retain the above /// "Use Terms"). Any redistribution of this code must retain the above
/// copyright and this notice and otherwise comply with the Use Terms. /// copyright and this notice and otherwise comply with the Use Terms.
// Do not cache any JSON files - see // Do not cache any JSON files - see
@ -27,6 +27,8 @@ function BrowserRunner() {
errorDetectorFileContents, errorDetectorFileContents,
simpleTestAPIContents, simpleTestAPIContents,
globalScopeContents, globalScopeContents,
timerContents,
startTime,
harnessDir = "harness/"; harnessDir = "harness/";
$.ajax({async: false, $.ajax({async: false,
@ -43,6 +45,11 @@ function BrowserRunner() {
dataType: "text", dataType: "text",
success: function(data){globalScopeContents = data;}, success: function(data){globalScopeContents = data;},
url:harnessDir+"gs.js"}); url:harnessDir+"gs.js"});
$.ajax({async: false,
dataType: "text",
success: function(data){timerContents = data;},
url:harnessDir+"timer.js"});
/* Called by the child window to notify that the test has /* Called by the child window to notify that the test has
* finished. This function call is put in a separate script block * finished. This function call is put in a separate script block
@ -70,6 +77,8 @@ function BrowserRunner() {
document.body.removeChild(iframe); document.body.removeChild(iframe);
instance.onComplete(currentTest); instance.onComplete(currentTest);
//update elapsed time
controller.testElapsedTime(new Date() - startTime);
} }
/* Called from the child window after the test has run. */ /* Called from the child window after the test has run. */
@ -85,8 +94,8 @@ function BrowserRunner() {
/* Run the test. */ /* Run the test. */
this.run = function (test, code) { this.run = function (test, code) {
var start = new Date(); startTime = new Date();
//--Detect proper window.onerror support //--Detect proper window.onerror support
if (instance.supportsWindowOnerror===undefined) { if (instance.supportsWindowOnerror===undefined) {
var iframePrereqs = document.createElement("iframe"); var iframePrereqs = document.createElement("iframe");
@ -96,25 +105,25 @@ function BrowserRunner() {
} }
document.body.appendChild(iframePrereqs); document.body.appendChild(iframePrereqs);
var iwinPrereqs = iframePrereqs.contentWindow; var iwinPrereqs = iframePrereqs.contentWindow;
var idocPrereqs = iwinPrereqs.document; var idocPrereqs = iwinPrereqs.document;
idocPrereqs.open(); idocPrereqs.open();
iwinPrereqs.failCount = 0; iwinPrereqs.failCount = 0;
var stuff = [ var stuff = [
"window.onerror = function(a, b, c) { this.failCount++; }", "window.onerror = function(a, b, c) { this.failCount++; }",
"va xyz =", "va xyz =",
"throw Error();" "throw Error();"
]; ];
for(var i in stuff) { for(var i in stuff) {
idocPrereqs.writeln("<script type='text/javascript'>"); idocPrereqs.writeln("<script type='text/javascript'>");
idocPrereqs.writeln(stuff[i]); idocPrereqs.writeln(stuff[i]);
idocPrereqs.writeln("</script>"); idocPrereqs.writeln("</script>");
} }
idocPrereqs.close(); idocPrereqs.close();
//TODO - 500ms *should* be a sufficient delay //TODO - 500ms *should* be a sufficient delay
setTimeout(function() { setTimeout(function() {
instance.supportsWindowOnerror = iwinPrereqs.failCount === 2; instance.supportsWindowOnerror = iwinPrereqs.failCount === 2;
@ -124,7 +133,7 @@ function BrowserRunner() {
}, 500); }, 500);
return 0; // initial config, ignore this timing. return 0; // initial config, ignore this timing.
} }
currentTest = {}; currentTest = {};
for (var tempIndex in test) { for (var tempIndex in test) {
if (test.hasOwnProperty(tempIndex)) { if (test.hasOwnProperty(tempIndex)) {
@ -132,8 +141,6 @@ function BrowserRunner() {
} }
} }
currentTest.code = code; currentTest.code = code;
iframe = document.createElement("iframe"); iframe = document.createElement("iframe");
iframe.setAttribute("id", "runnerIframe"); iframe.setAttribute("id", "runnerIframe");
@ -188,11 +195,25 @@ function BrowserRunner() {
iwin.testDescrip = currentTest; iwin.testDescrip = currentTest;
//Add an error handler capable of catching so-called early errors //Add an error handler capable of catching so-called early errors
//idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>"); //idoc.writeln("<script type='text/javascript' src='harness/ed.js'>" + "</script>")
idoc.writeln("<script type='text/javascript'>"); idoc.writeln("<script type='text/javascript'>");
idoc.writeln(errorDetectorFileContents); idoc.writeln(errorDetectorFileContents);
idoc.writeln("</script>"); idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
idoc.writeln("</script>");
//this is mainly applicable for consoles that do not have setTimeout support
//idoc.writeln("<script type='text/javascript' src='harness/timer.js' defer>" + "</script>");
if(setTimeout === undefined && /\$DONE()/.test(code)){
idoc.writeln("<script type='text/javascript'>");
idoc.writeln(timerContents);
idoc.writeln("</script>");
}
//Run the code //Run the code
idoc.writeln("<script type='text/javascript'>"); idoc.writeln("<script type='text/javascript'>");
if (! instance.supportsWindowOnerror) { if (! instance.supportsWindowOnerror) {
@ -201,17 +222,21 @@ function BrowserRunner() {
idoc.writeln(code); idoc.writeln(code);
} }
idoc.writeln("</script>"); idoc.writeln("</script>");
//Validate the results
//idoc.writeln("<script type='text/javascript' src='harness/gs.js' defer>" + "</script>");
idoc.writeln("<script type='text/javascript'>"); idoc.writeln("<script type='text/javascript'>");
idoc.writeln(globalScopeContents);
if(!/\$DONE()/.test(code))
//if the test is synchronous - call $DONE immediately
idoc.writeln("if(typeof $DONE === 'function') $DONE()");
else{
//in case the test does not call $DONE asynchronously then
//bailout after 1 min or given bailout time by calling $DONE
var asyncval = parseInt(test.timeout);
var testTimeout = asyncval !== asyncval ? 2000 : asyncval;
idoc.writeln("setTimeout(function() {$ERROR(\" Test Timed Out at " + testTimeout +"\" )} ," + testTimeout + ")");
}
idoc.writeln("</script>"); idoc.writeln("</script>");
idoc.close(); idoc.close();
var elapsed = new Date() - start;
return elapsed;
}; };
//--Helper functions------------------------------------------------------- //--Helper functions-------------------------------------------------------
@ -264,10 +289,10 @@ function TestLoader() {
else { else {
presenter.updateStatus("Loading file: " + testGroups[testGroupIndex].path); presenter.updateStatus("Loading file: " + testGroups[testGroupIndex].path);
testGroups[testGroupIndex].onLoaded = getNextXML; testGroups[testGroupIndex].onLoaded = getNextXML;
} }
} }
/* Get the test list xml */ /* Get the test list xml */
function loadTestXML() { function loadTestXML() {
var testsListLoader = new XMLHttpRequest(); var testsListLoader = new XMLHttpRequest();
@ -288,7 +313,7 @@ function TestLoader() {
onLoaded: function(){} onLoaded: function(){}
}; };
presenter.setTestWaiting(i, testSuite[i]); presenter.setTestWaiting(i, testSuite[i]);
var tr = $('#chapterSelector table tr').filter(':nth-child(' + (i+1) + ')'); var tr = $('#chapterSelector table tr').filter(':nth-child(' + (i+1) + ')');
tr.find('img').filter('[alt="Run"]').bind('click', {index:i}, function(event){ tr.find('img').filter('[alt="Run"]').bind('click', {index:i}, function(event){
controller.runIndividualTest(event.data.index); controller.runIndividualTest(event.data.index);
@ -298,25 +323,25 @@ function TestLoader() {
getFile(); getFile();
}}); }});
} }
/* Get the test file. Handles all the logic of figuring out the next file to load. */ /* Get the test file. Handles all the logic of figuring out the next file to load. */
function getFile(index) { function getFile(index) {
index = (arguments.length == 0) ? -1 : index; index = (arguments.length == 0) ? -1 : index;
// Look for selected waiting chapters (priority because you'll probably want to run them soon) // Look for selected waiting chapters (priority because you'll probably want to run them soon)
for(var i = 0; index == -1 && i < testGroups.length; i++) { for(var i = 0; index == -1 && i < testGroups.length; i++) {
if(testGroups[i].status == 'waiting' && testGroups[i].selected) { if(testGroups[i].status == 'waiting' && testGroups[i].selected) {
index = i; index = i;
} }
} }
// Look for just chapters waiting to be loaded. // Look for just chapters waiting to be loaded.
for(var i = 0; index == -1 && i < testGroups.length; i++) { for(var i = 0; index == -1 && i < testGroups.length; i++) {
if(testGroups[i].status == 'waiting') { if(testGroups[i].status == 'waiting') {
index = i; index = i;
} }
} }
if(index == -1) { if(index == -1) {
// Still -1? No more tests are waiting to be loaded then. // Still -1? No more tests are waiting to be loaded then.
if(controller.state == 'loading') { if(controller.state == 'loading') {
@ -324,13 +349,13 @@ function TestLoader() {
} }
return; return;
} }
presenter.setTestLoading(index, testGroups[index].path); presenter.setTestLoading(index, testGroups[index].path);
// the only other status that should be set when we get here is 'priorityloading' // the only other status that should be set when we get here is 'priorityloading'
if(testGroups[index].status == 'waiting') { if(testGroups[index].status == 'waiting') {
testGroups[index].status = 'loading'; testGroups[index].status = 'loading';
} }
loader.onTestStartLoading(index, testGroups[index].path); loader.onTestStartLoading(index, testGroups[index].path);
// Create the AJAX call to grab the file. // Create the AJAX call to grab the file.
$.ajax({ $.ajax({
@ -344,11 +369,11 @@ function TestLoader() {
}, },
error: function(xhr, textStatus, errorThrown) { error: function(xhr, textStatus, errorThrown) {
// TODO: Catch this error and update UI accordingly. Unlikely to happen, but errors would be 404 or 5-- errors. // TODO: Catch this error and update UI accordingly. Unlikely to happen, but errors would be 404 or 5-- errors.
} }
}); });
} }
/* Executes when a test file finishes loading. */ /* Executes when a test file finishes loading. */
function onTestLoaded(index, name, numTests) { function onTestLoaded(index, name, numTests) {
presenter.setTestLoaded(index, name, numTests); presenter.setTestLoaded(index, name, numTests);
@ -357,7 +382,7 @@ function TestLoader() {
loader.runningTests += numTests; loader.runningTests += numTests;
loader.onInitialized( loader.runningTests ); loader.onInitialized( loader.runningTests );
} }
// The loading status is only assigned when downloading files in sequence, otherwise it // The loading status is only assigned when downloading files in sequence, otherwise it
// gets the status of priorityloading. When loading out of order, we only want to download // gets the status of priorityloading. When loading out of order, we only want to download
// the single file, so we'll only tell it to get the next file when we see a status of // the single file, so we'll only tell it to get the next file when we see a status of
@ -371,10 +396,10 @@ function TestLoader() {
testGroups[index].status = 'loaded'; testGroups[index].status = 'loaded';
loader.setChapter(index); loader.setChapter(index);
} }
testGroups[index].onLoaded(); testGroups[index].onLoaded();
}; };
function getIdFromPath (path) { function getIdFromPath (path) {
//path is of the form "a/b/c.js" //path is of the form "a/b/c.js"
@ -420,7 +445,7 @@ function TestLoader() {
return; return;
} }
} }
// And if // And if
else { else {
// We don't have tests left in this test group, so move on // We don't have tests left in this test group, so move on
// to the next. // to the next.
@ -428,7 +453,7 @@ function TestLoader() {
} }
getNextXML(); getNextXML();
} }
// //
else { else {
// We're done. // We're done.
loader.onTestsExhausted(); loader.onTestsExhausted();
@ -446,18 +471,18 @@ function TestLoader() {
currentTestIndex = 0; currentTestIndex = 0;
testGroupIndex = 0; testGroupIndex = 0;
}; };
/* Begin downloading test files. */ /* Begin downloading test files. */
this.startLoadingTests = function() { this.startLoadingTests = function() {
loadTestXML(); loadTestXML();
}; };
/* Prepare for testing a single chapter. */ /* Prepare for testing a single chapter. */
this.setChapter = function(index) { this.setChapter = function(index) {
currentTestIndex = 0; currentTestIndex = 0;
testGroupIndex = index; testGroupIndex = index;
mode = "one"; mode = "one";
if(testGroups[index].status == 'loaded') { if(testGroups[index].status == 'loaded') {
loader.onInitialized(testGroups[index].tests.length); loader.onInitialized(testGroups[index].tests.length);
} }
@ -467,7 +492,7 @@ function TestLoader() {
loader.onInitialized(0); loader.onInitialized(0);
} }
}; };
/* Prepare for testing multiple chapters. Returns true if at least one chapter was selected. */ /* Prepare for testing multiple chapters. Returns true if at least one chapter was selected. */
this.setMultiple = function() { this.setMultiple = function() {
// Find the index of the first selection // Find the index of the first selection
@ -481,16 +506,16 @@ function TestLoader() {
if(firstSelectedIndex == -1) { if(firstSelectedIndex == -1) {
return false; return false;
} }
// Begin loading the file immediately, if necessary // Begin loading the file immediately, if necessary
if(testGroups[firstSelectedIndex].status == 'waiting') { if(testGroups[firstSelectedIndex].status == 'waiting') {
getFile(firstSelectedIndex); getFile(firstSelectedIndex);
} }
mode = "multiple"; mode = "multiple";
testGroupIndex = firstSelectedIndex; // start at this chapter testGroupIndex = firstSelectedIndex; // start at this chapter
currentTestIndex = 0; // start at test 0 currentTestIndex = 0; // start at test 0
// Count the number of tests // Count the number of tests
runningTests = 0; runningTests = 0;
for(var i = 0; i < testGroups.length; i++) { for(var i = 0; i < testGroups.length; i++) {
@ -499,16 +524,16 @@ function TestLoader() {
loader.onInitialized(runningTests); loader.onInitialized(runningTests);
return true; return true;
}; };
this.getNumTestFiles = function() { this.getNumTestFiles = function() {
return testGroups.length; return testGroups.length;
}; };
/* Toggle the selection of a file. */ /* Toggle the selection of a file. */
this.toggleSelection = function(index) { this.toggleSelection = function(index) {
testGroups[index].selected = !testGroups[index].selected; testGroups[index].selected = !testGroups[index].selected;
} }
} }
/* Controls test generation and running, and sends results to the presenter. */ /* Controls test generation and running, and sends results to the presenter. */
@ -523,8 +548,8 @@ function Controller() {
//into this test framework to handle test case failures and passes in their //into this test framework to handle test case failures and passes in their
//own way (e.g., logging failures to the filesystem) //own way (e.g., logging failures to the filesystem)
this.implementerHook = { this.implementerHook = {
//Adds a test result //Adds a test result
addTestResult: function (test) { }, addTestResult: function (test) { },
//Called whenever all tests have finished running. Provided with the //Called whenever all tests have finished running. Provided with the
//elapsed time in milliseconds. //elapsed time in milliseconds.
@ -550,7 +575,7 @@ function Controller() {
} }
presenter.setTotalTests(totalTests); presenter.setTotalTests(totalTests);
}; };
/* Executes when a test file starts loading. */ /* Executes when a test file starts loading. */
loader.onTestStartLoading = function(index, path) { loader.onTestStartLoading = function(index, path) {
presenter.setTestLoading(index, path); presenter.setTestLoading(index, path);
@ -559,14 +584,14 @@ function Controller() {
/* Executes when a test is ready to run. */ /* Executes when a test is ready to run. */
loader.onTestReady = function(testObj, testSrc) { loader.onTestReady = function(testObj, testSrc) {
presenter.updateStatus("Running Test: " + testObj.id); presenter.updateStatus("Running Test: " + testObj.id);
elapsed += runner.run(testObj, testSrc); runner.run(testObj, testSrc);
}; };
/* Executes when there are no more tests to run. */ /* Executes when there are no more tests to run. */
loader.onTestsExhausted = function() { loader.onTestsExhausted = function() {
elapsed = elapsed/(1000*60); //minutes elapsed = elapsed/(1000*60); //minutes
elapsed = elapsed.toFixed(3); elapsed = elapsed.toFixed(3);
state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading';
presenter.setState(state); presenter.setState(state);
presenter.finished(elapsed); presenter.finished(elapsed);
@ -574,7 +599,7 @@ function Controller() {
controller.implementerHook.finished(elapsed); controller.implementerHook.finished(elapsed);
} catch(e) { /*no-op*/} } catch(e) { /*no-op*/}
}; };
/* Start the test execution. */ /* Start the test execution. */
this.start = function() { this.start = function() {
elapsed = 0; elapsed = 0;
@ -594,25 +619,25 @@ function Controller() {
loader.onInitialized(); loader.onInitialized();
loader.reset(); loader.reset();
presenter.reset(); presenter.reset();
state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading';
presenter.setState(state); presenter.setState(state);
}; };
/* Start loading tests. */ /* Start loading tests. */
this.startLoadingTests = function() { this.startLoadingTests = function() {
state = 'loading'; state = 'loading';
presenter.setState(state); presenter.setState(state);
loader.startLoadingTests(); loader.startLoadingTests();
} }
/* Set the individual chapter in the laoder and start the controller. */ /* Set the individual chapter in the laoder and start the controller. */
this.runIndividualTest = function(index) { this.runIndividualTest = function(index) {
controller.reset(); controller.reset();
loader.setChapter(index); loader.setChapter(index);
controller.start(); controller.start();
} }
/* Compile a list of the selected tests and start the controller. */ /* Compile a list of the selected tests and start the controller. */
this.runSelected = function() { this.runSelected = function() {
controller.reset(); controller.reset();
@ -620,15 +645,19 @@ function Controller() {
controller.start(); controller.start();
} }
} }
this.runAll = function() { this.runAll = function() {
controller.reset(); controller.reset();
controller.start(); controller.start();
} }
this.toggleSelection = function(index) { this.toggleSelection = function(index) {
loader.toggleSelection(index); loader.toggleSelection(index);
} }
this.testElapsedTime = function(time){
elapsed += time;
}
} }
var controller = new Controller(); var controller = new Controller();
@ -698,7 +727,7 @@ $(function () {
presenter.setDate(data.date); presenter.setDate(data.date);
} }
}); });
// Start loading the files right away. // Start loading the files right away.
controller.startLoadingTests(); controller.startLoadingTests();

22
test/harness/timer.js Normal file
View File

@ -0,0 +1,22 @@
//setTimeout is not available, hence this script was loaded
if(Promise === undefined && this.setTimeout === undefined){
if(/\$DONE()/.test(code))
$ERROR("Async test capability is not supported in your test environment");
}
if(Promise !== undefined && this.setTimeout === undefined)
(function(that){
that.setTimeout = function(callback, delay) {
var p = Promise.resolve();
var start = Date.now();
var end = start + delay;
function check(){
var timeLeft = end - Date.now();
if(timeLeft)
p.then(check);
else
callback();
}
p.then(check);
}
})(this);

29
tools/packaging/test262.py
View File

@ -73,6 +73,7 @@ def BuildOptions():
result.add_option("--junitname", help="Filename to save test results in JUnit XML format") result.add_option("--junitname", help="Filename to save test results in JUnit XML format")
result.add_option("--loglevel", default="warning", result.add_option("--loglevel", default="warning",
help="sets log level to debug, info, warning, error, or critical") help="sets log level to debug, info, warning, error, or critical")
result.add_option("--print-handle", default="", help="Command to print from console")
return result return result
@ -193,15 +194,19 @@ class TestResult(object):
testpackage = testclass testpackage = testclass
return(testpackage,testclass,testcase) return(testpackage,testclass,testcase)
def HasFailed(self): def HasFailed(self):
return self.exit_code != 0 return self.exit_code != 0
def HasUnexpectedOutcome(self): def AsyncHasFailed(self):
if self.case.IsNegative(): return 'Test262:AsyncTestComplete' not in self.stdout
return not self.HasFailed()
else:
return self.HasFailed()
def HasUnexpectedOutcome(self):
if self.case.IsAsyncTest():
return self.AsyncHasFailed() or self.HasFailed()
elif self.case.IsNegative():
return not self.HasFailed()
else:
return self.HasFailed()
class TestCase(object): class TestCase(object):
@ -242,6 +247,9 @@ class TestCase(object):
def IsNoStrict(self): def IsNoStrict(self):
return 'noStrict' in self.testRecord return 'noStrict' in self.testRecord
def IsAsyncTest(self):
return '$DONE' in self.test
def GetSource(self): def GetSource(self):
# "var testDescrip = " + str(self.testRecord) + ';\n\n' + \ # "var testDescrip = " + str(self.testRecord) + ';\n\n' + \
source = self.suite.GetInclude("cth.js") + \ source = self.suite.GetInclude("cth.js") + \
@ -249,6 +257,8 @@ class TestCase(object):
self.suite.GetInclude("ed.js") + \ self.suite.GetInclude("ed.js") + \
self.suite.GetInclude("testBuiltInObject.js") + \ self.suite.GetInclude("testBuiltInObject.js") + \
self.suite.GetInclude("testIntl.js") + \ self.suite.GetInclude("testIntl.js") + \
self.suite.GetInclude("timer.js") + \
self.suite.GetInclude("doneprintHandle.js").replace('print', self.suite.print_handle) + \
self.test + '\n' self.test + '\n'
if self.strict_mode: if self.strict_mode:
@ -332,14 +342,16 @@ def MakePlural(n):
class TestSuite(object): class TestSuite(object):
def __init__(self, root, strict_only, non_strict_only, unmarked_default): def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
# TODO: derive from packagerConfig.py # TODO: derive from packagerConfig.py
self.test_root = path.join(root, 'test', 'suite') self.test_root = path.join(root, 'test', 'suite')
self.lib_root = path.join(root, 'test', 'harness') self.lib_root = path.join(root, 'test', 'harness')
self.strict_only = strict_only self.strict_only = strict_only
self.non_strict_only = non_strict_only self.non_strict_only = non_strict_only
self.unmarked_default = unmarked_default self.unmarked_default = unmarked_default
self.print_handle = print_handle
self.include_cache = { } self.include_cache = { }
def Validate(self): def Validate(self):
if not path.exists(self.test_root): if not path.exists(self.test_root):
@ -536,7 +548,8 @@ def Main():
test_suite = TestSuite(options.tests, test_suite = TestSuite(options.tests,
options.strict_only, options.strict_only,
options.non_strict_only, options.non_strict_only,
options.unmarked_default) options.unmarked_default,
options.print_handle)
test_suite.Validate() test_suite.Validate()
if options.loglevel == 'debug': if options.loglevel == 'debug':
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)