From 9aa6b917fa1ee33993899c984199e02e3ab4a9e9 Mon Sep 17 00:00:00 2001 From: NikSurya Date: Tue, 15 Jul 2014 13:47:59 -0700 Subject: [PATCH] Adding Support for Async Tests --- console/harness/doneprintHandle.js | 10 ++ console/harness/ed.js | 1 + console/harness/gs.js | 99 ++++++++++--------- console/harness/sth.js | 152 +++++++++++++++++------------ console/harness/timer.js | 22 +++++ test/harness/doneprintHandle.js | 10 ++ test/harness/ed.js | 1 + test/harness/gs.js | 99 ++++++++++--------- test/harness/sth.js | 151 ++++++++++++++++------------ test/harness/timer.js | 22 +++++ tools/packaging/test262.py | 29 ++++-- 11 files changed, 370 insertions(+), 226 deletions(-) create mode 100644 console/harness/doneprintHandle.js create mode 100644 console/harness/timer.js create mode 100644 test/harness/doneprintHandle.js create mode 100644 test/harness/timer.js diff --git a/console/harness/doneprintHandle.js b/console/harness/doneprintHandle.js new file mode 100644 index 0000000000..317aa9c4ef --- /dev/null +++ b/console/harness/doneprintHandle.js @@ -0,0 +1,10 @@ +function __consolePrintHandle__(msg){ + print(msg); +} + +function $DONE(){ + if(arguments.length === 0) + __consolePrintHandle__('Test262:AsyncTestComplete'); + else + __consolePrintHandle__('Error: ' + arguments[0]); +} \ No newline at end of file diff --git a/console/harness/ed.js b/console/harness/ed.js index 89d0dfc1ca..88e2cecc49 100644 --- a/console/harness/ed.js +++ b/console/harness/ed.js @@ -8,6 +8,7 @@ if (this.window!==undefined) { //for console support this.window.onerror = function(errorMsg, url, lineNumber) { this.window.iframeError = errorMsg; + if(typeof $DONE === 'function') $DONE(); }; } diff --git a/console/harness/gs.js b/console/harness/gs.js index 210c08548f..855cb85bf4 100644 --- a/console/harness/gs.js +++ b/console/harness/gs.js @@ -5,38 +5,60 @@ /// copyright and this notice and otherwise comply with the Use Terms. //Global Scope Test Case Validator +function $DONE() { -//An exception is expected -if (testDescrip.negative !== undefined) { - //TODO - come up with a generic way of catching the error type - //from this.onerror - testDescrip.negative = testDescrip.negative === "NotEarlyError" ? - testDescrip.negative : - (testDescrip.negative === "^((?!NotEarlyError).)*$" ? - testDescrip.negative : "."); - if (this.iframeError === undefined) { //no exception was thrown + //An exception is expected + if (testDescrip.negative !== undefined) { + //TODO - come up with a generic way of catching the error type + //from this.onerror + testDescrip.negative = testDescrip.negative === "NotEarlyError" ? + testDescrip.negative : + (testDescrip.negative === "^((?!NotEarlyError).)*$" ? + testDescrip.negative : "."); + if (this.iframeError === undefined) { //no exception was thrown + testRun(testDescrip.id, + testDescrip.path, + testDescrip.description, + testDescrip.code, + 'fail', + Error('No exception was thrown; expected an error "message"' + + ' property matching the regular expression "' + + testDescrip.negative + '".')); + } else if (!(new RegExp(testDescrip.negative, + "i").test(this.iframeError))) { + //wrong type of exception thrown + testRun(testDescrip.id, + testDescrip.path, + testDescrip.description, + testDescrip.code, + 'fail', + Error('Expected an exception with a "message"' + + ' property matching the regular expression "' + + testDescrip.negative + + '" to be thrown; actual was "' + + this.iframeError + '".')); + } else { + testRun(testDescrip.id, + testDescrip.path, + testDescrip.description, + testDescrip.code, + 'pass', + undefined); + } + } + + //Exception was not expected to be thrown + else if (this.iframeError !== undefined) { testRun(testDescrip.id, testDescrip.path, testDescrip.description, testDescrip.code, 'fail', - Error('No exception was thrown; expected an error "message"' + - ' property matching the regular expression "' + - testDescrip.negative + '".')); - } else if (!(new RegExp(testDescrip.negative, - "i").test(this.iframeError))) { - //wrong type of exception thrown - testRun(testDescrip.id, - testDescrip.path, - testDescrip.description, - testDescrip.code, - 'fail', - Error('Expected an exception with a "message"' + - ' property matching the regular expression "' + - testDescrip.negative + - '" to be thrown; actual was "' + - this.iframeError + '".')); - } else { + Error('Unexpected exception, "' + + this.iframeError + '" was thrown.')); + } + + else { testRun(testDescrip.id, testDescrip.path, testDescrip.description, @@ -44,26 +66,7 @@ if (testDescrip.negative !== undefined) { 'pass', undefined); } -} -//Exception was not expected to be thrown -else if (this.iframeError !== undefined) { - testRun(testDescrip.id, - testDescrip.path, - testDescrip.description, - testDescrip.code, - 'fail', - Error('Unexpected exception, "' + - this.iframeError + '" was thrown.')); -} - -else { - testRun(testDescrip.id, - testDescrip.path, - testDescrip.description, - testDescrip.code, - 'pass', - undefined); -} - -testFinished(); \ No newline at end of file + //teardown + testFinished(); +} \ No newline at end of file diff --git a/console/harness/sth.js b/console/harness/sth.js index c2ce493714..c61b6a7aa5 100644 --- a/console/harness/sth.js +++ b/console/harness/sth.js @@ -1,7 +1,7 @@ -/// Copyright (c) 2012 Ecma International. All rights reserved. +/// Copyright (c) 2012 Ecma International. All rights reserved. /// Ecma International makes this code available under the terms and conditions set -/// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the -/// "Use Terms"). Any redistribution of this code must retain the above +/// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the +/// "Use Terms"). Any redistribution of this code must retain the above /// copyright and this notice and otherwise comply with the Use Terms. // Do not cache any JSON files - see @@ -27,6 +27,8 @@ function BrowserRunner() { errorDetectorFileContents, simpleTestAPIContents, globalScopeContents, + timerContents, + startTime, harnessDir = "harness/"; $.ajax({async: false, @@ -44,6 +46,11 @@ function BrowserRunner() { success: function(data){globalScopeContents = data;}, url:harnessDir+"gs.js"}); + $.ajax({async: false, + dataType: "text", + success: function(data){timerContents = data;}, + url:harnessDir+"timer.js"}); + /* Called by the child window to notify that the test has * finished. This function call is put in a separate script block * at the end of the page so errors in the test script block @@ -70,6 +77,8 @@ function BrowserRunner() { document.body.removeChild(iframe); instance.onComplete(currentTest); + //update elapsed time + controller.testElapsedTime(new Date() - startTime); } /* Called from the child window after the test has run. */ @@ -85,8 +94,8 @@ function BrowserRunner() { /* Run the test. */ this.run = function (test, code) { - var start = new Date(); - + startTime = new Date(); + //--Detect proper window.onerror support if (instance.supportsWindowOnerror===undefined) { var iframePrereqs = document.createElement("iframe"); @@ -96,25 +105,25 @@ function BrowserRunner() { } document.body.appendChild(iframePrereqs); - var iwinPrereqs = iframePrereqs.contentWindow; + var iwinPrereqs = iframePrereqs.contentWindow; var idocPrereqs = iwinPrereqs.document; idocPrereqs.open(); - + iwinPrereqs.failCount = 0; - + var stuff = [ "window.onerror = function(a, b, c) { this.failCount++; }", "va xyz =", "throw Error();" ]; - + for(var i in stuff) { idocPrereqs.writeln(""); } idocPrereqs.close(); - + //TODO - 500ms *should* be a sufficient delay setTimeout(function() { instance.supportsWindowOnerror = iwinPrereqs.failCount === 2; @@ -124,7 +133,7 @@ function BrowserRunner() { }, 500); return 0; // initial config, ignore this timing. } - + currentTest = {}; for (var tempIndex in test) { if (test.hasOwnProperty(tempIndex)) { @@ -132,8 +141,6 @@ function BrowserRunner() { } } currentTest.code = code; - - iframe = document.createElement("iframe"); iframe.setAttribute("id", "runnerIframe"); @@ -188,11 +195,25 @@ function BrowserRunner() { iwin.testDescrip = currentTest; //Add an error handler capable of catching so-called early errors - //idoc.writeln(""); + //idoc.writeln("") idoc.writeln(""); + //Validate the results + //idoc.writeln(""); + idoc.writeln(""); + + //this is mainly applicable for consoles that do not have setTimeout support + //idoc.writeln(""); + if(setTimeout === undefined && /\$DONE()/.test(code)){ + idoc.writeln(""); + } + //Run the code idoc.writeln(""); - - //Validate the results - //idoc.writeln(""); + idoc.writeln(""); idoc.close(); - - var elapsed = new Date() - start; - - return elapsed; }; //--Helper functions------------------------------------------------------- @@ -264,10 +290,10 @@ function TestLoader() { else { presenter.updateStatus("Loading file: " + testGroups[testGroupIndex].path); testGroups[testGroupIndex].onLoaded = getNextXML; - + } } - + /* Get the test list xml */ function loadTestXML() { var testsListLoader = new XMLHttpRequest(); @@ -288,7 +314,7 @@ function TestLoader() { onLoaded: function(){} }; presenter.setTestWaiting(i, testSuite[i]); - + var tr = $('#chapterSelector table tr').filter(':nth-child(' + (i+1) + ')'); tr.find('img').filter('[alt="Run"]').bind('click', {index:i}, function(event){ controller.runIndividualTest(event.data.index); @@ -298,25 +324,25 @@ function TestLoader() { getFile(); }}); } - + /* Get the test file. Handles all the logic of figuring out the next file to load. */ function getFile(index) { index = (arguments.length == 0) ? -1 : index; - + // Look for selected waiting chapters (priority because you'll probably want to run them soon) for(var i = 0; index == -1 && i < testGroups.length; i++) { if(testGroups[i].status == 'waiting' && testGroups[i].selected) { index = i; } } - + // Look for just chapters waiting to be loaded. for(var i = 0; index == -1 && i < testGroups.length; i++) { if(testGroups[i].status == 'waiting') { index = i; } } - + if(index == -1) { // Still -1? No more tests are waiting to be loaded then. if(controller.state == 'loading') { @@ -324,13 +350,13 @@ function TestLoader() { } return; } - + presenter.setTestLoading(index, testGroups[index].path); // the only other status that should be set when we get here is 'priorityloading' if(testGroups[index].status == 'waiting') { testGroups[index].status = 'loading'; } - + loader.onTestStartLoading(index, testGroups[index].path); // Create the AJAX call to grab the file. $.ajax({ @@ -344,11 +370,11 @@ function TestLoader() { }, error: function(xhr, textStatus, errorThrown) { // TODO: Catch this error and update UI accordingly. Unlikely to happen, but errors would be 404 or 5-- errors. - + } }); } - + /* Executes when a test file finishes loading. */ function onTestLoaded(index, name, numTests) { presenter.setTestLoaded(index, name, numTests); @@ -357,7 +383,7 @@ function TestLoader() { loader.runningTests += numTests; loader.onInitialized( loader.runningTests ); } - + // The loading status is only assigned when downloading files in sequence, otherwise it // gets the status of priorityloading. When loading out of order, we only want to download // the single file, so we'll only tell it to get the next file when we see a status of @@ -371,10 +397,10 @@ function TestLoader() { testGroups[index].status = 'loaded'; loader.setChapter(index); } - + testGroups[index].onLoaded(); }; - + function getIdFromPath (path) { //path is of the form "a/b/c.js" @@ -420,7 +446,7 @@ function TestLoader() { return; } } - // And if + // And if else { // We don't have tests left in this test group, so move on // to the next. @@ -428,7 +454,7 @@ function TestLoader() { } getNextXML(); } - // + // else { // We're done. loader.onTestsExhausted(); @@ -446,18 +472,18 @@ function TestLoader() { currentTestIndex = 0; testGroupIndex = 0; }; - + /* Begin downloading test files. */ this.startLoadingTests = function() { loadTestXML(); }; - + /* Prepare for testing a single chapter. */ this.setChapter = function(index) { currentTestIndex = 0; testGroupIndex = index; mode = "one"; - + if(testGroups[index].status == 'loaded') { loader.onInitialized(testGroups[index].tests.length); } @@ -467,7 +493,7 @@ function TestLoader() { loader.onInitialized(0); } }; - + /* Prepare for testing multiple chapters. Returns true if at least one chapter was selected. */ this.setMultiple = function() { // Find the index of the first selection @@ -481,16 +507,16 @@ function TestLoader() { if(firstSelectedIndex == -1) { return false; } - + // Begin loading the file immediately, if necessary if(testGroups[firstSelectedIndex].status == 'waiting') { getFile(firstSelectedIndex); } - + mode = "multiple"; testGroupIndex = firstSelectedIndex; // start at this chapter currentTestIndex = 0; // start at test 0 - + // Count the number of tests runningTests = 0; for(var i = 0; i < testGroups.length; i++) { @@ -499,16 +525,16 @@ function TestLoader() { loader.onInitialized(runningTests); return true; }; - + this.getNumTestFiles = function() { return testGroups.length; }; - + /* Toggle the selection of a file. */ this.toggleSelection = function(index) { testGroups[index].selected = !testGroups[index].selected; } - + } /* Controls test generation and running, and sends results to the presenter. */ @@ -523,8 +549,8 @@ function Controller() { //into this test framework to handle test case failures and passes in their //own way (e.g., logging failures to the filesystem) this.implementerHook = { - //Adds a test result - addTestResult: function (test) { }, + //Adds a test result + addTestResult: function (test) { }, //Called whenever all tests have finished running. Provided with the //elapsed time in milliseconds. @@ -550,7 +576,7 @@ function Controller() { } presenter.setTotalTests(totalTests); }; - + /* Executes when a test file starts loading. */ loader.onTestStartLoading = function(index, path) { presenter.setTestLoading(index, path); @@ -559,14 +585,14 @@ function Controller() { /* Executes when a test is ready to run. */ loader.onTestReady = function(testObj, testSrc) { presenter.updateStatus("Running Test: " + testObj.id); - elapsed += runner.run(testObj, testSrc); + runner.run(testObj, testSrc); }; /* Executes when there are no more tests to run. */ loader.onTestsExhausted = function() { elapsed = elapsed/(1000*60); //minutes elapsed = elapsed.toFixed(3); - + state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; presenter.setState(state); presenter.finished(elapsed); @@ -574,7 +600,7 @@ function Controller() { controller.implementerHook.finished(elapsed); } catch(e) { /*no-op*/} }; - + /* Start the test execution. */ this.start = function() { elapsed = 0; @@ -594,25 +620,25 @@ function Controller() { loader.onInitialized(); loader.reset(); presenter.reset(); - + state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; presenter.setState(state); }; - + /* Start loading tests. */ this.startLoadingTests = function() { state = 'loading'; presenter.setState(state); loader.startLoadingTests(); } - + /* Set the individual chapter in the laoder and start the controller. */ this.runIndividualTest = function(index) { controller.reset(); loader.setChapter(index); controller.start(); } - + /* Compile a list of the selected tests and start the controller. */ this.runSelected = function() { controller.reset(); @@ -620,15 +646,19 @@ function Controller() { controller.start(); } } - + this.runAll = function() { controller.reset(); controller.start(); } - + this.toggleSelection = function(index) { loader.toggleSelection(index); } + + this.testElapsedTime = function(time){ + elapsed += time; + } } var controller = new Controller(); @@ -698,7 +728,7 @@ $(function () { presenter.setDate(data.date); } }); - + // Start loading the files right away. controller.startLoadingTests(); diff --git a/console/harness/timer.js b/console/harness/timer.js new file mode 100644 index 0000000000..69762d83f5 --- /dev/null +++ b/console/harness/timer.js @@ -0,0 +1,22 @@ +//setTimeout is not available, hence this script was loaded +if(Promise === undefined && this.setTimeout === undefined){ + if(/\$DONE()/.test(code)) + $ERROR("Async test capability is not supported in your test environment"); +} + +if(Promise !== undefined && this.setTimeout === undefined) + (function(that){ + that.setTimeout = function(callback, delay) { + var p = Promise.resolve(); + var start = Date.now(); + var end = start + delay; + function check(){ + var timeLeft = end - Date.now(); + if(timeLeft) + p.then(check); + else + callback(); + } + p.then(check); + } + })(this); \ No newline at end of file diff --git a/test/harness/doneprintHandle.js b/test/harness/doneprintHandle.js new file mode 100644 index 0000000000..317aa9c4ef --- /dev/null +++ b/test/harness/doneprintHandle.js @@ -0,0 +1,10 @@ +function __consolePrintHandle__(msg){ + print(msg); +} + +function $DONE(){ + if(arguments.length === 0) + __consolePrintHandle__('Test262:AsyncTestComplete'); + else + __consolePrintHandle__('Error: ' + arguments[0]); +} \ No newline at end of file diff --git a/test/harness/ed.js b/test/harness/ed.js index 89d0dfc1ca..88e2cecc49 100644 --- a/test/harness/ed.js +++ b/test/harness/ed.js @@ -8,6 +8,7 @@ if (this.window!==undefined) { //for console support this.window.onerror = function(errorMsg, url, lineNumber) { this.window.iframeError = errorMsg; + if(typeof $DONE === 'function') $DONE(); }; } diff --git a/test/harness/gs.js b/test/harness/gs.js index 210c08548f..855cb85bf4 100644 --- a/test/harness/gs.js +++ b/test/harness/gs.js @@ -5,38 +5,60 @@ /// copyright and this notice and otherwise comply with the Use Terms. //Global Scope Test Case Validator +function $DONE() { -//An exception is expected -if (testDescrip.negative !== undefined) { - //TODO - come up with a generic way of catching the error type - //from this.onerror - testDescrip.negative = testDescrip.negative === "NotEarlyError" ? - testDescrip.negative : - (testDescrip.negative === "^((?!NotEarlyError).)*$" ? - testDescrip.negative : "."); - if (this.iframeError === undefined) { //no exception was thrown + //An exception is expected + if (testDescrip.negative !== undefined) { + //TODO - come up with a generic way of catching the error type + //from this.onerror + testDescrip.negative = testDescrip.negative === "NotEarlyError" ? + testDescrip.negative : + (testDescrip.negative === "^((?!NotEarlyError).)*$" ? + testDescrip.negative : "."); + if (this.iframeError === undefined) { //no exception was thrown + testRun(testDescrip.id, + testDescrip.path, + testDescrip.description, + testDescrip.code, + 'fail', + Error('No exception was thrown; expected an error "message"' + + ' property matching the regular expression "' + + testDescrip.negative + '".')); + } else if (!(new RegExp(testDescrip.negative, + "i").test(this.iframeError))) { + //wrong type of exception thrown + testRun(testDescrip.id, + testDescrip.path, + testDescrip.description, + testDescrip.code, + 'fail', + Error('Expected an exception with a "message"' + + ' property matching the regular expression "' + + testDescrip.negative + + '" to be thrown; actual was "' + + this.iframeError + '".')); + } else { + testRun(testDescrip.id, + testDescrip.path, + testDescrip.description, + testDescrip.code, + 'pass', + undefined); + } + } + + //Exception was not expected to be thrown + else if (this.iframeError !== undefined) { testRun(testDescrip.id, testDescrip.path, testDescrip.description, testDescrip.code, 'fail', - Error('No exception was thrown; expected an error "message"' + - ' property matching the regular expression "' + - testDescrip.negative + '".')); - } else if (!(new RegExp(testDescrip.negative, - "i").test(this.iframeError))) { - //wrong type of exception thrown - testRun(testDescrip.id, - testDescrip.path, - testDescrip.description, - testDescrip.code, - 'fail', - Error('Expected an exception with a "message"' + - ' property matching the regular expression "' + - testDescrip.negative + - '" to be thrown; actual was "' + - this.iframeError + '".')); - } else { + Error('Unexpected exception, "' + + this.iframeError + '" was thrown.')); + } + + else { testRun(testDescrip.id, testDescrip.path, testDescrip.description, @@ -44,26 +66,7 @@ if (testDescrip.negative !== undefined) { 'pass', undefined); } -} -//Exception was not expected to be thrown -else if (this.iframeError !== undefined) { - testRun(testDescrip.id, - testDescrip.path, - testDescrip.description, - testDescrip.code, - 'fail', - Error('Unexpected exception, "' + - this.iframeError + '" was thrown.')); -} - -else { - testRun(testDescrip.id, - testDescrip.path, - testDescrip.description, - testDescrip.code, - 'pass', - undefined); -} - -testFinished(); \ No newline at end of file + //teardown + testFinished(); +} \ No newline at end of file diff --git a/test/harness/sth.js b/test/harness/sth.js index c2ce493714..cd05685e18 100644 --- a/test/harness/sth.js +++ b/test/harness/sth.js @@ -1,7 +1,7 @@ -/// Copyright (c) 2012 Ecma International. All rights reserved. +/// Copyright (c) 2012 Ecma International. All rights reserved. /// Ecma International makes this code available under the terms and conditions set -/// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the -/// "Use Terms"). Any redistribution of this code must retain the above +/// forth on http://hg.ecmascript.org/tests/test262/raw-file/tip/LICENSE (the +/// "Use Terms"). Any redistribution of this code must retain the above /// copyright and this notice and otherwise comply with the Use Terms. // Do not cache any JSON files - see @@ -27,6 +27,8 @@ function BrowserRunner() { errorDetectorFileContents, simpleTestAPIContents, globalScopeContents, + timerContents, + startTime, harnessDir = "harness/"; $.ajax({async: false, @@ -43,6 +45,11 @@ function BrowserRunner() { dataType: "text", success: function(data){globalScopeContents = data;}, url:harnessDir+"gs.js"}); + + $.ajax({async: false, + dataType: "text", + success: function(data){timerContents = data;}, + url:harnessDir+"timer.js"}); /* Called by the child window to notify that the test has * finished. This function call is put in a separate script block @@ -70,6 +77,8 @@ function BrowserRunner() { document.body.removeChild(iframe); instance.onComplete(currentTest); + //update elapsed time + controller.testElapsedTime(new Date() - startTime); } /* Called from the child window after the test has run. */ @@ -85,8 +94,8 @@ function BrowserRunner() { /* Run the test. */ this.run = function (test, code) { - var start = new Date(); - + startTime = new Date(); + //--Detect proper window.onerror support if (instance.supportsWindowOnerror===undefined) { var iframePrereqs = document.createElement("iframe"); @@ -96,25 +105,25 @@ function BrowserRunner() { } document.body.appendChild(iframePrereqs); - var iwinPrereqs = iframePrereqs.contentWindow; + var iwinPrereqs = iframePrereqs.contentWindow; var idocPrereqs = iwinPrereqs.document; idocPrereqs.open(); - + iwinPrereqs.failCount = 0; - + var stuff = [ "window.onerror = function(a, b, c) { this.failCount++; }", "va xyz =", "throw Error();" ]; - + for(var i in stuff) { idocPrereqs.writeln(""); } idocPrereqs.close(); - + //TODO - 500ms *should* be a sufficient delay setTimeout(function() { instance.supportsWindowOnerror = iwinPrereqs.failCount === 2; @@ -124,7 +133,7 @@ function BrowserRunner() { }, 500); return 0; // initial config, ignore this timing. } - + currentTest = {}; for (var tempIndex in test) { if (test.hasOwnProperty(tempIndex)) { @@ -132,8 +141,6 @@ function BrowserRunner() { } } currentTest.code = code; - - iframe = document.createElement("iframe"); iframe.setAttribute("id", "runnerIframe"); @@ -188,11 +195,25 @@ function BrowserRunner() { iwin.testDescrip = currentTest; //Add an error handler capable of catching so-called early errors - //idoc.writeln(""); + //idoc.writeln("") idoc.writeln(""); + //Validate the results + //idoc.writeln(""); + idoc.writeln(""); + + //this is mainly applicable for consoles that do not have setTimeout support + //idoc.writeln(""); + if(setTimeout === undefined && /\$DONE()/.test(code)){ + idoc.writeln(""); + } + //Run the code idoc.writeln(""); - - //Validate the results - //idoc.writeln(""); + idoc.writeln(""); idoc.close(); - - var elapsed = new Date() - start; - - return elapsed; }; //--Helper functions------------------------------------------------------- @@ -264,10 +289,10 @@ function TestLoader() { else { presenter.updateStatus("Loading file: " + testGroups[testGroupIndex].path); testGroups[testGroupIndex].onLoaded = getNextXML; - + } } - + /* Get the test list xml */ function loadTestXML() { var testsListLoader = new XMLHttpRequest(); @@ -288,7 +313,7 @@ function TestLoader() { onLoaded: function(){} }; presenter.setTestWaiting(i, testSuite[i]); - + var tr = $('#chapterSelector table tr').filter(':nth-child(' + (i+1) + ')'); tr.find('img').filter('[alt="Run"]').bind('click', {index:i}, function(event){ controller.runIndividualTest(event.data.index); @@ -298,25 +323,25 @@ function TestLoader() { getFile(); }}); } - + /* Get the test file. Handles all the logic of figuring out the next file to load. */ function getFile(index) { index = (arguments.length == 0) ? -1 : index; - + // Look for selected waiting chapters (priority because you'll probably want to run them soon) for(var i = 0; index == -1 && i < testGroups.length; i++) { if(testGroups[i].status == 'waiting' && testGroups[i].selected) { index = i; } } - + // Look for just chapters waiting to be loaded. for(var i = 0; index == -1 && i < testGroups.length; i++) { if(testGroups[i].status == 'waiting') { index = i; } } - + if(index == -1) { // Still -1? No more tests are waiting to be loaded then. if(controller.state == 'loading') { @@ -324,13 +349,13 @@ function TestLoader() { } return; } - + presenter.setTestLoading(index, testGroups[index].path); // the only other status that should be set when we get here is 'priorityloading' if(testGroups[index].status == 'waiting') { testGroups[index].status = 'loading'; } - + loader.onTestStartLoading(index, testGroups[index].path); // Create the AJAX call to grab the file. $.ajax({ @@ -344,11 +369,11 @@ function TestLoader() { }, error: function(xhr, textStatus, errorThrown) { // TODO: Catch this error and update UI accordingly. Unlikely to happen, but errors would be 404 or 5-- errors. - + } }); } - + /* Executes when a test file finishes loading. */ function onTestLoaded(index, name, numTests) { presenter.setTestLoaded(index, name, numTests); @@ -357,7 +382,7 @@ function TestLoader() { loader.runningTests += numTests; loader.onInitialized( loader.runningTests ); } - + // The loading status is only assigned when downloading files in sequence, otherwise it // gets the status of priorityloading. When loading out of order, we only want to download // the single file, so we'll only tell it to get the next file when we see a status of @@ -371,10 +396,10 @@ function TestLoader() { testGroups[index].status = 'loaded'; loader.setChapter(index); } - + testGroups[index].onLoaded(); }; - + function getIdFromPath (path) { //path is of the form "a/b/c.js" @@ -420,7 +445,7 @@ function TestLoader() { return; } } - // And if + // And if else { // We don't have tests left in this test group, so move on // to the next. @@ -428,7 +453,7 @@ function TestLoader() { } getNextXML(); } - // + // else { // We're done. loader.onTestsExhausted(); @@ -446,18 +471,18 @@ function TestLoader() { currentTestIndex = 0; testGroupIndex = 0; }; - + /* Begin downloading test files. */ this.startLoadingTests = function() { loadTestXML(); }; - + /* Prepare for testing a single chapter. */ this.setChapter = function(index) { currentTestIndex = 0; testGroupIndex = index; mode = "one"; - + if(testGroups[index].status == 'loaded') { loader.onInitialized(testGroups[index].tests.length); } @@ -467,7 +492,7 @@ function TestLoader() { loader.onInitialized(0); } }; - + /* Prepare for testing multiple chapters. Returns true if at least one chapter was selected. */ this.setMultiple = function() { // Find the index of the first selection @@ -481,16 +506,16 @@ function TestLoader() { if(firstSelectedIndex == -1) { return false; } - + // Begin loading the file immediately, if necessary if(testGroups[firstSelectedIndex].status == 'waiting') { getFile(firstSelectedIndex); } - + mode = "multiple"; testGroupIndex = firstSelectedIndex; // start at this chapter currentTestIndex = 0; // start at test 0 - + // Count the number of tests runningTests = 0; for(var i = 0; i < testGroups.length; i++) { @@ -499,16 +524,16 @@ function TestLoader() { loader.onInitialized(runningTests); return true; }; - + this.getNumTestFiles = function() { return testGroups.length; }; - + /* Toggle the selection of a file. */ this.toggleSelection = function(index) { testGroups[index].selected = !testGroups[index].selected; } - + } /* Controls test generation and running, and sends results to the presenter. */ @@ -523,8 +548,8 @@ function Controller() { //into this test framework to handle test case failures and passes in their //own way (e.g., logging failures to the filesystem) this.implementerHook = { - //Adds a test result - addTestResult: function (test) { }, + //Adds a test result + addTestResult: function (test) { }, //Called whenever all tests have finished running. Provided with the //elapsed time in milliseconds. @@ -550,7 +575,7 @@ function Controller() { } presenter.setTotalTests(totalTests); }; - + /* Executes when a test file starts loading. */ loader.onTestStartLoading = function(index, path) { presenter.setTestLoading(index, path); @@ -559,14 +584,14 @@ function Controller() { /* Executes when a test is ready to run. */ loader.onTestReady = function(testObj, testSrc) { presenter.updateStatus("Running Test: " + testObj.id); - elapsed += runner.run(testObj, testSrc); + runner.run(testObj, testSrc); }; /* Executes when there are no more tests to run. */ loader.onTestsExhausted = function() { elapsed = elapsed/(1000*60); //minutes elapsed = elapsed.toFixed(3); - + state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; presenter.setState(state); presenter.finished(elapsed); @@ -574,7 +599,7 @@ function Controller() { controller.implementerHook.finished(elapsed); } catch(e) { /*no-op*/} }; - + /* Start the test execution. */ this.start = function() { elapsed = 0; @@ -594,25 +619,25 @@ function Controller() { loader.onInitialized(); loader.reset(); presenter.reset(); - + state = (loader.loadedFiles == loader.getNumTestFiles()) ? 'loaded' : 'loading'; presenter.setState(state); }; - + /* Start loading tests. */ this.startLoadingTests = function() { state = 'loading'; presenter.setState(state); loader.startLoadingTests(); } - + /* Set the individual chapter in the laoder and start the controller. */ this.runIndividualTest = function(index) { controller.reset(); loader.setChapter(index); controller.start(); } - + /* Compile a list of the selected tests and start the controller. */ this.runSelected = function() { controller.reset(); @@ -620,15 +645,19 @@ function Controller() { controller.start(); } } - + this.runAll = function() { controller.reset(); controller.start(); } - + this.toggleSelection = function(index) { loader.toggleSelection(index); } + + this.testElapsedTime = function(time){ + elapsed += time; + } } var controller = new Controller(); @@ -698,7 +727,7 @@ $(function () { presenter.setDate(data.date); } }); - + // Start loading the files right away. controller.startLoadingTests(); diff --git a/test/harness/timer.js b/test/harness/timer.js new file mode 100644 index 0000000000..69762d83f5 --- /dev/null +++ b/test/harness/timer.js @@ -0,0 +1,22 @@ +//setTimeout is not available, hence this script was loaded +if(Promise === undefined && this.setTimeout === undefined){ + if(/\$DONE()/.test(code)) + $ERROR("Async test capability is not supported in your test environment"); +} + +if(Promise !== undefined && this.setTimeout === undefined) + (function(that){ + that.setTimeout = function(callback, delay) { + var p = Promise.resolve(); + var start = Date.now(); + var end = start + delay; + function check(){ + var timeLeft = end - Date.now(); + if(timeLeft) + p.then(check); + else + callback(); + } + p.then(check); + } + })(this); \ No newline at end of file diff --git a/tools/packaging/test262.py b/tools/packaging/test262.py index f9c1fe628c..b2d4397387 100755 --- a/tools/packaging/test262.py +++ b/tools/packaging/test262.py @@ -73,6 +73,7 @@ def BuildOptions(): result.add_option("--junitname", help="Filename to save test results in JUnit XML format") result.add_option("--loglevel", default="warning", help="sets log level to debug, info, warning, error, or critical") + result.add_option("--print-handle", default="", help="Command to print from console") return result @@ -193,15 +194,19 @@ class TestResult(object): testpackage = testclass return(testpackage,testclass,testcase) - def HasFailed(self): + def HasFailed(self): return self.exit_code != 0 - def HasUnexpectedOutcome(self): - if self.case.IsNegative(): - return not self.HasFailed() - else: - return self.HasFailed() + def AsyncHasFailed(self): + return 'Test262:AsyncTestComplete' not in self.stdout + def HasUnexpectedOutcome(self): + if self.case.IsAsyncTest(): + return self.AsyncHasFailed() or self.HasFailed() + elif self.case.IsNegative(): + return not self.HasFailed() + else: + return self.HasFailed() class TestCase(object): @@ -242,6 +247,9 @@ class TestCase(object): def IsNoStrict(self): return 'noStrict' in self.testRecord + def IsAsyncTest(self): + return '$DONE' in self.test + def GetSource(self): # "var testDescrip = " + str(self.testRecord) + ';\n\n' + \ source = self.suite.GetInclude("cth.js") + \ @@ -249,6 +257,8 @@ class TestCase(object): self.suite.GetInclude("ed.js") + \ self.suite.GetInclude("testBuiltInObject.js") + \ self.suite.GetInclude("testIntl.js") + \ + self.suite.GetInclude("timer.js") + \ + self.suite.GetInclude("doneprintHandle.js").replace('print', self.suite.print_handle) + \ self.test + '\n' if self.strict_mode: @@ -332,14 +342,16 @@ def MakePlural(n): class TestSuite(object): - def __init__(self, root, strict_only, non_strict_only, unmarked_default): + def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle): # TODO: derive from packagerConfig.py self.test_root = path.join(root, 'test', 'suite') self.lib_root = path.join(root, 'test', 'harness') self.strict_only = strict_only self.non_strict_only = non_strict_only self.unmarked_default = unmarked_default + self.print_handle = print_handle self.include_cache = { } + def Validate(self): if not path.exists(self.test_root): @@ -536,7 +548,8 @@ def Main(): test_suite = TestSuite(options.tests, options.strict_only, options.non_strict_only, - options.unmarked_default) + options.unmarked_default, + options.print_handle) test_suite.Validate() if options.loglevel == 'debug': logging.basicConfig(level=logging.DEBUG)