diff --git a/.travis.yml b/.travis.yml index 740286f10f..fce8a669ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,10 @@ jobs: include: - stage: Test name: AVA Regression Tests - script: ava --tap -c 1 test/tests + script: ava -c 1 test/tests + - stage: Test + name: Regression Tests Coverage Report + script: node test/util/report.js - stage: Test name: JS Linting script: npm run lint diff --git a/examples/alert/alert.html b/examples/alert/alert.html index 9e5298f79f..f061fefc87 100644 --- a/examples/alert/alert.html +++ b/examples/alert/alert.html @@ -85,7 +85,7 @@

Role, Property, State, and Tabindex Attributes

div Identifies the element as the container where alert content will be added or updated. - + aria-live=assertive Implicit on div @@ -96,7 +96,7 @@

Role, Property, State, and Tabindex Attributes

- + aria-atomic=true Implicit on div diff --git a/examples/checkbox/checkbox-1/checkbox-1.html b/examples/checkbox/checkbox-1/checkbox-1.html index 77d58982a4..874ef8758a 100644 --- a/examples/checkbox/checkbox-1/checkbox-1.html +++ b/examples/checkbox/checkbox-1/checkbox-1.html @@ -100,7 +100,7 @@

Role, Property, State, and Tabindex Attributes

- + h3 diff --git a/examples/combobox/aria1.0pattern/combobox-autocomplete-both.html b/examples/combobox/aria1.0pattern/combobox-autocomplete-both.html index d72be96761..6e362f56b2 100644 --- a/examples/combobox/aria1.0pattern/combobox-autocomplete-both.html +++ b/examples/combobox/aria1.0pattern/combobox-autocomplete-both.html @@ -181,7 +181,7 @@

Textbox

- + Standard single line text editing keys - + Standard single line text editing keys - + Standard single line text editing keys - + Standard single line text editing keys - + Standard single line text editing keys - + radio li @@ -256,7 +256,7 @@

Role, Property, State, and Tabindex Attributes

- + aria-checked="false" li @@ -268,7 +268,7 @@

Role, Property, State, and Tabindex Attributes

- + aria-checked="true" li diff --git a/package-lock.json b/package-lock.json index 3e5d91ba54..5f5b21a89a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -126,6 +126,11 @@ } } }, + "@types/node": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.11.0.tgz", + "integrity": "sha512-R4Dvw6KjSYn/SpvjRchBOwXr14vVVcFXCtnM3f0aLvlJS8a599rrcEoihcP2/+Z/f75E5GNPd4aWM7j1yei9og==" + }, "acorn": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.5.3.tgz", @@ -934,6 +939,11 @@ "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==", "dev": true }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -1121,6 +1131,19 @@ "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", + "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chokidar": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.7.0.tgz", @@ -1366,8 +1389,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "create-error-class": { "version": "3.0.2", @@ -1395,6 +1417,22 @@ "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=", "dev": true }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz", + "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0=" + }, "currently-unhandled": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", @@ -1476,6 +1514,44 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "requires": { + "domelementtype": "~1.1.1", + "entities": "~1.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" + } + } + }, + "domelementtype": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -1510,6 +1586,11 @@ "core-js": "^2.0.0" } }, + "entities": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", + "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" + }, "equal-length": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/equal-length/-/equal-length-1.0.1.tgz", @@ -2601,6 +2682,19 @@ "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", "dev": true }, + "htmlparser2": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", + "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=", + "requires": { + "domelementtype": "^1.3.0", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^2.0.2" + } + }, "https-proxy-agent": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", @@ -2709,8 +2803,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { "version": "1.3.5", @@ -2984,8 +3077,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isexe": { "version": "2.0.0", @@ -3178,8 +3270,7 @@ "lodash": { "version": "4.17.5", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", - "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", - "dev": true + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==" }, "lodash.clonedeep": { "version": "4.5.0", @@ -3573,6 +3664,14 @@ "path-key": "^2.0.0" } }, + "nth-check": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz", + "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=", + "requires": { + "boolbase": "~1.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", @@ -3761,6 +3860,14 @@ "integrity": "sha1-3T+iXtbC78e93hKtm0bBY6opIk4=", "dev": true }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -3923,8 +4030,7 @@ "process-nextick-args": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", - "dev": true + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, "progress": { "version": "2.0.0", @@ -4018,7 +4124,6 @@ "version": "2.3.6", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -4255,8 +4360,7 @@ "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==", - "dev": true + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, "safer-buffer": { "version": "2.1.2", @@ -4453,7 +4557,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -4724,8 +4827,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "validate-npm-package-license": { "version": "3.0.4", diff --git a/package.json b/package.json index 529e627902..142b2d939c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "fix": "eslint --fix .", "lint": "eslint .", "regression": "ava test/tests", + "regression-report": "node test/util/report", "test": "npm run lint && npm run vnu-jar && npm run regression", "vnu-jar": "java -jar node_modules/vnu-jar/build/dist/vnu.jar --errors-only --filterfile .vnurc --no-langdetect --skip-non-html *.html examples/" }, @@ -25,6 +26,7 @@ "homepage": "https://github.com/w3c/aria-practices#readme", "devDependencies": { "ava": "^0.25.0", + "cheerio": "^1.0.0-rc.2", "eslint": "^4.19.1", "geckodriver": "^1.11.0", "selenium-webdriver": "^4.0.0-alpha.1", diff --git a/test/index.js b/test/index.js index 9d08c9caf7..03c6568dc6 100644 --- a/test/index.js +++ b/test/index.js @@ -10,30 +10,33 @@ const startGeckodriver = require('./util/start-geckodriver'); let session, geckodriver; const firefoxArgs = process.env.CI ? [ '-headless' ] : []; const testWaitTime = parseInt(process.env.TEST_WAIT_TIME) || 500; +const coverageReportRun = process.env.REGRESSION_COVERAGE_REPORT; -test.before(async (t) => { - geckodriver = await startGeckodriver(1022, 12 * 1000); - session = new webdriver.Builder() - .usingServer('http://localhost:' + geckodriver.port) - .withCapabilities({ - 'moz:firefoxOptions': { - args: firefoxArgs - } - }) - .forBrowser('firefox') - .build(); - await session; -}); +if (!coverageReportRun) { + test.before(async (t) => { + geckodriver = await startGeckodriver(1022, 12 * 1000); + session = new webdriver.Builder() + .usingServer('http://localhost:' + geckodriver.port) + .withCapabilities({ + 'moz:firefoxOptions': { + args: firefoxArgs + } + }) + .forBrowser('firefox') + .build(); + await session; + }); -test.beforeEach((t) => { - t.context.session = session; - t.context.waitTime = testWaitTime; -}); + test.beforeEach((t) => { + t.context.session = session; + t.context.waitTime = testWaitTime; + }); -test.after.always(() => { - return Promise.resolve(session && session.close()) - .then(() => geckodriver && geckodriver.stop()); -}); + test.after.always(() => { + return Promise.resolve(session && session.close()) + .then(() => geckodriver && geckodriver.stop()); + }); +} /** * Declare a test for a behavior documented on and demonstrated by an @@ -66,6 +69,13 @@ const _ariaTest = (desc, page, testId, body, failing) => { const testName = page + ' ' + selector + ': ' + desc; + if (coverageReportRun) { + test(testName, async function (t) { + t.fail('All tests expect to fail. Running in coverage mode.'); + }); + return; + } + let runTest = failing ? test.failing : test.serial; runTest(testName, async function (t) { t.context.url = url; diff --git a/test/tests/checkbox-1.js b/test/tests/checkbox-1.js index e583bc4ffd..ba0985ce6c 100644 --- a/test/tests/checkbox-1.js +++ b/test/tests/checkbox-1.js @@ -55,6 +55,24 @@ const uncheckAllSelectedByDefault = async function (t) { // Attributes +ariaTest('element h3 exists', exampleFile, 'h3', async (t) => { + t.plan(2); + + let header = await t.context.session.findElements(By.css('#ex1 h3')); + + t.is( + header.length, + 1, + 'One h3 element exist within the example to label the checkboxes' + ); + + t.truthy( + await header[0].getText(), + 'One h3 element exist with readable content within the example to label the checkboxes' + ); + +}); + ariaTest('role="group" element exists', exampleFile, 'group-role', async (t) => { t.plan(1); await assertAriaRoles(t, 'ex1', 'group', '1', 'div'); diff --git a/test/tests/checkbox-2.js b/test/tests/checkbox-2.js index d19eb6f541..6d9d131255 100644 --- a/test/tests/checkbox-2.js +++ b/test/tests/checkbox-2.js @@ -36,7 +36,7 @@ ariaTest('role="checkbox" element exists', exampleFile, 'checkbox-role', async ( await assertAriaRoles(t, 'ex1', 'checkbox', '1', 'div'); }); -ariaTest('"tabindex" on checkbox element', exampleFile, 'checkbox-aria-controls', async (t) => { +ariaTest('"tabindex" on checkbox element', exampleFile, 'checkbox-tabindex', async (t) => { t.plan(1); await assertAttributeValues(t, ex.checkboxSelector, 'tabindex', '0'); }); diff --git a/test/tests/combobox-autocomplete-both.js b/test/tests/combo-10-autocomplete-both.js similarity index 92% rename from test/tests/combobox-autocomplete-both.js rename to test/tests/combo-10-autocomplete-both.js index ae16477d9d..40007397f6 100644 --- a/test/tests/combobox-autocomplete-both.js +++ b/test/tests/combo-10-autocomplete-both.js @@ -166,7 +166,7 @@ ariaTest('Test down key press with focus on textbox', .sendKeys(Key.ARROW_DOWN); // Check that the listbox is displayed - t.truthy( + t.true( await t.context.session.findElement(By.css(ex.listboxSelector)).isDisplayed(), 'In example ex3 listbox should display after ARROW_DOWN keypress' ); @@ -219,7 +219,7 @@ ariaTest('Test up key press with focus on textbox', .sendKeys(Key.ARROW_UP); // Check that the listbox is displayed - t.truthy( + t.true( await t.context.session.findElement(By.css(ex.listboxSelector)).isDisplayed(), 'In example ex3 listbox should display after ARROW_UP keypress' ); @@ -297,6 +297,43 @@ ariaTest('Test enter key press with focus on textbox', firstOption, 'key press "ENTER" should result in first option in textbox' ); + }); + +ariaTest('Test enter key press with focus on listbox', + exampleFile, 'listbox-key-enter', async (t) => { + + t.plan(2); + + // Send key "a" to the textbox, then two ARROW_DOWNS + + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .sendKeys('a', Key.ARROW_DOWN); + + // Get the value of the second option in the listbox + + const secondOption = await(await t.context.session.findElements(By.css(ex.optionsSelector)))[1] + .getText(); + + // Send key ENTER + + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .sendKeys(Key.ENTER); + + // Confirm that the listbox is still open + + await assertAttributeValues(t, ex.textboxSelector, 'aria-expanded', 'false'); + + // Confirm that the value of the textbox is now set to the second option + + t.is( + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .getAttribute('value'), + secondOption, + 'key press "ENTER" should result in second option in textbox' + ); }); @@ -469,3 +506,9 @@ ariaTest('Sending character keys while focus is on listbox moves focus', }); +ariaTest.failing('Expected behavior for all other standard single line editing keys', + exampleFile, 'standard-single-line-editing-keys', async (t) => { + t.plan(1); + t.fail(); + }); + diff --git a/test/tests/combobox-autocomplete-list.js b/test/tests/combo-10-autocomplete-list.js similarity index 93% rename from test/tests/combobox-autocomplete-list.js rename to test/tests/combo-10-autocomplete-list.js index b0e9147989..11741f4231 100644 --- a/test/tests/combobox-autocomplete-list.js +++ b/test/tests/combo-10-autocomplete-list.js @@ -173,7 +173,7 @@ ariaTest('Test down key press with focus on textbox', .sendKeys(Key.ARROW_DOWN); // Check that the listbox is displayed - t.truthy( + t.true( await t.context.session.findElement(By.css(ex.listboxSelector)).isDisplayed(), 'In example ex3 listbox should display after ARROW_DOWN keypress' ); @@ -239,7 +239,7 @@ ariaTest('Test up key press with focus on textbox', .sendKeys(Key.ARROW_UP); // Check that the listbox is displayed - t.truthy( + t.true( await t.context.session.findElement(By.css(ex.listboxSelector)).isDisplayed(), 'In example ex3 listbox should display after ARROW_UP keypress' ); @@ -279,6 +279,39 @@ ariaTest('Test up key press with focus on listbox', } }); +ariaTest('Test enter key press with focus on textbox', + exampleFile, 'textbox-key-enter', async (t) => { + + t.plan(2); + + // Send key "a" to the textbox, then key ARROW_DOWN to select the first item + + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .sendKeys('a'); + + // Send key ENTER + + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .sendKeys(Key.ENTER); + + // Confirm that the listbox is closed + + await assertAttributeValues(t, ex.textboxSelector, 'aria-expanded', 'false'); + + // Confirm that the value of the textbox is the same as the characters set to the listbox + + t.is( + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .getAttribute('value'), + 'a', + 'key press "ENTER" should not result in selecting an option' + ); + + }); + ariaTest('Test enter key press with focus on listbox', exampleFile, 'listbox-key-enter', async (t) => { @@ -300,7 +333,7 @@ ariaTest('Test enter key press with focus on listbox', .findElement(By.css(ex.textboxSelector)) .sendKeys(Key.ENTER); - // Confirm that the listbox is note + // Confirm that the listbox is closed await assertAttributeValues(t, ex.textboxSelector, 'aria-expanded', 'false'); @@ -482,3 +515,9 @@ ariaTest('Sending character keys while focus is on listbox moves focus', }); +ariaTest.failing('Expected behavior for all other standard single line editing keys', + exampleFile, 'standard-single-line-editing-keys', async (t) => { + t.plan(1); + t.fail(); + }); + diff --git a/test/tests/combobox-autocomplete-none.js b/test/tests/combo-10-autocomplete-none.js similarity index 93% rename from test/tests/combobox-autocomplete-none.js rename to test/tests/combo-10-autocomplete-none.js index f35978132a..61d912d11e 100644 --- a/test/tests/combobox-autocomplete-none.js +++ b/test/tests/combo-10-autocomplete-none.js @@ -173,7 +173,7 @@ ariaTest('Test down key press with focus on textbox', .sendKeys(Key.ARROW_DOWN); // Check that the listbox is displayed - t.truthy( + t.true( await t.context.session.findElement(By.css(ex.listboxSelector)).isDisplayed(), 'In example ex3 listbox should display after ARROW_DOWN keypress' ); @@ -239,7 +239,7 @@ ariaTest('Test up key press with focus on textbox', .sendKeys(Key.ARROW_UP); // Check that the listbox is displayed - t.truthy( + t.true( await t.context.session.findElement(By.css(ex.listboxSelector)).isDisplayed(), 'In example ex3 listbox should display after ARROW_UP keypress' ); @@ -279,6 +279,38 @@ ariaTest('Test up key press with focus on listbox', } }); +ariaTest('Test enter key press with focus on textbox', + exampleFile, 'textbox-key-enter', async (t) => { + + t.plan(2); + + // Send key "a" to the textbox, then key ARROW_DOWN to select the first item + + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .sendKeys('a'); + + // Send key ENTER + + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .sendKeys(Key.ENTER); + + // Confirm that the listbox is closed + + await assertAttributeValues(t, ex.textboxSelector, 'aria-expanded', 'false'); + + // Confirm that the value of the textbox is the same as the characters set to the listbox + + t.is( + await t.context.session + .findElement(By.css(ex.textboxSelector)) + .getAttribute('value'), + 'a', + 'key press "ENTER" should not result in selecting an option' + ); + }); + ariaTest('Test enter key press with focus on listbox', exampleFile, 'listbox-key-enter', async (t) => { @@ -460,3 +492,9 @@ ariaTest('Sending character keys while focus is on listbox moves focus', }); +ariaTest.failing('Expected behavior for all other standard single line editing keys', + exampleFile, 'standard-single-line-editing-keys', async (t) => { + t.plan(1); + t.fail(); + }); + diff --git a/test/tests/grid-combo.js b/test/tests/combo-11-grid.js similarity index 99% rename from test/tests/grid-combo.js rename to test/tests/combo-11-grid.js index cfa12d97b6..7379bba394 100644 --- a/test/tests/grid-combo.js +++ b/test/tests/combo-11-grid.js @@ -778,3 +778,9 @@ ariaTest('Sending character keys while focus is on grid moves focus', }); +ariaTest.failing('Expected behavior for all other standard single line editing keys', + exampleFile, 'standard-single-line-editing-keys', async (t) => { + t.plan(1); + t.fail(); + }); + diff --git a/test/tests/listbox-combo.js b/test/tests/combo-11-listbox.js similarity index 95% rename from test/tests/listbox-combo.js rename to test/tests/combo-11-listbox.js index ef55db2994..17868f17fc 100644 --- a/test/tests/listbox-combo.js +++ b/test/tests/combo-11-listbox.js @@ -842,6 +842,7 @@ ariaTest('Test enter key press with focus on textbox', ariaTest('Test escape key press with focus on textbox', exampleFile, 'textbox-key-escape', async (t) => { + t.plan(6); for (let exId in pageExamples) { let ex = pageExamples[exId]; @@ -868,6 +869,7 @@ ariaTest('Test escape key press with focus on textbox', ariaTest('Test escape key press with focus on textbox', exampleFile, 'listbox-key-escape', async (t) => { + t.plan(6); for (let exId in pageExamples) { let ex = pageExamples[exId]; @@ -895,12 +897,12 @@ ariaTest('Test escape key press with focus on textbox', ariaTest('left arrow from focus on list puts focus on listbox and moves cursor right', exampleFile, 'listbox-key-left-arrow', async (t) => { + t.plan(6); for (let exId in pageExamples) { let ex = pageExamples[exId]; - // Send key "a" then key "ARROW_DOWN" to put the focus on the listbox, - // then key ESCAPE to the textbox + // Send key "a" then key "ARROW_DOWN" to put the focus on the listbox const textbox = t.context.session.findElement(By.css(ex.textboxSelector)); await textbox.sendKeys('a', Key.ARROW_DOWN); @@ -923,16 +925,16 @@ ariaTest('left arrow from focus on list puts focus on listbox and moves cursor r ariaTest('Right arrow from focus on list puts focus on listbox', exampleFile, 'listbox-key-right-arrow', async (t) => { + t.plan(6); for (let exId in pageExamples) { let ex = pageExamples[exId]; // Send key "a" then key "ARROW_DOWN" to put the focus on the listbox, - // then key ESCAPE to the textbox const textbox = t.context.session.findElement(By.css(ex.textboxSelector)); await textbox.sendKeys('a', Key.ARROW_DOWN); - // Send key "RIGHT_ARROW" + // Send key "ARROW_RIGHT" await textbox.sendKeys(Key.ARROW_RIGHT); t.true( @@ -950,16 +952,16 @@ ariaTest('Right arrow from focus on list puts focus on listbox', ariaTest('Home arrow from focus on list puts focus on listbox', exampleFile, 'listbox-key-home', async (t) => { + t.plan(6); for (let exId in pageExamples) { let ex = pageExamples[exId]; // Send key "a" then key "ARROW_DOWN" to put the focus on the listbox, - // then key ESCAPE to the textbox const textbox = t.context.session.findElement(By.css(ex.textboxSelector)); await textbox.sendKeys('a', Key.ARROW_DOWN); - // Send key "ARROW_HOME" + // Send key "HOME" await textbox.sendKeys(Key.HOME); t.true( @@ -970,23 +972,23 @@ ariaTest('Home arrow from focus on list puts focus on listbox', t.is( await textbox.getAttribute('aria-activedescendant'), '', - 'Focus should be on the textbox after one ARROW_HOME key', + 'Focus should be on the textbox after one HOME key', ); } }); ariaTest('End arrow from focus on list puts focus on listbox', exampleFile, 'listbox-key-end', async (t) => { + t.plan(6); for (let exId in pageExamples) { let ex = pageExamples[exId]; - // Send key "a" then key "ARROW_DOWN" to put the focus on the listbox, - // then key ESCAPE to the textbox + // Send key "a" then key "ARROW_DOWN" to put the focus on the listbox const textbox = t.context.session.findElement(By.css(ex.textboxSelector)); await textbox.sendKeys('a', Key.ARROW_DOWN); - // Send key "END_ARROW" + // Send key "END" await textbox.sendKeys(Key.END); t.true( @@ -997,8 +999,42 @@ ariaTest('End arrow from focus on list puts focus on listbox', t.is( await textbox.getAttribute('aria-activedescendant'), '', - 'Focus should be on the textbox after on ARROW_END key', + 'Focus should be on the textbox after one END key', + ); + } + }); + +ariaTest('Typing characters while focus is on list puts focus on listbox', + exampleFile, 'listbox-characters', async (t) => { + t.plan(6); + + for (let exId in pageExamples) { + let ex = pageExamples[exId]; + + // Send key "z" then key "ARROW_DOWN" to put the focus on the listbox + const textbox = t.context.session.findElement(By.css(ex.textboxSelector)); + await textbox.sendKeys('Z', Key.ARROW_DOWN); + + // Then send "z" again + await textbox.sendKeys('Z'); + + t.is( + await textbox.getAttribute('value'), + 'ZZ', + 'Focus should be on the textbox after sending a character key', + ); + + t.is( + await textbox.getAttribute('aria-activedescendant'), + '', + 'Focus should be on the textbox after sending a character key', ); } }); +ariaTest.failing('Expected behavior for all other standard single line editing keys', + exampleFile, 'standard-single-line-editing-keys', async (t) => { + t.plan(1); + t.fail(); + }); + diff --git a/test/tests/dataGrids.js b/test/tests/dataGrids.js index 6482c929f7..ed7b4b4fd4 100644 --- a/test/tests/dataGrids.js +++ b/test/tests/dataGrids.js @@ -36,9 +36,6 @@ const ex = { }; const checkFocusOnOrInCell = async function (t, gridSelector, rowIndex, columnIndex) { - t.log('check focus: ' + gridSelector + ' tr:nth-of-type(' + rowIndex + - ') td:nth-of-type(' + columnIndex + ')'); - return t.context.session.executeScript(function () { const [gridSelector, rowIndex, columnIndex] = arguments; @@ -69,9 +66,6 @@ const sendKeyToGridcell = async function (t, gridSelector, rowIndex, columnIndex let selector = gridSelector + ' tr:nth-of-type(' + rowIndex + ') td:nth-of-type(' + columnIndex + ')'; - t.log(selector); - t.log(await t.context.session.findElement(By.css(selector)).isDisplayed()); - // If the element has "tabindex", send KEY here const cellElement = await t.context.session.findElement(By.css(selector)); if (await cellElement.getAttribute('tabindex') !== null) { diff --git a/test/tests/feed.js b/test/tests/feed.js index fc8ffe523b..1a62a05a90 100644 --- a/test/tests/feed.js +++ b/test/tests/feed.js @@ -117,7 +117,7 @@ ariaTest('aria-describedby set on article elements', exampleFile, 'article-descr await assertAriaDescribedby(t, ex.articleSelector); }); -ariaTest('', exampleFile, 'article-aria-posinset', async (t) => { +ariaTest('aria-posinset on article element', exampleFile, 'article-aria-posinset', async (t) => { t.plan(30); await navigateToFeed(t); await waitForArticlesToLoad(t); @@ -144,7 +144,7 @@ ariaTest('', exampleFile, 'article-aria-posinset', async (t) => { } }); -ariaTest('', exampleFile, 'article-aria-setsize', async (t) => { +ariaTest('aria-setsize on article element', exampleFile, 'article-aria-setsize', async (t) => { t.plan(30); await navigateToFeed(t); @@ -222,7 +222,7 @@ ariaTest('CONTROL+END moves focus out of feed', exampleFile, 'key-control-end', }); // This bug has been reported in issue: https://github.com/w3c/aria-practices/issues/911 -ariaTest.failing('', exampleFile, 'key-control-home', async (t) => { +ariaTest.failing('key home moves focus out of feed', exampleFile, 'key-control-home', async (t) => { t.fail(); }); diff --git a/test/tests/listbox-rearrangeable.js b/test/tests/listbox-rearrangeable.js new file mode 100644 index 0000000000..51c8008ff2 --- /dev/null +++ b/test/tests/listbox-rearrangeable.js @@ -0,0 +1,372 @@ +'use strict'; + +const { ariaTest } = require('..'); +const { By, Key } = require('selenium-webdriver'); +const assertAttributeValues = require('../util/assertAttributeValues'); +const assertAriaLabelledby = require('../util/assertAriaLabelledby'); +const assertAriaRoles = require('../util/assertAriaRoles'); +const assertAriaSelectedAndActivedescendant = require('../util/assertAriaSelectedAndActivedescendant'); +const assertAriaActivedescendant = require('../util/assertAriaActivedescendant'); +const assertAttributeDNE = require('../util/assertAttributeDNE'); + +const exampleFile = 'listbox/listbox-rearrangeable.html'; + +const ex = { + 1: { + listboxSelector: '#ex1 [role="listbox"]', + importantSelector: '#ex1 [role="listbox"]#ss_imp_list', + optionSelector: '#ex1 [role="option"]', + numOptions: 10, + firstOptionSelector: '#ex1 #ss_opt1', + lastOptionSelector: '#ex1 #ss_opt10' + }, + 2: { + listboxSelector: '#ex2 [role="listbox"]', + availableSelector: '#ex2 [role="listbox"]#ms_imp_list', + optionSelector: '#ex2 [role="option"]', + numOptions: 10, + firstOptionSelector: '#ex2 #ms_opt1', + lastOptionSelector: '#ex2 #ms_opt10' + } +}; + +// Attributes + +ariaTest('role="listbox" on ul element', exampleFile, 'listbox-role', async (t) => { + t.plan(2); + await assertAriaRoles(t, 'ex1', 'listbox', 2, 'ul'); + await assertAriaRoles(t, 'ex2', 'listbox', 2, 'ul'); +}); + +ariaTest('"aria-labelledby" on listbox element', exampleFile, 'listbox-aria-labelledby', async (t) => { + t.plan(2); + await assertAriaLabelledby(t, ex[1].listboxSelector); + await assertAriaLabelledby(t, ex[2].listboxSelector); +}); + +ariaTest('tabindex="0" on listbox element', exampleFile, 'listbox-tabindex', async (t) => { + t.plan(2); + await assertAttributeValues(t, ex[1].listboxSelector, 'tabindex', '0'); + await assertAttributeValues(t, ex[2].listboxSelector, 'tabindex', '0'); +}); + +ariaTest('aria-multiselectable on listbox element', exampleFile, 'listbox-aria-multiselectable', async (t) => { + t.plan(1); + await assertAttributeValues(t, ex[2].listboxSelector, 'aria-multiselectable', 'true'); +}); + + +ariaTest('aria-activedescendant on listbox element', exampleFile, 'listbox-aria-activedescendant', async (t) => { + t.plan(2); + + // Put the focus on the listbox. In this example, focusing on the listbox + // will automatically select the first option. + await t.context.session.findElement(By.css(ex[1].firstOptionSelector)).click(); + + let options = await t.context.session.findElements(By.css(ex[1].optionSelector)); + let optionId = await options[0].getAttribute('id'); + + t.is( + await t.context.session + .findElement(By.css(ex[1].listboxSelector)) + .getAttribute('aria-activedescendant'), + optionId, + 'aria-activedescendant should be set to ' + optionId + ' for items: ' + ex.listboxSelector + ); + + // Put the focus on the listbox. In this example, focusing on the listbox + // will automatically select the first option. + await t.context.session.findElement(By.css(ex[2].firstOptionSelector)).click(); + + options = await t.context.session.findElements(By.css(ex[2].optionSelector)); + optionId = await options[0].getAttribute('id'); + + t.is( + await t.context.session + .findElement(By.css(ex[2].listboxSelector)) + .getAttribute('aria-activedescendant'), + optionId, + 'aria-activedescendant should be set to ' + optionId + ' for items: ' + ex.listboxSelector + ); + +}); + +ariaTest('role="option" on li elements', exampleFile, 'option-role', async (t) => { + t.plan(2); + await assertAriaRoles(t, 'ex1', 'option', 10, 'li'); + await assertAriaRoles(t, 'ex2', 'option', 10, 'li'); +}); + +ariaTest('"aria-selected" on option elements', exampleFile, 'option-aria-selected', async (t) => { + t.plan(4); + + await assertAttributeDNE(t, ex[1].optionSelector, 'aria-selected'); + await t.context.session.findElement(By.css(ex[1].firstOptionSelector)).click(); + await assertAttributeValues(t, ex[1].optionSelector + ':nth-child(1)', 'aria-selected', 'true'); + + await assertAttributeValues(t, ex[2].optionSelector, 'aria-selected', 'false'); + await t.context.session.findElement(By.css(ex[2].firstOptionSelector)).click(); + await assertAttributeValues(t, ex[2].optionSelector + ':nth-child(1)', 'aria-selected', 'true'); +}); + +// Keys + +ariaTest('down arrow moves focus and selects', exampleFile, 'key-down-arrow', async (t) => { + t.plan(28); + + // Example 1 + + let listbox = (await t.context.session.findElements(By.css(ex[1].listboxSelector)))[0]; + let options = await t.context.session.findElements(By.css(ex[1].optionSelector)); + + // Put the focus on the first item + await t.context.session.findElement(By.css(ex[1].firstOptionSelector)).click(); + for (let index = 0; index < options.length - 1; index++) { + await listbox.sendKeys(Key.ARROW_DOWN); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, index + 1); + } + + // Send down arrow to the last option, focus should not move + await listbox.sendKeys(Key.ARROW_DOWN); + let lastOption = options.length - 1; + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, lastOption); + + // Example 2 + + listbox = (await t.context.session.findElements(By.css(ex[2].listboxSelector)))[0]; + options = await t.context.session.findElements(By.css(ex[2].optionSelector)); + + // Put the focus on the first item, and selects item, so skip by sending down arrow once + await t.context.session.findElement(By.css(ex[2].firstOptionSelector)).click(); + await listbox.sendKeys(Key.ARROW_DOWN); + + for (let index = 1; index < options.length - 1; index++) { + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, index); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[index] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with down arrow in example 2' + ); + await listbox.sendKeys(Key.ARROW_DOWN); + } + + // Send down arrow to the last option, focus should not move + await listbox.sendKeys(Key.ARROW_DOWN); + lastOption = options.length - 1; + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, lastOption); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[lastOption] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with down arrow in example 2' + ); +}); + +ariaTest('up arrow moves focus and selects', exampleFile, 'key-up-arrow', async (t) => { + t.plan(28); + + // Example 1 + + let listbox = (await t.context.session.findElements(By.css(ex[1].listboxSelector)))[0]; + let options = await t.context.session.findElements(By.css(ex[1].optionSelector)); + + // Put the focus on the first item + await t.context.session.findElement(By.css(ex[1].lastOptionSelector)).click(); + for (let index = options.length - 1; index > 0; index--) { + await listbox.sendKeys(Key.ARROW_UP); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, index - 1); + } + + // Sending up arrow to first option, focus should not move + await listbox.sendKeys(Key.ARROW_UP); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, 0); + + // Example 2 + + listbox = (await t.context.session.findElements(By.css(ex[2].listboxSelector)))[0]; + options = await t.context.session.findElements(By.css(ex[2].optionSelector)); + + // Put the focus on the last item, and selects item, so skip by sending down arrow once + await t.context.session.findElement(By.css(ex[2].lastOptionSelector)).click(); + await listbox.sendKeys(Key.ARROW_UP); + + for (let index = options.length - 2; index > 0; index--) { + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, index); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[index] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with down arrow in example 2' + ); + await listbox.sendKeys(Key.ARROW_UP); + } + + // Send down arrow to the last option, focus should not move + await listbox.sendKeys(Key.ARROW_UP); + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, 0); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[0] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with down arrow in example 2' + ); +}); + +ariaTest('home moves focus and selects', exampleFile, 'key-home', async (t) => { + t.plan(6); + + // Example 1 + + let listbox = (await t.context.session.findElements(By.css(ex[1].listboxSelector)))[0]; + let options = await t.context.session.findElements(By.css(ex[1].optionSelector)); + + // Put the focus on the second item + await options[1].click(); + await listbox.sendKeys(Key.HOME); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, 0); + + // Put the focus on the last item + await options[options.length - 1].click(); + await listbox.sendKeys(Key.HOME); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, 0); + + + // Example 2 + + listbox = (await t.context.session.findElements(By.css(ex[2].listboxSelector)))[0]; + options = await t.context.session.findElements(By.css(ex[2].optionSelector)); + + // Put the focus on the second item + await options[1].click(); + await listbox.sendKeys(Key.HOME); + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, 0); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[0] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with HOME in example 2' + ); + + // Put the focus on the last item + await options[options.length - 1].click(); + await listbox.sendKeys(Key.HOME); + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, 0); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[0] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with HOME in example 2' + ); +}); + +ariaTest('end moves focus and selects', exampleFile, 'key-end', async (t) => { + t.plan(6); + + // Example 1 + + let listbox = (await t.context.session.findElements(By.css(ex[1].listboxSelector)))[0]; + let options = await t.context.session.findElements(By.css(ex[1].optionSelector)); + let lastOption = options.length - 1; + + // Put the focus on the second item + await options[1].click(); + await listbox.sendKeys(Key.END); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, lastOption); + + // Put the focus on the last item + await options[lastOption - 1].click(); + await listbox.sendKeys(Key.END); + await assertAriaSelectedAndActivedescendant(t, ex[1].importantSelector, ex[1].optionSelector, lastOption); + + + // Example 2 + + listbox = (await t.context.session.findElements(By.css(ex[2].listboxSelector)))[0]; + options = await t.context.session.findElements(By.css(ex[2].optionSelector)); + + // Put the focus on the second item + await options[1].click(); + await listbox.sendKeys(Key.END); + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, lastOption); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[lastOption] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with END in example 2' + ); + + // Put the focus on the last item + await options[lastOption - 1].click(); + await listbox.sendKeys(Key.END); + await assertAriaActivedescendant(t, ex[2].availableSelector, ex[2].optionSelector, lastOption); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[lastOption] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is false when moving between options with END in example 2' + ); + +}); + +ariaTest('key space selects', exampleFile, 'key-space', async (t) => { + t.plan(19); + + const listbox = (await t.context.session.findElements(By.css(ex[2].listboxSelector)))[0]; + const options = await t.context.session.findElements(By.css(ex[2].optionSelector)); + + // Put the focus on the first item, and selects item + await t.context.session.findElement(By.css(ex[2].firstOptionSelector)).click(); + + for (let index = 0; index < options.length - 1; index++) { + await listbox.sendKeys(Key.ARROW_DOWN, Key.SPACE); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[index + 1] + .getAttribute('aria-selected'), + 'true', + 'aria-selected is true when sending space key to item at index: ' + (index + 1) + ); + } + + for (let index = options.length - 1; index >= 0 ; index--) { + await listbox.sendKeys(Key.SPACE); + t.is( + await(await t.context.session.findElements(By.css(ex[2].optionSelector)))[index] + .getAttribute('aria-selected'), + 'false', + 'aria-selected is true when sending space key to item at index: ' + (index) + ); + await listbox.sendKeys(Key.ARROW_UP); + } +}); + +// Bug: https://github.com/w3c/aria-practices/issues/919 +ariaTest.failing('key shift+down arrow moves focus and selects', exampleFile, 'key-shift-down-arrow', async (t) => { + t.plan(1); + t.fail(); +}); + +// Bug: https://github.com/w3c/aria-practices/issues/919 +ariaTest.failing('key shift+up arrow moves focus and selects', exampleFile, 'key-shift-up-arrow', async (t) => { + t.plan(1); + t.fail(); +}); + +// Bug: https://github.com/w3c/aria-practices/issues/919 +ariaTest.failing('key control+shift+home moves focus and selects', exampleFile, 'key-control-shift-home', async (t) => { + t.plan(1); + t.fail(); +}); + +// Bug: https://github.com/w3c/aria-practices/issues/919 +ariaTest.failing('key control+shift+end moves focus and selects', exampleFile, 'key-control-shift-end', async (t) => { + t.plan(1); + t.fail(); +}); + +// Bug: https://github.com/w3c/aria-practices/issues/919 +ariaTest.failing('key control+A selects all options', exampleFile, 'key-control-a', async (t) => { + t.plan(1); + t.fail(); +}); + diff --git a/test/tests/menu-button-links.js b/test/tests/menu-button-links.js index 5bab6160fa..c800c27ae5 100644 --- a/test/tests/menu-button-links.js +++ b/test/tests/menu-button-links.js @@ -108,7 +108,12 @@ ariaTest('"aria-labelledby" on role="menu"', exampleFile, 'menu-aria-labelledby' await assertAriaLabelledby(t, ex.menuSelector); }); -ariaTest('role="menuitem" on li element', exampleFile, 'menuitem-role', async (t) => { +ariaTest('role="none" on li element', exampleFile, 'none-role', async (t) => { + t.plan(1); + await assertAriaRoles(t, 'ex1', 'none', ex.numMenuitems, 'li'); +}); + +ariaTest('role="menuitem" on a element', exampleFile, 'menuitem-role', async (t) => { t.plan(1); await assertAriaRoles(t, 'ex1', 'menuitem', ex.numMenuitems, 'a'); }); diff --git a/test/tests/multithumb-slider.js b/test/tests/multithumb-slider.js index a9b6b8cfc9..6b66921ab5 100644 --- a/test/tests/multithumb-slider.js +++ b/test/tests/multithumb-slider.js @@ -90,10 +90,6 @@ ariaTest('"aria-valuemax" set on sliders', exampleFile, 'aria-valuemax', async ( 'Value of "aria-valuemax" for first flight slider on page load should be: ' + ex.flightDefaultVals[1] ); - t.log(ex); - t.log(await flightSliders[1].getAttribute('aria-valuemax')); - - t.is( await flightSliders[1].getAttribute('aria-valuemax'), ex.flightMax, diff --git a/test/tests/radio-1.js b/test/tests/radio-1.js index ec4ee00fc1..682cb3c2ba 100644 --- a/test/tests/radio-1.js +++ b/test/tests/radio-1.js @@ -83,6 +83,8 @@ ariaTest('"aria-checked" set on role="radio"', exampleFile, 'radio-aria-checked' } }); +// Keys + ariaTest('Moves focus to first or checked item', exampleFile, 'key-tab', async (t) => { t.plan(2); diff --git a/test/tests/radio-2.js b/test/tests/radio-2.js new file mode 100644 index 0000000000..5d0366322c --- /dev/null +++ b/test/tests/radio-2.js @@ -0,0 +1,222 @@ +'use strict'; + +const { ariaTest } = require('..'); +const { By, Key } = require('selenium-webdriver'); +const assertAttributeValues = require('../util/assertAttributeValues'); +const assertAriaLabelledby = require('../util/assertAriaLabelledby'); +const assertAriaRoles = require('../util/assertAriaRoles'); +const assertTabOrder = require('../util/assertTabOrder'); +const assertAriaActivedescendant = require('../util/assertAriaActivedescendant'); + +const exampleFile = 'radio/radio-2/radio-2.html'; + +const ex = { + radiogroupSelector: '#ex1 [role="radiogroup"]', + radioSelector: '#ex1 [role="radio"]', + radiogroupSelectors: [ + '#ex1 [role="radiogroup"]:nth-of-type(1)', + '#ex1 [role="radiogroup"]:nth-of-type(2)' + ], + radioSelectors: [ + '#ex1 [role="radiogroup"]:nth-of-type(1) [role="radio"]', + '#ex1 [role="radiogroup"]:nth-of-type(2) [role="radio"]' + ], + innerRadioSelector: '[role="radio"]' +}; + +// Attributes + +ariaTest('role="radigroup" on div element', exampleFile, 'radiogroup-role', async (t) => { + t.plan(1); + await assertAriaRoles(t, 'ex1', 'radiogroup', '2', 'ul'); +}); + +ariaTest('"aria-labelledby" attribute on radiogroup', exampleFile, 'radiogroup-aria-labelledby', async (t) => { + t.plan(1); + await assertAriaLabelledby(t, ex.radiogroupSelector); +}); + +ariaTest('tabindex on radiogroup', exampleFile, 'radiogroup-tabindex', async (t) => { + t.plan(1); + await assertAttributeValues(t, ex.radiogroupSelector, 'tabindex', '0'); +}); + +ariaTest('aria-activedescendant', exampleFile, 'radiogroup-aria-activedescendant', async (t) => { + t.plan(2); + await assertAttributeValues(t, ex.radiogroupSelectors[0], 'aria-activedescendant', 'rb11'); + await assertAttributeValues(t, ex.radiogroupSelectors[1], 'aria-activedescendant', 'rb21'); +}); + +ariaTest('role="radio" on div elements', exampleFile, 'radio-role', async (t) => { + t.plan(1); + await assertAriaRoles(t, 'ex1', 'radio', '6', 'li'); +}); + +ariaTest('"aria-checked" set on role="radio"', exampleFile, 'radio-aria-checked', async (t) => { + t.plan(19); + + // The radio groups will be all unchecked on page load + await assertAttributeValues(t, ex.radioSelector, 'aria-checked', 'false'); + + const radiogroups = await t.context.session.findElements(By.css(ex.radiogroupSelector)); + for (let radiogroup of radiogroups) { + const radios = await radiogroup.findElements(By.css(ex.innerRadioSelector)); + + for (let checked = 0; checked < radios.length; checked++) { + + await radios[checked].click(); + for (let el = 0; el < radios.length; el++) { + + // test only one element has aria-checked="true" + const isChecked = el === checked ? 'true' : 'false'; + t.is( + await radios[el].getAttribute('aria-checked'), + isChecked, + 'Tab at index ' + checked + ' is checked, therefore, tab at index ' + + el + ' should have aria-checked="' + checked + '"' + ); + } + } + } +}); + +// Keys + +ariaTest('Moves focus to first or checked item', exampleFile, 'key-tab', async (t) => { + t.plan(2); + + // On page load, the first item in the radio list should be in tab index + await assertTabOrder(t, [ + ex.radiogroupSelectors[0], + ex.radiogroupSelectors[1] + ]); + + // Click the second item in each radio list + await t.context.session + .findElement(By.css(ex.radioSelectors[0] + ':nth-of-type(2)')).click(); + await t.context.session + .findElement(By.css(ex.radioSelectors[1] + ':nth-of-type(2)')).click(); + + // Now the second radio item in each list should be selected, but tab will take + // the focus between the radio groups + await assertTabOrder(t, [ + ex.radiogroupSelectors[0], + ex.radiogroupSelectors[1] + ]); +}); + +ariaTest('Selects radio item', exampleFile, 'key-space', async (t) => { + t.plan(2); + + await t.context.session.findElement(By.css(ex.radiogroupSelectors[0])).sendKeys(Key.SPACE); + const firstCrustRadioOption = ex.radioSelectors[0] + ':nth-of-type(1)'; + await assertAttributeValues(t, firstCrustRadioOption, 'aria-checked', 'true'); + + await t.context.session.findElement(By.css(ex.radiogroupSelectors[1])).sendKeys(Key.SPACE); + const firstDeliveryRadioOption = ex.radioSelectors[1] + ':nth-of-type(1)'; + await assertAttributeValues(t, firstDeliveryRadioOption, 'aria-checked', 'true'); +}); + +ariaTest('RIGHT ARROW changes focus and checks radio', exampleFile, 'key-right-arrow', async (t) => { + t.plan(12); + + for (let groupIndex = 0; groupIndex < 2; groupIndex++) { + + const radiogroupSelector = ex.radiogroupSelectors[groupIndex]; + const radioSelector = ex.radioSelectors[groupIndex]; + const radiogroup = await t.context.session.findElement(By.css(radiogroupSelector)); + const radios = await t.context.session.findElements(By.css(radioSelector)); + + // Right arrow moves focus to right + for (let index = 0; index < radios.length; index++) { + await radiogroup.sendKeys(Key.ARROW_RIGHT); + const newindex = (index + 1) % radios.length; + + await assertAriaActivedescendant(t, radiogroupSelector, radioSelector, newindex); + t.is( + await radios[newindex].getAttribute('aria-checked'), + 'true', + 'Radio at index ' + newindex + ' should be checked after ARROW_RIGHT to radio' + + ' at index ' + index + ); + } + } +}); + +ariaTest('DOWN ARROW changes focus and checks radio', exampleFile, 'key-down-arrow', async (t) => { + t.plan(12); + + for (let groupIndex = 0; groupIndex < 2; groupIndex++) { + + const radiogroupSelector = ex.radiogroupSelectors[groupIndex]; + const radioSelector = ex.radioSelectors[groupIndex]; + const radiogroup = await t.context.session.findElement(By.css(radiogroupSelector)); + const radios = await t.context.session.findElements(By.css(radioSelector)); + + // Down arrow moves focus to right + for (let index = 0; index < radios.length; index++) { + await radiogroup.sendKeys(Key.ARROW_DOWN); + const newindex = (index + 1) % radios.length; + + await assertAriaActivedescendant(t, radiogroupSelector, radioSelector, newindex); + t.is( + await radios[newindex].getAttribute('aria-checked'), + 'true', + 'Radio at index ' + newindex + ' should be checked after ARROW_DOWN to radio' + + ' at index ' + index + ); + } + } +}); + +ariaTest('LEFT ARROW changes focus and checks radio', exampleFile, 'key-left-arrow', async (t) => { + t.plan(12); + + for (let groupIndex = 0; groupIndex < 2; groupIndex++) { + + const radiogroupSelector = ex.radiogroupSelectors[groupIndex]; + const radioSelector = ex.radioSelectors[groupIndex]; + const radiogroup = await t.context.session.findElement(By.css(radiogroupSelector)); + const radios = await t.context.session.findElements(By.css(radioSelector)); + + // Left arrow moves focus to left + for (let index of [0, 2, 1]) { + await radiogroup.sendKeys(Key.ARROW_LEFT); + const newindex = index - 1 === -1 ? 2 : index - 1; + + await assertAriaActivedescendant(t, radiogroupSelector, radioSelector, newindex); + t.is( + await radios[newindex].getAttribute('aria-checked'), + 'true', + 'Radio at index ' + newindex + ' should be checked after ARROW_LEFT to radio' + + ' at index ' + index + ); + } + } +}); + +ariaTest('UP ARROW changes focus and checks radio', exampleFile, 'key-up-arrow', async (t) => { + t.plan(12); + + for (let groupIndex = 0; groupIndex < 2; groupIndex++) { + + const radiogroupSelector = ex.radiogroupSelectors[groupIndex]; + const radioSelector = ex.radioSelectors[groupIndex]; + const radiogroup = await t.context.session.findElement(By.css(radiogroupSelector)); + const radios = await t.context.session.findElements(By.css(radioSelector)); + + // Up arrow moves focus to up + for (let index of [0, 2, 1]) { + await radiogroup.sendKeys(Key.ARROW_UP); + const newindex = index - 1 === -1 ? 2 : index - 1; + + await assertAriaActivedescendant(t, radiogroupSelector, radioSelector, newindex); + t.is( + await radios[newindex].getAttribute('aria-checked'), + 'true', + 'Radio at index ' + newindex + ' should be checked after ARROW_UP to radio' + + ' at index ' + index + ); + } + } +}); diff --git a/test/tests/treegrid-1.js b/test/tests/treegrid-1.js index df18d62d46..a8562ab788 100644 --- a/test/tests/treegrid-1.js +++ b/test/tests/treegrid-1.js @@ -1,7 +1,7 @@ 'use strict'; const { ariaTest } = require('..'); -const { By, Key } = require('selenium-webdriver'); +const { By, Key, until } = require('selenium-webdriver'); const assertAriaRoles = require('../util/assertAriaRoles'); const assertAttributeValues = require('../util/assertAttributeValues'); const assertAriaLabelExists = require('../util/assertAriaLabelExists'); @@ -679,51 +679,40 @@ ariaTest('CTRL+END moves focus', exampleFile, 'key-control-end', async (t) => { }); // This test fails due to: https://github.com/w3c/aria-practices/issues/790#issuecomment-422079276 -// When the bug is fixed this test should be uncommented. - -// ariaTest('ENTER actives interative items item', exampleFile, 'key-enter', async (t) => { -// t.plan(16); - - -// // INTERACTIVE ITEM 1: Enter sent while focus is on email row should open email altert -// await openAllThreads(t); -// const emailRows = await t.context.session.findElements(By.css(ex.emailRowSelector)); +ariaTest.failing('ENTER actives interactive items item', exampleFile, 'key-enter', async (t) => { + t.plan(2); -// for (let email of emailRows) { -// await email.sendKeys(Key.ENTER); + // INTERACTIVE ITEM 1: Enter sent while focus is on email row should open email alert -// t.truthy( -// await t.context.session.switchTo().alert().getText(), -// 'Sending "enter" to any email row should open alert with the test of the email' -// ); -// await t.context.session.switchTo().alert().accept(); -// } + const email = await t.context.session.findElement(By.css(ex.emailRowSelector)); + await email.sendKeys(Key.ENTER); -// // INTERACTIVE ITEM 1: Enter sent while focus is email gridcell should trigger link -// for (let index = 0; index <= ex.lastRowIndex; index++) { + const alert = await t.context.session.wait(until.alertIsPresent(), t.context.waitTime); + t.truthy( + await alert.getText(), + 'Sending "enter" to any email row should open alert with the rest of the email' + ); + await alert.accept(); -// // Reload the page and open all emails -// await t.context.session.get(t.context.url); -// await openAllThreads(t); + // INTERACTIVE ITEM 1: Enter sent while focus is email gridcell should trigger link -// const selector = '#ex1 [role="row"]:nth-of-type(' + (index+1) + ') a'; -// const newUrl = t.context.url + '#test-url-change' + const selector = '#ex1 [role="row"]:nth-of-type(1) a'; + const newUrl = t.context.url + '#test-url-change'; -// // Reset the href to not be an email link in order to test -// await t.context.session.executeScript(function () { -// let [selector, newUrl] = arguments; -// document.querySelector(selector).href = newUrl; -// }, selector, newUrl); + // Reset the href to not be an email link in order to test + await t.context.session.executeScript(function () { + let [selector, newUrl] = arguments; + document.querySelector(selector).href = newUrl; + }, selector, newUrl); -// await t.context.session.findElement(By.css(selector)).sendKeys(Key.ENTER); + await t.context.session.findElement(By.css(selector)).sendKeys(Key.ENTER); -// // Test the the URL is updated. -// t.is( -// await t.context.session.getCurrentUrl(), -// newUrl, -// 'Sending "enter" to a link within the gridcell should activate the link' -// ); -// } -// }); + // Test the the URL is updated. + t.is( + await t.context.session.getCurrentUrl(), + newUrl, + 'Sending "enter" to a link within the gridcell should activate the link' + ); +}); diff --git a/test/util/assertAriaActivedescendant.js b/test/util/assertAriaActivedescendant.js index b9cf561c37..01f9bbccbf 100644 --- a/test/util/assertAriaActivedescendant.js +++ b/test/util/assertAriaActivedescendant.js @@ -25,7 +25,7 @@ module.exports = async function assertAriaSelectedAndActivedescendant (t, active .findElement(By.css(activedescendantSelector)) .getAttribute('aria-activedescendant'), optionId, - 'aria-activedescendant should be set to ' + optionId + ' for items: ' + activedescendantSelector + 'aria-activedescendant should be set to ' + optionId + ' for item: ' + activedescendantSelector ); // Confirm the focus is on the aria-activedescendent element diff --git a/test/util/assertAriaLabelExists.js b/test/util/assertAriaLabelExists.js index 2c8cd5cec7..fc62f37b8d 100644 --- a/test/util/assertAriaLabelExists.js +++ b/test/util/assertAriaLabelExists.js @@ -12,24 +12,33 @@ const assert = require('assert'); module.exports = async function assertAriaLabel (t, elementSelector) { - let ariaLabelExists = await t.context.session.executeScript(async function () { - const selector = arguments[0]; - let els = document.querySelector(selector); - return els.hasAttribute('aria-label'); - }, elementSelector); - - assert( - ariaLabelExists, - '"aria-label" attribute should exist on element(s): ' + elementSelector - ); - - let element = t.context.session.findElement(By.css(elementSelector)); - let labelValue = await element.getAttribute('aria-label'); + const elements = await t.context.session.findElements(By.css(elementSelector)); assert.ok( - labelValue, - '"aria-label" attribute should have a value on element(s): ' + elementSelector + elements.length, + 'CSS elector returned no results: ' + elementSelector ); + for (let index = 0; index < elements.length; index++) { + + let ariaLabelExists = await t.context.session.executeScript(async function () { + const [selector, index] = arguments; + let els = document.querySelectorAll(selector); + return els[index].hasAttribute('aria-label'); + }, elementSelector, index); + + assert( + ariaLabelExists, + '"aria-label" attribute should exist on element(s): ' + elementSelector + ); + + let labelValue = await elements[index].getAttribute('aria-label'); + + assert.ok( + labelValue, + '"aria-label" attribute should have a value on element(s): ' + elementSelector + ); + } + t.pass(); }; diff --git a/test/util/assertAttributeDNE.js b/test/util/assertAttributeDNE.js index 16e2197a2e..b0886aef02 100644 --- a/test/util/assertAttributeDNE.js +++ b/test/util/assertAttributeDNE.js @@ -14,6 +14,11 @@ module.exports = async function assertAttributeDNE (t, selector, attribute) { const numElements = (await t.context.session.findElements(By.css(selector))).length; + assert.ok( + numElements, + 'CSS elector returned no results: ' + selector + ); + for (let index = 0; index < numElements; index++) { const attributeExists = await t.context.session.executeScript(function () { let [selector, index, attribute] = arguments; diff --git a/test/util/assertRovingTabindex.js b/test/util/assertRovingTabindex.js index bf403cbc12..9e0c57f315 100644 --- a/test/util/assertRovingTabindex.js +++ b/test/util/assertRovingTabindex.js @@ -16,6 +16,11 @@ module.exports = async function assertRovingTabindex (t, elementsSelector, key) // tabindex='0' is expected on the first element let elements = await t.context.session.findElements(By.css(elementsSelector)); + assert.ok( + elements.length, + 'CSS elector returned no results: ' + elementsSelector + ); + // test only one element has tabindex="0" for (let tabableEl = 0; tabableEl < elements.length; tabableEl++) { for (let el = 0; el < elements.length; el++) { diff --git a/test/util/report.js b/test/util/report.js new file mode 100644 index 0000000000..d917079c81 --- /dev/null +++ b/test/util/report.js @@ -0,0 +1,254 @@ +#!/usr/bin/env node +'use strict'; + +const cheerio = require('cheerio'); +const path = require('path'); +const fs = require('fs'); +const htmlparser2 = require('htmlparser2'); +const { spawnSync } = require('child_process'); + +const examplePath = path.resolve(__dirname, '..', '..', 'examples'); +const testsPath = path.resolve(__dirname, '..', 'tests'); +const ignoreExampleDirs = path.resolve(__dirname, 'report_files', 'ignore_test_directories'); +const ignoreExampleFiles = path.resolve(__dirname, 'report_files', 'ignore_html_files'); +const ignoredDataTestId = 'test-not-required'; + +const ignoreDirectories = fs.readFileSync(ignoreExampleDirs) + .toString() + .trim() + .split('\n') + .map(d => path.resolve(examplePath, d)); +const ignoreFiles = fs.readFileSync(ignoreExampleFiles) + .toString() + .trim() + .split('\n') + .map(f => path.resolve(examplePath, f)); + +/** + * Recursively find all example pages, saves to exampleFiles global + * obect. + * + * @param {String} currentDirPath - root example directory + */ +const getExampleFiles = function (currentDirPath, exampleFiles) { + fs.readdirSync(currentDirPath).forEach(function (name) { + var filePath = path.join(currentDirPath, name); + var stat = fs.statSync(filePath); + if ( + stat.isFile() && + path.extname(filePath) == '.html' && + ignoreFiles.indexOf(filePath) === -1 + ) { + exampleFiles.push(filePath); + } + else if ( + stat.isDirectory() && + ignoreDirectories.indexOf(filePath) === -1 + ) { + getExampleFiles(filePath, exampleFiles); + } + }); +}; + +/** + * Return human readible name for a "Keyboard Support" table row. + * + * @param {jQuery object} $ - loaded Cheerio dom + * @param {jQuery object} $tableRow - root example directory + */ +const getKeyboardRowName = function ($, $tableRow) { + return $('th', $tableRow).text().replace(/\n/g, ', '); +}; + +/** + * Return human readible name for an "Attributes" table row. + * + * @param {jQuery object} $ - loaded Cheerio dom + * @param {jQuery object} $tableRow - root example directory + */ +const getAttributeRowName = function ($, $tableRow) { + // use the containt 'th' text to identify the row. If there is no text + // in the 'th' element, use the 'element' column text. + let rowName = $('th', $tableRow).text(); + if (!rowName) { + rowName = $(':nth-child(3)', $tableRow).text(); + } + return rowName; +}; + + +/** + * Processes all example files to find data-test-ids and missing data-test-ids + * Builds exampleCoverage object: + * { + * : { + * existingTestIds: + * missingTests: + * missingAttrs: + * missingKeys: + * } + * } + * + * @param {Array} exampleFiles - all example files to process + * @param {Object} exampleCoverage - object to add coverage information to + */ +const processDocumentationInExampleFiles = function (exampleFiles, exampleCoverage) { + for (let exampleFile of exampleFiles) { + var data = fs.readFileSync(exampleFile); + const dom = htmlparser2.parseDOM(data); + const $ = cheerio.load(dom); + + let dataTestIds = new Set(); + let attrsMissingIds = new Set(); + let keysMissingIds = new Set(); + + // Find all the "Keyboard Interaction" table rows + $('table.def tbody tr').each(function () { + let $row = $(this); + let dataTestId = $row.attr('data-test-id'); + + if (dataTestId === ignoredDataTestId) { return; }; + + if (dataTestId !== undefined) { + dataTestIds.add(dataTestId); + } + else { + keysMissingIds.add(getKeyboardRowName($, $row)); + } + }); + + // Find all the "Attribute" table rows + $('table.attributes tbody tr').each(function () { + let $row = $(this); + let dataTestId = $row.attr('data-test-id'); + + if (dataTestId === ignoredDataTestId) { return; }; + + if (dataTestId !== undefined) { + dataTestIds.add(dataTestId); + } + else { + attrsMissingIds.add(getAttributeRowName($, $row)); + } + }); + + // Use the relative path to identify the example page + const example = path.relative(examplePath, exampleFile); + + exampleCoverage[example] = { + existingTestIds: dataTestIds, + missingTests: new Set(dataTestIds), + missingAttrs: attrsMissingIds, + missingKeys: keysMissingIds + }; + } +}; + +/** + * Runs ava tests in coverage mode to collect data on which tests exist. + * After running, `exampleCoverage[example].missingTests` will be an array of + * only data-test-ids for which no regresison test was found. + * + * @param {Object} exampleCoverage - object with existing coverage information + */ +const getRegressionTestCoverage = function (exampleCoverage) { + process.env.REGRESSION_COVERAGE_REPORT = 1; + + const cmd = path.resolve(__dirname, '..', '..', 'node_modules', 'ava', 'cli.js'); + const cmdargs = [testsPath, '--tap', '-c', '1']; + + const output = spawnSync(cmd, cmdargs); + const avaResults = output.stdout.toString(); + + let testRegex = /^# (\S+) [>›] (\S+\.html) \[data-test-id="(\S+)"\]/gm; + let matchResults; + while (matchResults = testRegex.exec(avaResults)) { + let example = matchResults[2]; + let dataTestId = matchResults[3]; + + // If the test file has a data-test-id, the data-test-id must exist on + // the test page. + exampleCoverage[example].missingTests.delete(matchResults[3]); + } +}; + +// Process example files and results of regression test to find coverage information. +const exampleFiles = []; +const exampleCoverage = {}; +getExampleFiles(examplePath, exampleFiles); +processDocumentationInExampleFiles(exampleFiles, exampleCoverage); +getRegressionTestCoverage(exampleCoverage); + +let examplesWithNoTests = 0; +let examplesWithNoTestsReport = ''; +let examplesMissingSomeTests = 0; +let examplesMissingSomeTestsReport = ''; +let missingTestIds = 0; +let missingTestIdsReport = ''; +let totalTestIds = 0; + +for (let example in exampleCoverage) { + const existingTestIds = exampleCoverage[example].existingTestIds.size; + const missingTests = exampleCoverage[example].missingTests.size; + const missingKeys = exampleCoverage[example].missingKeys.size; + const missingAttrs = exampleCoverage[example].missingAttrs.size; + + if (existingTestIds !== missingTests) { + totalTestIds += existingTestIds; + } + + if (missingTests || missingKeys || missingAttrs) { + let exampleName = example; + + if (existingTestIds === missingTests) { + examplesWithNoTestsReport += exampleName + '\n'; + examplesWithNoTests++; + } + else if (missingTests) { + examplesMissingSomeTestsReport += exampleName + ':\n'; + + for (let testId of exampleCoverage[example].missingTests) { + examplesMissingSomeTestsReport += ' ' + testId + '\n'; + } + + examplesMissingSomeTests += 1; + missingTestIds += missingTests; + } + + if (missingKeys || missingAttrs) { + missingTestIdsReport += exampleName + '\n'; + if (missingKeys) { + missingTestIdsReport += ' "Keyboard Support" table(s):\n'; + for (let row of exampleCoverage[example].missingKeys) { + missingTestIdsReport += ' ' + row + '\n'; + } + } + + if (missingAttrs) { + missingTestIdsReport += ' "Attributes" table(s):\n'; + for (let row of exampleCoverage[example].missingAttrs) { + missingTestIdsReport += ' ' + row + '\n'; + } + } + } + } +} + +console.log('\nExamples without regression tests:\n'); +console.log(examplesWithNoTestsReport || 'None found.\n'); +console.log('\nExamples missing regression tests:\n'); +console.log(examplesMissingSomeTestsReport || 'None found.\n'); +console.log('\nExamples documentation table rows without data-test-ids:\n'); +console.log(missingTestIdsReport || 'None found.\n'); + +console.log('SUMMARTY:\n'); +console.log(' ' + exampleFiles.length + ' example pages found.'); +console.log(' ' + examplesWithNoTests + ' example pages have no regression tests.'); +console.log(' ' + examplesMissingSomeTests + ' example pages are missing approximately ' + + missingTestIds + ' out of approximately ' + totalTestIds + ' tests.\n'); + +if (examplesMissingSomeTests) { + console.log('ERROR:\n\n Please write missing tests for this report to pass.\n'); + + process.exitCode = 1; +} diff --git a/test/util/report_files/ignore_html_files b/test/util/report_files/ignore_html_files new file mode 100644 index 0000000000..84a1e7eec6 --- /dev/null +++ b/test/util/report_files/ignore_html_files @@ -0,0 +1,5 @@ +grid/advancedDataGrid.html +menubar/menubar-1/mb-about.html +menubar/menubar-1/mb-academics.html +feed/feedDisplay.html +menubar/menubar-1/mb-admissions.html diff --git a/test/util/report_files/ignore_test_directories b/test/util/report_files/ignore_test_directories new file mode 100644 index 0000000000..d5ee863390 --- /dev/null +++ b/test/util/report_files/ignore_test_directories @@ -0,0 +1,2 @@ +coding-template +landmarks