Merge pull request #3025 from BryceLTaylor/Skip-Flakey-Tests

Skip flakey tests

package.json

@@ -6,9 +6,9 @@
     "start": "make start",
     "stop": "make stop",
     "test": "make test",
-    "smoke": "tap ./test/integration/smoke-testing/*.js --timeout=3600",
-    "smoke-verbose": "tap ./test/integration/smoke-testing/*.js --timeout=3600 -R spec",
-    "smoke-sauce": "SMOKE_REMOTE=true tap ./test/integration/smoke-testing/*.js --timeout=60000",
+    "smoke": "tap ./test/integration/smoke-testing/*.js --timeout=3600 --no-coverage -R classic",
+    "smoke-verbose": "tap ./test/integration/smoke-testing/*.js --timeout=3600 --no-coverage -R spec",
+    "smoke-sauce": "SMOKE_REMOTE=true tap ./test/integration/smoke-testing/*.js --timeout=60000 --no-coverage -R classic",
     "watch": "make watch",
     "build": "make build",
     "dev": "make watch && make start &",
@@ -110,7 +110,7 @@
     "slick-carousel": "1.6.0",
     "source-map-support": "0.3.2",
     "style-loader": "0.12.3",
-    "tap": "7.1.2",
+    "tap": "14.2.0",
     "url-loader": "0.5.6",
     "watch": "0.16.0",
     "webpack": "2.7.0",

test/integration/selenium-helpers.js

@@ -9,6 +9,8 @@ const buildID = process.env.TRAVIS_BUILD_NUMBER;
 const {SAUCE_USERNAME, SAUCE_ACCESS_KEY} = process.env;
 const {By, Key, until} = webdriver;
 
+const DEFAULT_TIMEOUT_MILLISECONDS = 20 * 1000;
+
 class SeleniumHelper {
     constructor () {
         bindAll(this, [
@@ -25,7 +27,8 @@ class SeleniumHelper {
             'getDriver',
             'getLogs',
             'getSauceDriver',
-            'urlMatches'
+            'urlMatches',
+            'waitUntilGone'
         ]);
     }
     buildDriver (name) {
@@ -86,8 +89,16 @@ class SeleniumHelper {
         return Key[keyName];
     }
 
-    findByXpath (xpath) {
-        return this.driver.wait(until.elementLocated(By.xpath(xpath), 5 * 1000));
+    findByXpath (xpath, timeoutMessage = `findByXpath timed out for path: ${xpath}`) {
+        return this.driver.wait(until.elementLocated(By.xpath(xpath)), DEFAULT_TIMEOUT_MILLISECONDS, timeoutMessage)
+            .then(el => (
+                this.driver.wait(el.isDisplayed(), DEFAULT_TIMEOUT_MILLISECONDS, `${xpath} is not visible`)
+                    .then(() => el)
+            ));
+    }
+
+    waitUntilGone (element) {
+        return this.driver.wait(until.stalenessOf(element));
     }
 
     clickXpath (xpath) {

test/integration/smoke-testing/test-my-stuff.js

@@ -115,7 +115,7 @@ test('Add To button should bring up a list of studios', t => {
         .then(() => t.end());
 });
 
-test('+ New Studio button should take you to the studio page', t => {
+test('+ New Studio button should take you to the studio page', {skip: true}, t => {
     clickXpath('//a[contains(@class, "mystuff-icon")]')
         .then(() => clickXpath('//form[@id="new_studio"]/button[@type="submit"]'))
         .then(() => findByXpath('//div[@id="show-add-project"]'))

test/integration/smoke-testing/test_footer_links.js

@@ -194,8 +194,9 @@ tap.test('clickContactUsLink', options, t => {
     });
 });
 
+// skip this test since it points to an external site
 // SCRATCH STORE
-tap.test('clickScratchStoreLink', options, t => {
+tap.test('clickScratchStoreLink', {skip: true}, t => {
     const linkText = 'Scratch Store';
     const expectedUrl = 'https://scratch-foundation.myshopify.com/';
     clickFooterLinks(linkText).then(url => {
@@ -204,8 +205,9 @@ tap.test('clickScratchStoreLink', options, t => {
     });
 });
 
+// skip this test since it points to an external site
 // DONATE
-tap.test('clickDonateLink', options, t => {
+tap.test('clickDonateLink', {skip: true}, t => {
     const linkText = 'Donate';
     const expectedUrl = 'https://secure.donationpay.org/scratchfoundation/';
     clickFooterLinks(linkText).then(url => {
@@ -248,8 +250,9 @@ tap.test('clickDMCALink', options, t => {
 
 // ==== SCRATCH FAMILY column ====
 
+// skip this test since it points to an external site
 // SCRATCH ED (SCRATCHED)
-tap.test('clickScratchEdLink', options, t => {
+tap.test('clickScratchEdLink', {skip: true}, t => {
     const linkText = 'ScratchEd';
     const expectedUrl = 'http://scratched.gse.harvard.edu/';
     clickFooterLinks(linkText).then(url => {
@@ -258,8 +261,9 @@ tap.test('clickScratchEdLink', options, t => {
     });
 });
 
+// skip this test since it points to an external site
 // SCRATCH JR (SCRATCHJR)
-tap.test('clickScratchJrLink', options, t => {
+tap.test('clickScratchJrLink', {skip: true}, t => {
     const linkText = 'ScratchJr';
     const expectedUrl = 'https://www.scratchjr.org/';
     clickFooterLinks(linkText).then(url => {
@@ -268,8 +272,9 @@ tap.test('clickScratchJrLink', options, t => {
     });
 });
 
+// skip this test since it points to an external site
 // SCRATCH DAY
-tap.test('clickScratchDayLink', options, t => {
+tap.test('clickScratchDayLink', {skip: true}, t => {
     const linkText = 'Scratch Day';
     const expectedUrl = 'https://day.scratch.mit.edu/';
     clickFooterLinks(linkText).then(url => {
@@ -288,8 +293,9 @@ tap.test('clickScratchConferenceLink', options, t => {
     });
 });
 
+// skip this test since it points to an external site
 // SCRATCH FOUNDATION
-tap.test('clickScratchFoundationLink', options, t => {
+tap.test('clickScratchFoundationLink', {skip: true}, t => {
     const linkText = 'Scratch Foundation';
     const expectedUrl = 'https://www.scratchfoundation.org/';
     clickFooterLinks(linkText).then(url => {

test/integration/smoke-testing/test_project_page.js

@@ -9,7 +9,9 @@ const driver = helper.buildDriver('www-smoke test_sign_in_out_homepage');
 const {
     clickText,
     clickXpath,
-    dragFromXpathToXpath
+    dragFromXpathToXpath,
+    findByXpath,
+    waitUntilGone
 } = helper;
 
 const rootUrl = process.env.ROOT_URL || 'https://scratch.ly';
@@ -27,14 +29,17 @@ tap.beforeEach(function () {
 });
 
 test('Find fullscreen button', t => {
-    clickXpath('//div[starts-with(@class, "loader_background")]')
+    findByXpath('//div[starts-with(@class, "loader_background")]')
+        .then(el => waitUntilGone(el))
         .then(() => clickXpath('//div[starts-with(@class, "stage_green-flag-overlay")]'))
         .then(() => clickXpath('//img[contains(@alt, "Enter full screen mode")]'))
         .then(() => t.end());
 });
 
 test('Open and close Copy Link modal', t => {
-    clickText('Copy Link')
+    findByXpath('//div[starts-with(@class, "loader_background")]')
+        .then(el => waitUntilGone(el))
+        .then(() => clickText('Copy Link'))
         .then(() => clickXpath('//div[contains(@class, "social-label-title")]'))
         .then(() => clickXpath('//img[contains(@alt, "close-icon")]'))
         .then(() => clickXpath('//img[contains(@alt, "Enter full screen mode")]'))
@@ -42,7 +47,8 @@ test('Open and close Copy Link modal', t => {
 });
 
 test('Dragging out of modal should not close modal', t => {
-    clickXpath('//div[starts-with(@class, "loader_background")]')
+    findByXpath('//div[starts-with(@class, "loader_background")]')
+        .then(el => waitUntilGone(el))
         .then(() => clickXpath('//div[starts-with(@class, "stage_green-flag-overlay")]'))
         .then(() => clickText('Copy Link'))
         .then(() => clickXpath('//div[contains(@class, "social-label-title")]'))

test/integration/smoke-testing/test_search.js

@@ -13,7 +13,7 @@ const tap = require('tap');
 const test = tap.test;
 
 // Set test url through environment variable
-const rootUrl = process.env.ROOT_URL || 'http://localhost:8333';
+const rootUrl = process.env.ROOT_URL || 'https://scratch.ly';
 const searchBaseUrl = `${rootUrl}/search/`;
 
 // chrome driver