Merge pull request #4923 from MaximKhlobystov/selenium-visual-regression

[HTML5 Client] Initial Visual Regression Tests Setup

bigbluebutton-html5/.babelrc

@@ -0,0 +1,9 @@
+{
+  "env": {
+    "production": {
+      "plugins": [
+        "react-remove-properties"
+      ]
+    }
+  }
+}

bigbluebutton-html5/imports/ui/components/audio/audio-modal/component.jsx

@@ -193,17 +193,17 @@ class AudioModal extends Component {
         <Button
           className={styles.audioBtn}
           label={intl.formatMessage(intlMessages.microphoneLabel)}
-          icon={'unmute'}
+          icon="unmute"
           circle
-          size={'jumbo'}
+          size="jumbo"
           onClick={this.handleGoToEchoTest}
         />
         <Button
           className={styles.audioBtn}
           label={intl.formatMessage(intlMessages.listenOnlyLabel)}
-          icon={'listen'}
+          icon="listen"
           circle
-          size={'jumbo'}
+          size="jumbo"
           onClick={this.handleJoinListenOnly}
         />
       </span>
@@ -304,13 +304,17 @@ class AudioModal extends Component {
           onRequestClose={this.closeModal}
         >
           { isConnecting ? null :
-          <header className={styles.header}>
+          <header
+            data-test="audioModalHeader"
+            className={styles.header}
+          >
             <h3 className={styles.title}>
               { content ?
                 this.contents[content].title :
                 intl.formatMessage(intlMessages.audioChoiceLabel)}
             </h3>
             <Button
+              data-test="modalBaseCloseButton"
               className={styles.closeBtn}
               label={intl.formatMessage(intlMessages.closeLabel)}
               icon={'close'}

bigbluebutton-html5/imports/ui/components/chat/chat-dropdown/component.jsx

@@ -121,6 +121,7 @@ class ChatDropdown extends Component {
       >
         <DropdownTrigger tabIndex={0}>
           <Button
+            data-test="chatDropdownTrigger"
             className={styles.btn}
             icon="more"
             ghost

bigbluebutton-html5/imports/ui/components/chat/component.jsx

@@ -41,9 +41,15 @@ const Chat = (props) => {
   } = props;
 
   return (
-    <div className={styles.chat}>
+    <div
+      data-test="publicChat"
+      className={styles.chat}
+    >
       <header className={styles.header}>
-        <div className={styles.title}>
+        <div
+          data-test="chatTitle"
+          className={styles.title}
+        >
           <Link
             to="/users"
             role="button"

bigbluebutton-html5/imports/ui/components/dropdown/content/component.jsx

@@ -25,7 +25,9 @@ const defaultProps = {
 
 export default class DropdownContent extends Component {
   render() {
-    const { placement, className, children, style } = this.props;
+    const {
+      placement, className, children, style,
+    } = this.props;
     const { dropdownToggle, dropdownShow, dropdownHide } = this.props;
 
     const placementName = placement.split(' ').join('-');
@@ -40,6 +42,7 @@ export default class DropdownContent extends Component {
       <div
         style={style}
         aria-expanded={this.props['aria-expanded']}
+        data-test="dropdownContent"
         className={cx(styles.content, styles[placementName], className)}
       >
         <div className={styles.scrollable}>

bigbluebutton-html5/imports/ui/components/dropdown/list/item/component.jsx

@@ -35,8 +35,10 @@ export default class DropdownListItem extends Component {
   }
 
   render() {
-    const { id, label, description, children, injectRef, tabIndex, onClick, onKeyDown,
-      className, style } = this.props;
+    const {
+      id, label, description, children, injectRef, tabIndex, onClick, onKeyDown,
+      className, style,
+    } = this.props;
 
     return (
       <li
@@ -50,6 +52,7 @@ export default class DropdownListItem extends Component {
         className={cx(styles.item, className)}
         style={style}
         role="menuitem"
+        data-test={this.props['data-test']}
       >
         {
           children || this.renderDefault()

bigbluebutton-html5/imports/ui/components/modal/fullscreen/component.jsx

@@ -65,6 +65,7 @@ class ModalFullscreen extends Component {
           <h1 className={styles.title}>{title}</h1>
           <div className={styles.actions}>
             <Button
+              data-test='modalDismissButton'
               className={styles.dismiss}
               label={dismiss.label}
               disabled={dismiss.disabled}
@@ -72,6 +73,7 @@ class ModalFullscreen extends Component {
               aria-describedby={'modalDismissDescription'}
             />
             <Button
+              data-test='modalConfirmButton'
               color={'primary'}
               className={styles.confirm}
               label={confirm.label}

bigbluebutton-html5/imports/ui/components/nav-bar/component.jsx

@@ -169,6 +169,7 @@ class NavBar extends Component {
       <div className={styles.navbar}>
         <div className={styles.left}>
           <Button
+            data-test="userListToggleButton"
             onClick={this.handleToggleUserList}
             ghost
             circle

bigbluebutton-html5/imports/ui/components/user-avatar/component.jsx

@@ -39,6 +39,7 @@ const UserAvatar = ({
   className,
 }) => (
   <div
+    data-test="userAvatar"
     className={cx(styles.avatar, {
       [styles.moderator]: moderator,
       [styles.presenter]: presenter,

bigbluebutton-html5/imports/ui/components/user-list/chat-list-item/component.jsx

@@ -65,6 +65,7 @@ const ChatListItem = (props) => {
 
   return (
     <Link
+      data-test="publicChatLink"
       to={linkPath}
       className={cx(styles.chatListItem, linkClasses)}
       role="button"

bigbluebutton-html5/imports/ui/components/user-list/user-list-content/component.jsx

@@ -37,7 +37,10 @@ const defaultProps = {
 class UserContent extends Component {
   render() {
     return (
-      <div className={styles.content}>
+      <div
+        data-test="userListContent"
+        className={styles.content}
+      >
         <UserMessages
           isPublicChat={this.props.isPublicChat}
           openChats={this.props.openChats}

bigbluebutton-html5/package.json

@@ -3,6 +3,10 @@
   "description": "BigBlueButton HTML5 Client",
   "scripts": {
     "start": "if test \"$NODE_ENV\" = \"production\" ; then npm run start:prod; else npm run start:dev; fi",
+    "test-visual-regression": "export BROWSER_NAME=firefox; wdio ./tests/webdriverio/wdio.vreg.conf.js; export BROWSER_NAME=chrome; wdio ./tests/webdriverio/wdio.vreg.conf.js; export BROWSER_NAME=chrome_mobile; DEVICE_NAME='iPhone 6 Plus'; export DEVICE_NAME; wdio ./tests/webdriverio/wdio.vreg.conf.js; DEVICE_NAME='Nexus 5X'; export DEVICE_NAME; wdio ./tests/webdriverio/wdio.vreg.conf.js",
+    "generate-refs-visual-regression": "rm -rf tests/webdriverio/screenshots; npm run test-visual-regression",
+    "test-visual-regression-desktop": "export BROWSER_NAME=firefox; wdio ./tests/webdriverio/wdio.vreg.conf.js; export BROWSER_NAME=chrome; wdio ./tests/webdriverio/wdio.vreg.conf.js",
+    "generate-refs-visual-regression-desktop": "rm -rf tests/webdriverio/screenshots; npm run test-visual-regression-desktop",
     "start:prod": "meteor reset && ROOT_URL=http://127.0.0.1/html5client NODE_ENV=production meteor --production --settings private/config/settings-production.json",
     "start:dev": "ROOT_URL=http://127.0.0.1/html5client NODE_ENV=development meteor --settings private/config/settings-development.json",
     "test": "wdio ./tests/webdriverio/wdio.conf.js",
@@ -59,6 +63,7 @@
   },
   "devDependencies": {
     "autoprefixer": "~7.1.6",
+    "babel-plugin-react-remove-properties": "~0.2.5",
     "chai": "~4.1.2",
     "eslint": "~4.9.0",
     "eslint-config-airbnb": "~16.1.0",
@@ -76,6 +81,8 @@
     "wdio-jasmine-framework": "~0.3.2",
     "wdio-junit-reporter": "~0.3.1",
     "wdio-spec-reporter": "~0.1.2",
+    "wdio-visual-regression-service": "~0.8.0",
+    "webdriver-manager": "~12.0.6",
     "webdriverio": "~4.8.0"
   },
   "cssModules": {

bigbluebutton-html5/tests/webdriverio/pageobjects/home.page.js

@@ -0,0 +1,154 @@
+'use strict';
+
+let Page = require('./page');
+let pageObject = new Page();
+let chai = require('chai');
+
+class HomePage extends Page {
+  login(username, meeting) {
+    super.open('demo/demoHTML5.jsp?username=' + username + '&meetingname=' + meeting.replace(/\s+/g, '+') + '&action=create');
+  }
+
+  get audioModalHeaderSelector() {
+    return '[data-test=audioModalHeader]';
+  }
+  get audioModalHeaderElement() {
+    return browser.element(this.audioModalSelector);
+  }
+
+  get audioModalSelector() {
+    return '.ReactModal__Content--after-open._imports_ui_components_audio_audio_modal__styles__modal';
+  }
+
+  get modalBaseCloseButtonSelector() {
+    return '[data-test=modalBaseCloseButton]';
+  }
+  get modalBaseCloseButtonElement() {
+    return $(this.modalBaseCloseButtonSelector);
+  }
+
+  get settingsDropdownTriggerSelector() {
+    return '[data-test=settingsDropdownTrigger]';
+  }
+  get settingsDropdownTriggerElement() {
+    return browser.element(this.settingsDropdownTriggerSelector);
+  }
+
+  get settingsDropdownSelector() {
+    return '[data-test=settingsDropdownTrigger] + [data-test=dropdownContent][aria-expanded="true"]';
+  }
+  get settingsDropdownElement() {
+    return browser.element(this.settingsDropdownSelector);
+  }
+
+  // Make Fullscreen button
+  get settingsDropdownFullscreenButtonSelector() {
+    return '[data-test=settingsDropdownFullscreenButton]';
+  }
+  get settingsDropdownFullscreenButtonElement() {
+    return browser.element(this.settingsDropdownFullscreenButtonSelector);
+  }
+
+  // Open Settings button
+  get settingsDropdownSettingsButtonSelector() {
+    return '[data-test=settingsDropdownSettingsButton]';
+  }
+  get settingsDropdownSettingsButtonElement() {
+    return browser.element(this.settingsDropdownSettingsButtonSelector);
+  }
+
+  // About button
+  get settingsDropdownAboutButtonSelector() {
+    return '[data-test=settingsDropdownAboutButton]';
+  }
+  get settingsDropdownAboutButtonElement() {
+    return browser.element(this.settingsDropdownAboutButtonSelector);
+  }
+
+  // Logout button
+  get settingsDropdownLogoutButtonSelector() {
+    return '[data-test=settingsDropdownLogoutButton]';
+  }
+  get settingsDropdownLogoutButtonElement() {
+    return browser.element(this.settingsDropdownLogoutButtonSelector);
+  }
+
+  // Fullscreen modal buttons
+  get modalDismissButtonSelector() {
+    return '[data-test=modalDismissButton]';
+  }
+  get modalDismissButtonElement() {
+    return browser.element(this.modalDismissButtonSelector);
+  }
+  get modalConfirmButtonSelector() {
+    return '[data-test=modalConfirmButton]';
+  }
+  get modalConfirmButtonElement() {
+    return browser.element(this.modalConfirmButtonSelector);
+  }
+
+  get logoutModalSelector() {
+    return '.ReactModal__Content--after-open._imports_ui_components_modal_fullscreen__styles__modal';
+  }
+  get logoutModalElement() {
+    return browser.element(this.logoutModalSelector);
+  }
+
+  get userListToggleButtonSelector() {
+    return '[data-test=userListToggleButton]';
+  }
+  get userListToggleButtonElement() {
+    return browser.element(this.userListToggleButtonSelector);
+  }
+
+  // User list
+  get userListContentSelector() {
+    return '[data-test=userListContent]';
+  }
+  get userListContentElement() {
+    return browser.element(this.userListContentSelector);
+  }
+
+  // User avatar icon
+  get userAvatarIconSelector() {
+    return '[data-test=userAvatarIcon]';
+  }
+  get userAvatarIconElement() {
+    return browser.element(this.userAvatarIconSelector);
+  }
+
+  // chat item that points to the Public chat
+  get publicChatLinkSelector() {
+    return '[data-test=publicChatLink]';
+  }
+  get publicChatLinkElement() {
+    return browser.element(this.publicChatLinkSelector);
+  }
+
+  // Public chat
+  get publicChatSelector() {
+    return '[data-test=publicChat]';
+  }
+  get publicChatElement() {
+    return browser.element(this.publicChatSelector);
+  }
+
+  // Chat dropdown trigger
+  get chatDropdownTriggerSelector() {
+    return '[data-test=chatDropdownTrigger]';
+  }
+  get chatDropdownTriggerElement() {
+    return browser.element(this.chatDropdownTriggerSelector);
+  }
+
+  // Chat title
+  get chatTitleSelector() {
+    return '[data-test=chatTitle]';
+  }
+  get chatTitleElement() {
+    return browser.element(this.chatTitleSelector);
+  }
+}
+
+module.exports = new HomePage();
+

bigbluebutton-html5/tests/webdriverio/specs/acceptance/login.spec.js

@@ -1,8 +1,8 @@
 'use strict';
 
-let LandingPage = require('../pageobjects/landing.page');
+let LandingPage = require('../../pageobjects/landing.page');
 let chai = require('chai');
-let utils = require('../utils');
+let utils = require('../../utils');
 
 describe('Landing page', function () {
 
@@ -21,9 +21,9 @@ describe('Landing page', function () {
 
       utils.setUsername(new Map([
         ['chromeBrowser', 'Alex'],
-        ['chromeDevBrowser', 'Anton'],
+        ['chromeBrowser', 'Anton'],
         ['firefoxBrowser', 'Danny'],
-        ['firefoxNightlyBrowser', 'Maxim'],
+        ['firefoxBrowser', 'Maxim'],
         ['chromeMobileBrowser', 'Oswaldo']
       ]));
 

bigbluebutton-html5/tests/webdriverio/specs/visual-regression/main.spec.js

@@ -0,0 +1,83 @@
+'use strict';
+
+let HomePage = require('../../pageobjects/home.page');
+let expect = require('chai').expect;
+let utils = require('../../utils');
+
+describe('Screenshots:', function() {
+  it('Join Audio modal', function() {
+    HomePage.login('testuser', 'Demo Meeting');
+    HomePage.audioModalHeaderElement.waitForExist(7000);
+    utils.expectImageMatch(browser.checkElement(HomePage.audioModalSelector), 'Join Audio modal isn\'t the same');
+  });
+
+  it('Home page viewport', function() {
+    HomePage.modalBaseCloseButtonElement.click();
+    utils.expectImageMatch(browser.checkViewport(), 'Home page viewport isn\'t the same');
+  });
+
+  //////////////////////////////
+  // Userlist + Chat
+
+  it('Userlist', function() {
+    HomePage.userListToggleButtonElement.click();
+    utils.expectImageMatch(browser.checkElement(HomePage.userListContentSelector), 'Userlist content isn\'t the same');
+  });
+
+  it('Viewport with userlist open', function() {
+    utils.expectImageMatch(browser.checkViewport(), 'Home page viewport isn\'t the same after we open userlist');
+  });
+
+  /*it('Userlist avatar', function() {
+    utils.expectImageMatch(browser.checkElement(HomePage.userAvatarElement), 'Userlist avatar isn\'t the same');
+  });*/
+
+  it('Public chat', function() {
+    HomePage.publicChatLinkElement.click();
+    utils.expectImageMatch(browser.checkElement(HomePage.publicChatSelector), 'Public chat isn\'t the same');
+  });
+
+  it('Viewport with both userlist and public chat open', function() {
+    browser.moveToObject(HomePage.chatTitleSelector); // avoid hover effect on the Public Chat tab
+    utils.expectImageMatch(browser.checkViewport(), 'Home page viewport isn\'t the same after we open both userlist and public chat');
+  });
+
+  /*it('Public chat dropdown', function() {
+    HomePage.chatDropdownTriggerElement.click();
+    utils.expectImageMatch(browser.checkElement(HomePage.publicChatSelector), 'Public chat dropdown isn\'t the same');
+  });*/
+
+  it('Public chat closes', function() {
+    HomePage.chatTitleElement.click();
+    utils.expectImageMatch(browser.checkViewport(), 'Home page viewport isn\'t the same after we closed public chat');
+  });
+
+  it('Userlist closes', function() {
+    HomePage.userListToggleButtonElement.click();
+    utils.expectImageMatch(browser.checkViewport(), 'Home page viewport isn\'t the same after we close the userlist');
+  });
+
+  //////////////////////////////
+  // Settings:
+
+  it('Settings dropdown', function() {
+    HomePage.settingsDropdownTriggerElement.waitForExist(2000);
+    HomePage.settingsDropdownTriggerElement.click();
+    HomePage.settingsDropdownElement.waitForExist(2000);
+    browser.moveToObject(HomePage.settingsDropdownLogoutButtonSelector); // avoid Options tooltip
+    utils.expectImageMatch(browser.checkElement(HomePage.settingsDropdownSelector), 'Settings dropdown isn\'t the same');
+  });
+
+  it('Logout popup', function() {
+    HomePage.settingsDropdownLogoutButtonElement.waitForExist(2000);
+    HomePage.settingsDropdownLogoutButtonElement.click();
+    HomePage.logoutModalElement.waitForExist(2000);
+    utils.expectImageMatch(browser.checkElement(HomePage.logoutModalSelector));
+  });
+
+  it('Logout popup closes', function() {
+    HomePage.modalDismissButtonElement.click();
+    utils.expectImageMatch(browser.checkViewport(), 'Home page viewport isn\'t the same after we close Logout modal');
+  });
+});
+

bigbluebutton-html5/tests/webdriverio/utils.js

@@ -1,22 +1,25 @@
 'use strict';
 
-let chai = require('chai');
+let expect = require('chai').expect;
 let LandingPage = require('./pageobjects/landing.page');
 
 class Utils {
   assertTitle(title) {
     browser.remotes.forEach(function(browserName) {
-      chai.expect(browser.select(browserName).getTitle()).to.equal(title);
+      expect(browser.select(browserName).getTitle()).to.equal(title);
     });
   }
   assertUrl(url) {
     browser.remotes.forEach(function(browserName) {
-      chai.expect(browser.getUrl()[browserName]).to.equal(url);
+      expect(browser.getUrl()[browserName]).to.equal(url);
     });
   }
   setUsername(map) {
     map.forEach((v, k) => browser.select(k).setValue(LandingPage.usernameInputSelector, v));
   }
+  expectImageMatch(results, errorMessage) {
+    results.forEach((result) => expect(result.isExactSameImage, errorMessage).to.be.true);
+  }
 }
 
 module.exports = new Utils();

bigbluebutton-html5/tests/webdriverio/wdio.accept.conf.js

@@ -0,0 +1,42 @@
+let merge = require('deepmerge');
+let wdioBaseConf = require('./wdio.base.conf');
+
+exports.config = merge(wdioBaseConf.config, {
+
+  specs: ['tests/webdriverio/specs/acceptance/**/*.spec.js'],
+
+  capabilities: {
+    chromeBrowser: {
+      desiredCapabilities: {
+        browserName: 'chrome'
+      }
+    },
+    firefoxBrowser: {
+      desiredCapabilities: {
+        browserName: 'firefox'
+      }
+    },
+    chromeMobileBrowser: {
+      desiredCapabilities: {
+        browserName: 'chrome',
+        chromeOptions: {
+          mobileEmulation: {
+            deviceName: 'iPhone 6'
+          }
+        }
+      }
+    }
+  },
+
+  suites: {
+    login: [
+      'tests/webdriverio/specs/acceptance/login.spec.js',
+    ],
+  },
+  before: function() {
+    // make the properties that browsers share and the list of browserNames available:
+    browser.remotes = Object.keys(exports.config.capabilities);
+    browser.baseUrl = exports.config.baseUrl;
+  },
+});
+

bigbluebutton-html5/tests/webdriverio/wdio.base.conf.js

@@ -0,0 +1,6 @@
+exports.config = {
+  baseUrl: 'http://localhost:8080',
+  framework: 'jasmine',
+  reporters: ['spec'],
+};
+

bigbluebutton-html5/tests/webdriverio/wdio.conf.js

@@ -1,59 +0,0 @@
-exports.config = {
-  specs: ['tests/webdriverio/specs/**/*.spec.js'],
-  capabilities: {
-    chromeBrowser: {
-      desiredCapabilities: {
-        browserName: 'chrome'
-      }
-    },
-    chromeDevBrowser: {
-      desiredCapabilities: {
-        browserName: 'chrome',
-        chromeOptions: {
-          binary: '/opt/google/chrome-unstable/google-chrome-unstable'
-        }
-      }
-    },
-    firefoxBrowser: {
-      desiredCapabilities: {
-        browserName: 'firefox'
-      }
-    },
-    firefoxNightlyBrowser: {
-      desiredCapabilities: {
-        browserName: 'firefox',
-        firefox_binary: '/usr/lib/firefox-trunk/firefox-trunk'
-      }
-    },
-    chromeMobileBrowser: {
-      desiredCapabilities: {
-        browserName: 'chrome',
-        chromeOptions: {
-          mobileEmulation: {
-            deviceName: 'Apple iPhone 6'
-          }
-        }
-      }
-    }
-  },
-  baseUrl: 'http://localhost:8080',
-  framework: 'jasmine',
-  reporters: ['spec', 'junit'],
-  reporterOptions: {
-    junit: {
-      outputDir: './tests/webdriverio/reports',
-    },
-  },
-  screenshotPath: 'screenshots',
-  suites: {
-    login: [
-      'tests/webdriverio/specs/login.spec.js',
-    ],
-  },
-  before: function() {
-    // make the properties that browsers share and the list of browserNames available:
-    browser.remotes = Object.keys(exports.config.capabilities);
-    browser.baseUrl = exports.config.baseUrl;
-  },
-};
-

bigbluebutton-html5/tests/webdriverio/wdio.vreg.conf.js

@@ -0,0 +1,71 @@
+let path = require('path');
+let VisualRegressionCompare = require('wdio-visual-regression-service/compare');
+let _ = require('lodash');
+let wdioBaseConf = require('./wdio.base.conf');
+
+function getScreenshotName(basePath) {
+  return function(context) {
+    var type = context.type;
+    var testName = context.test.title;
+    var browserVersion = parseInt(context.browser.version, 10);
+    var browserName = process.env.BROWSER_NAME === 'chrome_mobile' ? process.env.DEVICE_NAME : context.browser.name;
+    var browserViewport = context.meta.viewport;
+    var browserWidth = browserViewport.width;
+    var browserHeight = browserViewport.height;
+    return path.join(
+      basePath,
+      browserName,
+      `${browserWidth}x${browserHeight}`,
+      `${testName}_${type}.png`
+    );
+  };
+}
+
+exports.config = _.merge(wdioBaseConf.config, {
+  specs: [
+    'tests/webdriverio/specs/visual-regression/**/*.spec.js'
+  ],
+
+  capabilities: [process.env.BROWSER_NAME=='chrome_mobile' ? {
+    maxInstances: 5,
+    browserName: 'chrome',
+    chromeOptions: {
+      mobileEmulation: {
+        deviceName: process.env.DEVICE_NAME
+      }
+    }
+  } : {
+    maxInstances: 5,
+    browserName: process.env.BROWSER_NAME
+  }],
+
+  baseUrl: 'http://localhost:8080',
+  framework: 'jasmine',
+  reporters: ['spec'],
+
+  sync: true,
+  logLevel: 'silent',
+  bail: 0,
+  host: 'localhost',
+  port: 4444,
+  waitforTimeout: 30000,
+  connectionRetryTimeout: 90000,
+  connectionRetryCount: 3,
+  services: ['visual-regression'],
+  visualRegression: {
+    compare: new VisualRegressionCompare.LocalCompare({
+      referenceName: getScreenshotName(path.join(process.cwd(), 'tests/webdriverio/screenshots/reference')),
+      screenshotName: getScreenshotName(path.join(process.cwd(), 'tests/webdriverio/screenshots/screen')),
+      diffName: getScreenshotName(path.join(process.cwd(), 'tests/webdriverio/screenshots/diff')),
+      misMatchTolerance: 0.01,
+    }),
+    viewports: process.env.BROWSER_NAME === 'chrome_mobile' ? [] : [{ width: 1920, height: 1200 }, { width: 960, height: 1200 }],
+    viewportChangePause: 300,
+    orientations: ['landscape'],
+  },
+
+  jasmineNodeOpts: {
+    defaultTimeoutInterval: 30000
+  },
+});
+