restructure: move bin/ and tests/ to src/

Also add symlinks from the old `bin/` and `tests/` locations to avoid
breaking scripts and other tools.

Motivations:

  * Scripts and tests no longer have to do dubious things like:

        require('ep_etherpad-lite/node_modules/foo')

    to access packages installed as dependencies in
    `src/package.json`.

  * Plugins can access the backend test helper library in a non-hacky
    way:

        require('ep_etherpad-lite/tests/backend/common')

  * We can delete the top-level `package.json` without breaking our
    ability to lint the files in `bin/` and `tests/`.

    Deleting the top-level `package.json` has downsides: It will cause
    `npm` to print warnings whenever plugins are installed, npm will
    no longer be able to enforce a plugin's peer dependency on
    ep_etherpad-lite, and npm will keep deleting the
    `node_modules/ep_etherpad-lite` symlink that points to `../src`.

    But there are significant upsides to deleting the top-level
    `package.json`: It will drastically speed up plugin installation
    because `npm` doesn't have to recursively walk the dependencies in
    `src/package.json`. Also, deleting the top-level `package.json`
    avoids npm's horrible dependency hoisting behavior (where it moves
    stuff from `src/node_modules/` to the top-level `node_modules/`
    directory). Dependency hoisting causes numerous mysterious
    problems such as silent failures in `npm outdated` and `npm
    update`. Dependency hoisting also breaks plugins that do:

        require('ep_etherpad-lite/node_modules/foo')

src/tests/frontend/travis/.gitignore

@@ -0,0 +1,2 @@
+sauce_connect.log
+sauce_connect.log.*

src/tests/frontend/travis/remote_runner.js

@@ -0,0 +1,183 @@
+'use strict';
+
+const async = require('async');
+const wd = require('wd');
+
+const config = {
+  host: 'ondemand.saucelabs.com',
+  port: 80,
+  username: process.env.SAUCE_USER,
+  accessKey: process.env.SAUCE_ACCESS_KEY,
+};
+
+let allTestsPassed = true;
+// overwrite the default exit code
+// in case not all worker can be run (due to saucelabs limits),
+// `queue.drain` below will not be called
+// and the script would silently exit with error code 0
+process.exitCode = 2;
+process.on('exit', (code) => {
+  if (code === 2) {
+    console.log('\x1B[31mFAILED\x1B[39m Not all saucelabs runner have been started.');
+  }
+});
+
+const sauceTestWorker = async.queue((testSettings, callback) => {
+  const browser = wd.promiseChainRemote(
+      config.host, config.port, config.username, config.accessKey);
+  const name =
+      `${process.env.GIT_HASH} - ${testSettings.browserName} ` +
+      `${testSettings.version}, ${testSettings.platform}`;
+  testSettings.name = name;
+  testSettings.public = true;
+  testSettings.build = process.env.GIT_HASH;
+  // console.json can be downloaded via saucelabs,
+  // don't know how to print them into output of the tests
+  testSettings.extendedDebugging = true;
+  testSettings.tunnelIdentifier = process.env.TRAVIS_JOB_NUMBER;
+
+  browser.init(testSettings).get('http://localhost:9001/tests/frontend/', () => {
+    const url = `https://saucelabs.com/jobs/${browser.sessionID}`;
+    console.log(`Remote sauce test '${name}' started! ${url}`);
+
+    // tear down the test excecution
+    const stopSauce = (success, timesup) => {
+      clearInterval(getStatusInterval);
+      clearTimeout(timeout);
+
+      browser.quit(() => {
+        if (!success) {
+          allTestsPassed = false;
+        }
+
+        // if stopSauce is called via timeout
+        // (in contrast to via getStatusInterval) than the log of up to the last
+        // five seconds may not be available here. It's an error anyway, so don't care about it.
+        printLog(logIndex);
+
+        if (timesup) {
+          console.log(`[${testSettings.browserName} ${testSettings.platform}` +
+            `${testSettings.version === '' ? '' : (` ${testSettings.version}`)}]` +
+            ' \x1B[31mFAILED\x1B[39m allowed test duration exceeded');
+        }
+        console.log(`Remote sauce test '${name}' finished! ${url}`);
+
+        callback();
+      });
+    };
+
+    /**
+       * timeout if a test hangs or the job exceeds 14.5 minutes
+       * It's necessary because if travis kills the saucelabs session due to inactivity,
+       * we don't get any output
+       * @todo this should be configured in testSettings, see
+       * https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-Timeouts
+       */
+    const timeout = setTimeout(() => {
+      stopSauce(false, true);
+    }, 870000); // travis timeout is 15 minutes, set this to a slightly lower value
+
+    let knownConsoleText = '';
+    // how many characters of the log have been sent to travis
+    let logIndex = 0;
+    const getStatusInterval = setInterval(() => {
+      browser.eval("$('#console').text()", (err, consoleText) => {
+        if (!consoleText || err) {
+          return;
+        }
+        knownConsoleText = consoleText;
+
+        if (knownConsoleText.indexOf('FINISHED') > 0) {
+          const match = knownConsoleText.match(
+              /FINISHED.*([0-9]+) tests passed, ([0-9]+) tests failed/);
+          // finished without failures
+          if (match[2] && match[2] === '0') {
+            stopSauce(true);
+
+            // finished but some tests did not return or some tests failed
+          } else {
+            stopSauce(false);
+          }
+        } else {
+          // not finished yet
+          printLog(logIndex);
+          logIndex = knownConsoleText.length;
+        }
+      });
+    }, 5000);
+
+    /**
+       * Replaces color codes in the test runners log, appends
+       * browser name, platform etc. to every line and prints them.
+       *
+       * @param {number} index offset from where to start
+       */
+    const printLog = (index) => {
+      let testResult = knownConsoleText.substring(index)
+          .replace(/\[red\]/g, '\x1B[31m').replace(/\[yellow\]/g, '\x1B[33m')
+          .replace(/\[green\]/g, '\x1B[32m').replace(/\[clear\]/g, '\x1B[39m');
+      testResult = testResult.split('\\n').map((line) => `[${testSettings.browserName} ` +
+        `${testSettings.platform}` +
+        `${testSettings.version === '' ? '' : (` ${testSettings.version}`)}]` +
+        `${line}`).join('\n');
+
+      console.log(testResult);
+    };
+  });
+}, 6); // run 6 tests in parrallel
+
+// 1) Firefox on Linux
+sauceTestWorker.push({
+  platform: 'Windows 7',
+  browserName: 'firefox',
+  version: '52.0',
+});
+
+// 2) Chrome on Linux
+sauceTestWorker.push({
+  platform: 'Windows 7',
+  browserName: 'chrome',
+  version: '55.0',
+  args: ['--use-fake-device-for-media-stream'],
+});
+
+/*
+// 3) Safari on OSX 10.15
+sauceTestWorker.push({
+    'platform'       : 'OS X 10.15'
+  , 'browserName'    : 'safari'
+  , 'version'        : '13.1'
+});
+*/
+
+// 4) Safari on OSX 10.14
+sauceTestWorker.push({
+  platform: 'OS X 10.15',
+  browserName: 'safari',
+  version: '13.1',
+});
+// IE 10 doesn't appear to be working anyway
+/*
+// 4) IE 10 on Win 8
+sauceTestWorker.push({
+    'platform'       : 'Windows 8'
+  , 'browserName'    : 'iexplore'
+  , 'version'        : '10.0'
+});
+*/
+// 5) Edge on Win 10
+sauceTestWorker.push({
+  platform: 'Windows 10',
+  browserName: 'microsoftedge',
+  version: '83.0',
+});
+// 6) Firefox on Win 7
+sauceTestWorker.push({
+  platform: 'Windows 7',
+  browserName: 'firefox',
+  version: '78.0',
+});
+
+sauceTestWorker.drain(() => {
+  process.exit(allTestsPassed ? 0 : 1);
+});

src/tests/frontend/travis/runner.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+try() { "$@" || fatal "'$@' failed"; }
+
+[ -n "${SAUCE_USERNAME}" ] || fatal "SAUCE_USERNAME is unset - exiting"
+[ -n "${SAUCE_ACCESS_KEY}" ] || fatal "SAUCE_ACCESS_KEY is unset - exiting"
+
+MY_DIR=$(try cd "${0%/*}" && try pwd) || exit 1
+
+# reliably move to the etherpad base folder before running it
+try cd "${MY_DIR}/../../../"
+
+log "Assuming bin/installDeps.sh has already been run"
+node node_modules/ep_etherpad-lite/node/server.js --experimental-worker "${@}" &
+ep_pid=$!
+
+log "Waiting for Etherpad to accept connections (http://localhost:9001)..."
+connected=false
+can_connect() {
+    curl -sSfo /dev/null http://localhost:9001/ || return 1
+    connected=true
+}
+now() { date +%s; }
+start=$(now)
+while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do
+    sleep 1
+done
+[ "$connected" = true ] \
+    || fatal "Timed out waiting for Etherpad to accept connections"
+log "Successfully connected to Etherpad on http://localhost:9001"
+
+# start the remote runner
+try cd "${MY_DIR}"
+log "Starting the remote runner..."
+node remote_runner.js
+exit_code=$?
+
+kill "$(cat /tmp/sauce.pid)"
+kill "$ep_pid" && wait "$ep_pid"
+log "Done."
+exit "$exit_code"

src/tests/frontend/travis/runnerBackend.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+try() { "$@" || fatal "'$@' failed"; }
+
+MY_DIR=$(try cd "${0%/*}" && try pwd) || fatal "failed to find script directory"
+
+# reliably move to the etherpad base folder before running it
+try cd "${MY_DIR}/../../../"
+
+try sed -e '
+s!"soffice":[^,]*!"soffice": "/usr/bin/soffice"!
+# Reduce rate limit aggressiveness
+s!"max":[^,]*!"max": 100!
+s!"points":[^,]*!"points": 1000!
+# GitHub does not like our output
+s!"loglevel":[^,]*!"loglevel": "WARN"!
+' settings.json.template >settings.json
+
+log "Assuming bin/installDeps.sh has already been run"
+node node_modules/ep_etherpad-lite/node/server.js "${@}" &
+ep_pid=$!
+
+log "Waiting for Etherpad to accept connections (http://localhost:9001)..."
+connected=false
+can_connect() {
+    curl -sSfo /dev/null http://localhost:9001/ || return 1
+    connected=true
+}
+now() { date +%s; }
+start=$(now)
+while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do
+    sleep 1
+done
+[ "$connected" = true ] \
+    || fatal "Timed out waiting for Etherpad to accept connections"
+log "Successfully connected to Etherpad on http://localhost:9001"
+
+log "Running the backend tests..."
+try cd src
+npm test
+exit_code=$?
+
+kill "$ep_pid" && wait "$ep_pid"
+log "Done."
+exit "$exit_code"

src/tests/frontend/travis/runnerLoadTest.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+try() { "$@" || fatal "'$@' failed"; }
+
+MY_DIR=$(try cd "${0%/*}" && try pwd) || exit 1
+
+# reliably move to the etherpad base folder before running it
+try cd "${MY_DIR}/../../../"
+
+try sed -e '
+s!"loadTest":[^,]*!"loadTest": true!
+# Reduce rate limit aggressiveness
+s!"points":[^,]*!"points": 1000!
+' settings.json.template >settings.json
+
+log "Assuming bin/installDeps.sh has already been run"
+node node_modules/ep_etherpad-lite/node/server.js "${@}" >/dev/null &
+ep_pid=$!
+
+log "Waiting for Etherpad to accept connections (http://localhost:9001)..."
+connected=false
+can_connect() {
+    curl -sSfo /dev/null http://localhost:9001/ || return 1
+    connected=true
+}
+now() { date +%s; }
+start=$(now)
+while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do
+    sleep 1
+done
+[ "$connected" = true ] \
+    || fatal "Timed out waiting for Etherpad to accept connections"
+log "Successfully connected to Etherpad on http://localhost:9001"
+
+# Build the minified files
+try curl http://localhost:9001/p/minifyme -f -s >/dev/null
+
+# just in case, let's wait for another 10 seconds before going on
+sleep 10
+
+log "Running the load tests..."
+etherpad-loadtest -d 25
+exit_code=$?
+
+kill "$ep_pid" && wait "$ep_pid"
+log "Done."
+exit "$exit_code"

src/tests/frontend/travis/sauce_tunnel.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+try() { "$@" || fatal "'$@' failed"; }
+
+[ -n "${SAUCE_USERNAME}" ] || fatal "SAUCE_USERNAME is unset - exiting"
+[ -n "${SAUCE_ACCESS_KEY}" ] || fatal "SAUCE_ACCESS_KEY is unset - exiting"
+
+# download and unzip the sauce connector
+#
+# ACHTUNG: as of 2019-12-21, downloading sc-latest-linux.tar.gz does not work.
+# It is necessary to explicitly download a specific version, for example
+# https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz Supported versions are
+# currently listed at:
+# https://wiki.saucelabs.com/display/DOCS/Downloading+Sauce+Connect+Proxy
+try curl -o /tmp/sauce.tar.gz \
+    https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz
+try tar zxf /tmp/sauce.tar.gz --directory /tmp
+try mv /tmp/sc-*-linux /tmp/sauce_connect
+
+# start the sauce connector in background and make sure it doesn't output the
+# secret key
+try rm -f /tmp/tunnel
+/tmp/sauce_connect/bin/sc \
+    --user "${SAUCE_USERNAME}" \
+    --key "${SAUCE_ACCESS_KEY}" \
+    -i "${TRAVIS_JOB_NUMBER}" \
+    --pidfile /tmp/sauce.pid \
+    --readyfile /tmp/tunnel >/dev/null &
+
+# wait for the tunnel to build up
+while ! [ -e "/tmp/tunnel" ]; do
+    sleep 1
+done