restructure: move bin/ and tests/ to src/

Also add symlinks from the old `bin/` and `tests/` locations to avoid
breaking scripts and other tools.

Motivations:

  * Scripts and tests no longer have to do dubious things like:

        require('ep_etherpad-lite/node_modules/foo')

    to access packages installed as dependencies in
    `src/package.json`.

  * Plugins can access the backend test helper library in a non-hacky
    way:

        require('ep_etherpad-lite/tests/backend/common')

  * We can delete the top-level `package.json` without breaking our
    ability to lint the files in `bin/` and `tests/`.

    Deleting the top-level `package.json` has downsides: It will cause
    `npm` to print warnings whenever plugins are installed, npm will
    no longer be able to enforce a plugin's peer dependency on
    ep_etherpad-lite, and npm will keep deleting the
    `node_modules/ep_etherpad-lite` symlink that points to `../src`.

    But there are significant upsides to deleting the top-level
    `package.json`: It will drastically speed up plugin installation
    because `npm` doesn't have to recursively walk the dependencies in
    `src/package.json`. Also, deleting the top-level `package.json`
    avoids npm's horrible dependency hoisting behavior (where it moves
    stuff from `src/node_modules/` to the top-level `node_modules/`
    directory). Dependency hoisting causes numerous mysterious
    problems such as silent failures in `npm outdated` and `npm
    update`. Dependency hoisting also breaks plugins that do:

        require('ep_etherpad-lite/node_modules/foo')

src/bin/buildDebian.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# IMPORTANT
+# Protect against misspelling a var and rm -rf /
+set -u
+set -e
+
+SRC=/tmp/etherpad-deb-src
+DIST=/tmp/etherpad-deb-dist
+SYSROOT=${SRC}/sysroot
+DEBIAN=${SRC}/DEBIAN
+
+rm -rf ${DIST}
+mkdir -p ${DIST}/
+
+rm -rf ${SRC}
+rsync -a bin/deb-src/ ${SRC}/
+mkdir -p ${SYSROOT}/opt/
+
+rsync --exclude '.git' -a . ${SYSROOT}/opt/etherpad/ --delete
+mkdir -p ${SYSROOT}/usr/share/doc
+cp README.md ${SYSROOT}/usr/share/doc/etherpad
+find ${SRC}/ -type d -exec chmod 0755 {} \;
+find ${SRC}/ -type f -exec chmod go-w {} \;
+chown -R root:root ${SRC}/
+
+let SIZE=$(du -s ${SYSROOT} | sed s'/\s\+.*//')+8
+pushd ${SYSROOT}/
+tar czf ${DIST}/data.tar.gz [a-z]*
+popd
+sed s"/SIZE/${SIZE}/" -i ${DEBIAN}/control
+pushd ${DEBIAN}
+tar czf ${DIST}/control.tar.gz *
+popd
+
+pushd ${DIST}/
+echo 2.0 > ./debian-binary
+
+find ${DIST}/ -type d -exec chmod 0755 {} \;
+find ${DIST}/ -type f -exec chmod go-w {} \;
+chown -R root:root ${DIST}/
+ar r ${DIST}/etherpad-1.deb debian-binary control.tar.gz data.tar.gz
+popd
+rsync -a ${DIST}/etherpad-1.deb ./

src/bin/buildForWindows.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Is wget installed?
+is_cmd wget || fatal "Please install wget"
+
+# Is zip installed?
+is_cmd zip || fatal "Please install zip"
+
+# Is zip installed?
+is_cmd unzip || fatal "Please install unzip"
+
+START_FOLDER=$(pwd);
+TMP_FOLDER=$(mktemp -d)
+
+log "create a clean environment in $TMP_FOLDER..."
+cp -ar . "$TMP_FOLDER"
+cd "$TMP_FOLDER"
+rm -rf node_modules
+rm -f etherpad-lite-win.zip
+
+# setting NODE_ENV=production ensures that dev dependencies are not installed,
+# making the windows package smaller
+export NODE_ENV=production
+
+log "do a normal unix install first..."
+bin/installDeps.sh || exit 1
+
+log "copy the windows settings template..."
+cp settings.json.template settings.json
+
+log "resolve symbolic links..."
+cp -rL node_modules node_modules_resolved
+rm -rf node_modules
+mv node_modules_resolved node_modules
+
+log "download windows node..."
+cd bin
+wget "https://nodejs.org/dist/latest-erbium/win-x86/node.exe" -O ../node.exe
+
+log "remove git history to reduce folder size"
+rm -rf .git/objects
+
+log "remove windows jsdom-nocontextify/test folder"
+rm -rf "$TMP_FOLDER"/src/node_modules/wd/node_modules/request/node_modules/form-data/node_modules/combined-stream/test
+rm -rf "$TMP_FOLDER"/src/node_modules/nodemailer/node_modules/mailcomposer/node_modules/mimelib/node_modules/encoding/node_modules/iconv-lite/encodings/tables
+
+log "create the zip..."
+cd "$TMP_FOLDER"
+zip -9 -r "$START_FOLDER"/etherpad-lite-win.zip ./*
+
+log "clean up..."
+rm -rf "$TMP_FOLDER"
+
+log "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"

src/bin/checkAllPads.js

@@ -0,0 +1,88 @@
+'use strict';
+/*
+ * This is a debug tool. It checks all revisions for data corruption
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js');
+
+(async () => {
+  // initialize the database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load modules
+  const Changeset = require('../static/js/Changeset');
+  const padManager = require('../node/db/PadManager');
+
+  let revTestedCount = 0;
+
+  // get all pads
+  const res = await padManager.listAllPads();
+  for (const padId of res.padIDs) {
+    const pad = await padManager.getPad(padId);
+
+    // check if the pad has a pool
+    if (pad.pool == null) {
+      console.error(`[${pad.id}] Missing attribute pool`);
+      continue;
+    }
+    // create an array with key kevisions
+    // key revisions always save the full pad atext
+    const head = pad.getHeadRevisionNumber();
+    const keyRevisions = [];
+    for (let rev = 0; rev < head; rev += 100) {
+      keyRevisions.push(rev);
+    }
+
+    // run through all key revisions
+    for (const keyRev of keyRevisions) {
+      // create an array of revisions we need till the next keyRevision or the End
+      const revisionsNeeded = [];
+      for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
+        revisionsNeeded.push(rev);
+      }
+
+      // this array will hold all revision changesets
+      const revisions = [];
+
+      // run through all needed revisions and get them from the database
+      for (const revNum of revisionsNeeded) {
+        const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
+        revisions[revNum] = revision;
+      }
+
+      // check if the revision exists
+      if (revisions[keyRev] == null) {
+        console.error(`[${pad.id}] Missing revision ${keyRev}`);
+        continue;
+      }
+
+      // check if there is a atext in the keyRevisions
+      let {meta: {atext} = {}} = revisions[keyRev];
+      if (atext == null) {
+        console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
+        continue;
+      }
+
+      const apool = pad.pool;
+      for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
+        try {
+          const cs = revisions[rev].changeset;
+          atext = Changeset.applyToAText(cs, atext, apool);
+          revTestedCount++;
+        } catch (e) {
+          console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
+        }
+      }
+    }
+  }
+  if (revTestedCount === 0) {
+    throw new Error('No revisions tested');
+  }
+  console.log(`Finished: Tested ${revTestedCount} revisions`);
+})();

src/bin/checkPad.js

@@ -0,0 +1,82 @@
+'use strict';
+/*
+ * This is a debug tool. It checks all revisions for data corruption
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+let checkRevisionCount = 0;
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load modules
+  const Changeset = require('../static/js/Changeset');
+  const padManager = require('../node/db/PadManager');
+
+  const exists = await padManager.doesPadExists(padId);
+  if (!exists) throw new Error('Pad does not exist');
+
+  // get the pad
+  const pad = await padManager.getPad(padId);
+
+  // create an array with key revisions
+  // key revisions always save the full pad atext
+  const head = pad.getHeadRevisionNumber();
+  const keyRevisions = [];
+  for (let rev = 0; rev < head; rev += 100) {
+    keyRevisions.push(rev);
+  }
+
+  // run through all key revisions
+  for (let keyRev of keyRevisions) {
+    keyRev = parseInt(keyRev);
+    // create an array of revisions we need till the next keyRevision or the End
+    const revisionsNeeded = [];
+    for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
+      revisionsNeeded.push(rev);
+    }
+
+    // this array will hold all revision changesets
+    const revisions = [];
+
+    // run through all needed revisions and get them from the database
+    for (const revNum of revisionsNeeded) {
+      const revision = await db.get(`pad:${padId}:revs:${revNum}`);
+      revisions[revNum] = revision;
+    }
+
+    // check if the pad has a pool
+    if (pad.pool == null) throw new Error('Attribute pool is missing');
+
+    // check if there is an atext in the keyRevisions
+    let {meta: {atext} = {}} = revisions[keyRev] || {};
+    if (atext == null) {
+      console.error(`No atext in key revision ${keyRev}`);
+      continue;
+    }
+
+    const apool = pad.pool;
+
+    for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
+      checkRevisionCount++;
+      try {
+        const cs = revisions[rev].changeset;
+        atext = Changeset.applyToAText(cs, atext, apool);
+      } catch (e) {
+        console.error(`Bad changeset at revision ${rev} - ${e.message}`);
+        continue;
+      }
+    }
+    console.log(`Finished: Checked ${checkRevisionCount} revisions`);
+  }
+})();

src/bin/checkPadDeltas.js

@@ -0,0 +1,103 @@
+'use strict';
+/*
+ * This is a debug tool. It checks all revisions for data corruption
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 3) throw new Error('Use: node bin/checkPadDeltas.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+const expect = require('../tests/frontend/lib/expect');
+const diff = require('diff');
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load modules
+  const Changeset = require('../static/js/Changeset');
+  const padManager = require('../node/db/PadManager');
+
+  const exists = await padManager.doesPadExists(padId);
+  if (!exists) throw new Error('Pad does not exist');
+
+  // get the pad
+  const pad = await padManager.getPad(padId);
+
+  // create an array with key revisions
+  // key revisions always save the full pad atext
+  const head = pad.getHeadRevisionNumber();
+  const keyRevisions = [];
+  for (let i = 0; i < head; i += 100) {
+    keyRevisions.push(i);
+  }
+
+  // create an array with all revisions
+  const revisions = [];
+  for (let i = 0; i <= head; i++) {
+    revisions.push(i);
+  }
+
+  let atext = Changeset.makeAText('\n');
+
+  // run through all revisions
+  for (const revNum of revisions) {
+    // console.log('Fetching', revNum)
+    const revision = await db.get(`pad:${padId}:revs:${revNum}`);
+    // check if there is a atext in the keyRevisions
+    const {meta: {atext: revAtext} = {}} = revision || {};
+    if (~keyRevisions.indexOf(revNum) && revAtext == null) {
+      console.error(`No atext in key revision ${revNum}`);
+      continue;
+    }
+
+    // try glue everything together
+    try {
+      // console.log("check revision ", revNum);
+      const cs = revision.changeset;
+      atext = Changeset.applyToAText(cs, atext, pad.pool);
+    } catch (e) {
+      console.error(`Bad changeset at revision ${revNum} - ${e.message}`);
+      continue;
+    }
+
+    // check things are working properly
+    if (~keyRevisions.indexOf(revNum)) {
+      try {
+        expect(revision.meta.atext.text).to.eql(atext.text);
+        expect(revision.meta.atext.attribs).to.eql(atext.attribs);
+      } catch (e) {
+        console.error(`Atext in key revision ${revNum} doesn't match computed one.`);
+        console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => {
+          if (!op.added && !op.removed) op.value = op.value.length;
+          return op;
+        }));
+        // console.error(e)
+        // console.log('KeyRev. :', revision.meta.atext)
+        // console.log('Computed:', atext)
+        continue;
+      }
+    }
+  }
+
+  // check final text is right...
+  if (pad.atext.text === atext.text) {
+    console.log('ok');
+  } else {
+    console.error('Pad AText doesn\'t match computed one! (Computed ',
+        atext.text.length, ', db', pad.atext.text.length, ')');
+    console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => {
+      if (!op.added && !op.removed) {
+        op.value = op.value.length;
+        return op;
+      }
+    }));
+  }
+})();

src/bin/cleanRun.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+#Was this script started in the bin folder? if yes move out
+if [ -d "../bin" ]; then
+  cd "../"
+fi
+
+ignoreRoot=0
+for ARG in "$@"
+do
+  if [ "$ARG" = "--root" ]; then
+    ignoreRoot=1
+  fi
+done
+
+#Stop the script if it's started as root
+if [ "$(id -u)" -eq 0 ] && [ $ignoreRoot -eq 0 ]; then
+   echo "You shouldn't start Etherpad as root!"
+   echo "Please type 'Etherpad rocks my socks' or supply the '--root' argument if you still want to start it as root"
+   read rocks
+   if [ ! $rocks = "Etherpad rocks my socks" ]
+   then
+     echo "Your input was incorrect"
+     exit 1
+   fi
+fi
+
+#Clean the current environment
+rm -rf src/node_modules
+
+#Prepare the environment
+bin/installDeps.sh "$@" || exit 1
+
+#Move to the node folder and start
+echo "Started Etherpad..."
+
+SCRIPTPATH=$(pwd -P)
+node $(compute_node_args) "${SCRIPTPATH}/node_modules/ep_etherpad-lite/node/server.js" "$@"

src/bin/convertSettings.json.template

@@ -0,0 +1,10 @@
+{
+  "etherpadDB":
+  {
+    "host": "localhost",
+    "port": 3306,
+    "database": "etherpad",
+    "user": "etherpaduser",
+    "password": "yourpassword"
+  }
+}

src/bin/createRelease.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+#
+# WARNING: since Etherpad 1.7.0 (2018-08-17), this script is DEPRECATED, and
+#          will be removed/modified in a future version.
+#          It's left here just for documentation.
+#          The branching policies for releases have been changed.
+#
+# This script is used to publish a new release/version of etherpad on github
+#
+# Work that is done by this script:
+# ETHER_REPO:
+# - Add text to CHANGELOG.md
+# - Replace version of etherpad in src/package.json
+# - Create a release branch and push it to github
+# - Merges this release branch into master branch
+# - Creating the windows build and the docs
+# ETHER_WEB_REPO:
+# - Creating a new branch with the docs and the windows build
+# - Replacing the version numbers in the index.html
+# - Push this branch and merge it to master
+# ETHER_REPO:
+# - Create a new release on github
+
+printf "WARNING: since Etherpad 1.7.0 this script is DEPRECATED, and will be removed/modified in a future version.\n\n"
+while true; do
+    read -p "Do you want to continue? This is discouraged. [y/N]" yn
+    case $yn in
+        [Yy]* ) break;;
+        [Nn]* ) exit;;
+        * ) printf "Please answer yes or no.\n\n";;
+    esac
+done
+
+ETHER_REPO="https://github.com/ether/etherpad-lite.git"
+ETHER_WEB_REPO="https://github.com/ether/ether.github.com.git"
+TMP_DIR="/tmp/"
+
+echo "WARNING: You can only run this script if your github api token is allowed to create and merge branches on $ETHER_REPO and $ETHER_WEB_REPO."
+echo "This script automatically changes the version number in package.json and adds a text to CHANGELOG.md."
+echo "When you use this script you should be in the branch that you want to release (develop probably) on latest version. Any changes that are currently not committed will be committed."
+echo "-----"
+
+# Get the latest version
+LATEST_GIT_TAG=$(git tag | tail -n 1)
+
+# Current environment
+echo "Current environment: "
+echo "- branch: $(git branch | grep '* ')"
+echo "- last commit date: $(git show --quiet --pretty=format:%ad)"
+echo "- current version: $LATEST_GIT_TAG"
+echo "- temp dir: $TMP_DIR"
+
+# Get new version number
+# format: x.x.x
+echo -n "Enter new version (x.x.x): "
+read VERSION
+
+# Get the message for the changelogs
+read -p "Enter new changelog entries (press enter): "
+tmp=$(mktemp)
+"${EDITOR:-vi}" $tmp
+changelogText=$(<$tmp)
+echo "$changelogText"
+rm $tmp
+
+if [ "$changelogText" != "" ]; then
+  changelogText="# $VERSION\n$changelogText"
+fi
+
+# get the token for the github api
+echo -n "Enter your github api token: "
+read API_TOKEN
+
+function check_api_token {
+  echo "Checking if github api token is valid..."
+  CURL_RESPONSE=$(curl --silent -i https://api.github.com/user?access_token=$API_TOKEN | iconv -f utf8)
+  HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+  [[ $HTTP_STATUS != "200" ]] && echo "Aborting: Invalid github api token" && exit 1
+}
+
+function modify_files {
+  # Add changelog text to first line of CHANGELOG.md
+
+  msg=""
+  # source: https://unix.stackexchange.com/questions/9784/how-can-i-read-line-by-line-from-a-variable-in-bash#9789
+  while IFS= read -r line
+  do
+    # replace newlines with literal "\n" for using with sed
+    msg+="$line\n"
+  done < <(printf '%s\n' "${changelogText}")
+
+  sed -i "1s/^/${msg}\n/" CHANGELOG.md
+  [[ $? != 0 ]] && echo "Aborting: Error modifying CHANGELOG.md" && exit 1
+
+  # Replace version number of etherpad in package.json
+  sed -i -r "s/(\"version\"[ ]*: \").*(\")/\1$VERSION\2/" src/package.json
+  [[ $? != 0 ]] && echo "Aborting: Error modifying package.json" && exit 1
+}
+
+function create_release_branch {
+  echo "Creating new release branch..."
+  git rev-parse --verify release/$VERSION 2>/dev/null
+  if [ $? == 0 ]; then
+    echo "Aborting: Release branch already present"
+    exit 1
+  fi
+  git checkout -b release/$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error creating release branch" && exit 1
+
+  echo "Committing CHANGELOG.md and package.json"
+  git add CHANGELOG.md
+  git add src/package.json
+  git commit -m "Release version $VERSION"
+
+  echo "Pushing release branch to github..."
+  git push -u $ETHER_REPO release/$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error pushing release branch to github" && exit 1
+}
+
+function merge_release_branch {
+  echo "Merging release to master branch on github..."
+  API_JSON=$(printf '{"base": "master","head": "release/%s","commit_message": "Merge new release into master branch!"}' $VERSION)
+  CURL_RESPONSE=$(curl --silent -i -N --data "$API_JSON" https://api.github.com/repos/ether/etherpad-lite/merges?access_token=$API_TOKEN  | iconv -f utf8)
+  echo $CURL_RESPONSE
+  HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+  [[ $HTTP_STATUS != "200" ]] && echo "Aborting: Error merging release branch on github" && exit 1
+}
+
+function create_builds {
+  echo "Cloning etherpad-lite repo and ether.github.com repo..."
+  cd $TMP_DIR
+  rm -rf etherpad-lite ether.github.com
+  git clone $ETHER_REPO --branch master
+  git clone $ETHER_WEB_REPO
+  echo "Creating windows build..."
+  cd etherpad-lite
+  bin/buildForWindows.sh
+  [[ $? != 0 ]] && echo "Aborting: Error creating build for windows" && exit 1
+  echo "Creating docs..."
+  make docs
+  [[ $? != 0 ]] && echo "Aborting: Error generating docs" && exit 1
+}
+
+function push_builds {
+  cd $TMP_DIR/etherpad-lite/
+  echo "Copying windows build and docs to website repo..."
+  GIT_SHA=$(git rev-parse HEAD | cut -c1-10)
+  mv etherpad-lite-win.zip $TMP_DIR/ether.github.com/downloads/etherpad-lite-win-$VERSION-$GIT_SHA.zip
+
+  mv out/doc $TMP_DIR/ether.github.com/doc/v$VERSION
+
+  cd $TMP_DIR/ether.github.com/
+  sed -i "s/etherpad-lite-win.*\.zip/etherpad-lite-win-$VERSION-$GIT_SHA.zip/" index.html
+  sed -i "s/$LATEST_GIT_TAG/$VERSION/g" index.html
+  git checkout -b release_$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error creating new release branch" && exit 1
+  git add doc/
+  git add downloads/
+  git commit -a -m "Release version $VERSION"
+  git push -u $ETHER_WEB_REPO release_$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error pushing release branch to github" && exit 1
+}
+
+function merge_web_branch {
+        echo "Merging release to master branch on github..."
+        API_JSON=$(printf '{"base": "master","head": "release_%s","commit_message": "Release version %s"}' $VERSION $VERSION)
+         CURL_RESPONSE=$(curl --silent -i -N --data "$API_JSON" https://api.github.com/repos/ether/ether.github.com/merges?access_token=$API_TOKEN | iconv -f utf8)
+  echo $CURL_RESPONSE
+  HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+  [[ $HTTP_STATUS != "200" ]] && echo "Aborting: Error merging release branch" && exit 1
+}
+
+function publish_release {
+  echo -n "Do you want to publish a new release on github (y/n)? "
+  read PUBLISH_RELEASE
+  if [ $PUBLISH_RELEASE = "y" ]; then
+    # create a new release on github
+    API_JSON=$(printf '{"tag_name": "%s","target_commitish": "master","name": "Release %s","body": "%s","draft": false,"prerelease": false}' $VERSION $VERSION $changelogText)
+    CURL_RESPONSE=$(curl --silent -i -N --data "$API_JSON" https://api.github.com/repos/ether/etherpad-lite/releases?access_token=$API_TOKEN | iconv -f utf8)
+    HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+    [[ $HTTP_STATUS != "201" ]] && echo "Aborting: Error publishing release on github" && exit 1
+  else
+    echo "No release published on github!"
+  fi
+}
+
+function todo_notification {
+  echo "Release procedure was successful, but you have to do some steps manually:"
+  echo "- Update the wiki at https://github.com/ether/etherpad-lite/wiki"
+  echo "- Create a pull request on github to merge the master branch back to develop"
+  echo "- Announce the new release on the mailing list, blog.etherpad.org and Twitter"
+}
+
+# Call functions
+check_api_token
+modify_files
+create_release_branch
+merge_release_branch
+create_builds
+push_builds
+merge_web_branch
+publish_release
+todo_notification

src/bin/createUserSession.js

@@ -0,0 +1,51 @@
+'use strict';
+
+/*
+ * A tool for generating a test user session which can be used for debugging configs
+ * that require sessions.
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const fs = require('fs');
+const path = require('path');
+const querystring = require('querystring');
+const settings = require('../node/utils/Settings');
+const supertest = require('supertest');
+
+(async () => {
+  const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+  const filePath = path.join(__dirname, '../../APIKEY.txt');
+  const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+
+  let res;
+
+  res = await api.get('/api/');
+  const apiVersion = res.body.currentVersion;
+  if (!apiVersion) throw new Error('No version set in API');
+  const uri = (cmd, args) => `/api/${apiVersion}/${cmd}?${querystring.stringify(args)}`;
+
+  res = await api.post(uri('createGroup', {apikey}));
+  if (res.body.code === 1) throw new Error(`Error creating group: ${res.body}`);
+  const groupID = res.body.data.groupID;
+  console.log('groupID', groupID);
+
+  res = await api.post(uri('createGroupPad', {apikey, groupID}));
+  if (res.body.code === 1) throw new Error(`Error creating group pad: ${res.body}`);
+  console.log('Test Pad ID ====> ', res.body.data.padID);
+
+  res = await api.post(uri('createAuthor', {apikey}));
+  if (res.body.code === 1) throw new Error(`Error creating author: ${res.body}`);
+  const authorID = res.body.data.authorID;
+  console.log('authorID', authorID);
+
+  const validUntil = Math.floor(new Date() / 1000) + 60000;
+  console.log('validUntil', validUntil);
+  res = await api.post(uri('createSession', {apikey, groupID, authorID, validUntil}));
+  if (res.body.code === 1) throw new Error(`Error creating session: ${res.body}`);
+  console.log('Session made: ====> create a cookie named sessionID and set the value to',
+      res.body.data.sessionID);
+})();

src/bin/deb-src/DEBIAN/control

@@ -0,0 +1,9 @@
+Package: etherpad
+Version: 1.3
+Section: base
+Priority: optional
+Architecture: i386
+Installed-Size: SIZE
+Depends:
+Maintainer: John McLear <john@mclear.co.uk>
+Description: Etherpad is a collaborative editor.  

src/bin/deb-src/DEBIAN/postinst

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Start the services!
+
+service etherpad start
+echo "Give Etherpad about 3 minutes to install dependencies then visit http://localhost:9001 in your web browser"
+echo "To stop etherpad type 'service etherpad stop', To restart type 'service etherpad restart'".
+rm -f /tmp/etherpad.log /tmp/etherpad.err

src/bin/deb-src/DEBIAN/preinst

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Installs node if it isn't already installed
+#
+# Don't steamroll over a previously installed node version
+# TODO provide a local version of node?
+
+VER="0.10.4"
+ARCH="x86"
+if [ `arch | grep 64` ]
+then
+  ARCH="x64"
+fi
+
+# TODO test version
+if [ ! -f /usr/local/bin/node ]
+then
+  pushd /tmp
+  wget -c "http://nodejs.org/dist/v${VER}/node-v${VER}-linux-${ARCH}.tar.gz"
+  rm -rf /tmp/node-v${VER}-linux-${ARCH}
+  tar xf node-v${VER}-linux-${ARCH}.tar.gz -C /tmp/
+  cp -a /tmp/node-v${VER}-linux-${ARCH}/* /usr/local/
+fi
+
+# Create Etherpad user
+adduser --system etherpad

src/bin/deb-src/DEBIAN/prerm

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Stop the appserver:
+service etherpad stop || true

src/bin/deb-src/sysroot/etc/init/etherpad.conf

@@ -0,0 +1,28 @@
+description "etherpad"
+
+start on started networking
+stop on runlevel [!2345]
+
+env EPHOME=/opt/etherpad
+env EPLOGS=/var/log/etherpad
+env EPUSER=etherpad
+
+respawn
+
+pre-start script
+    cd $EPHOME
+    mkdir $EPLOGS                        ||true
+    chown $EPUSER $EPLOGS                ||true
+    chmod 0755 $EPLOGS                   ||true
+    chown -R $EPUSER $EPHOME/var         ||true
+    $EPHOME/bin/installDeps.sh >> $EPLOGS/error.log || { stop; exit 1; }
+end script
+
+script
+  cd $EPHOME/
+  exec su -s /bin/sh -c 'exec "$0" "$@"' $EPUSER -- node node_modules/ep_etherpad-lite/node/server.js \
+                        >> $EPLOGS/access.log \
+                        2>> $EPLOGS/error.log
+  echo "Etherpad is running on http://localhost:9001 - To change settings edit /opt/etherpad/settings.json"
+
+end script

src/bin/debugRun.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+# Prepare the environment
+bin/installDeps.sh || exit 1
+
+echo "If you are new to debugging Node.js with Chrome DevTools, take a look at this page:"
+echo "https://medium.com/@paul_irish/debugging-node-js-nightlies-with-chrome-devtools-7c4a1b95ae27"
+echo "Open 'chrome://inspect' on Chrome to start debugging."
+
+# Use 0.0.0.0 to allow external connections to the debugger
+# (ex: running Etherpad on a docker container). Use default port # (9229)
+node $(compute_node_args) --inspect=0.0.0.0:9229 node_modules/ep_etherpad-lite/node/server.js "$@"

src/bin/deleteAllGroupSessions.js

@@ -0,0 +1,47 @@
+'use strict';
+
+/*
+* A tool for deleting ALL GROUP sessions Etherpad user sessions from the CLI,
+* because sometimes a brick is required to fix a face.
+*/
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const path = require('path');
+const fs = require('fs');
+const supertest = require('supertest');
+
+// Set a delete counter which will increment on each delete attempt
+// TODO: Check delete is successful before incrementing
+let deleteCount = 0;
+
+// get the API Key
+const filePath = path.join(__dirname, '../../APIKEY.txt');
+console.log('Deleting all group sessions, please be patient.');
+
+(async () => {
+  const settings = require('../tests/container/loadSettings').loadSettings();
+  const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+  const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+  const apiVersionResponse = await api.get('/api/');
+  const apiVersion = apiVersionResponse.body.currentVersion; // 1.12.5
+
+  const groupsResponse = await api.get(`/api/${apiVersion}/listAllGroups?apikey=${apikey}`);
+  const groups = groupsResponse.body.data.groupIDs; // ['whateverGroupID']
+
+  for (const groupID of groups) {
+    const sessionURI = `/api/${apiVersion}/listSessionsOfGroup?apikey=${apikey}&groupID=${groupID}`;
+    const sessionsResponse = await api.get(sessionURI);
+    const sessions = sessionsResponse.body.data;
+
+    for (const sessionID of Object.keys(sessions)) {
+      const deleteURI = `/api/${apiVersion}/deleteSession?apikey=${apikey}&sessionID=${sessionID}`;
+      await api.post(deleteURI); // delete
+      deleteCount++;
+    }
+  }
+  console.log(`Deleted ${deleteCount} sessions`);
+})();

src/bin/deletePad.js

@@ -0,0 +1,38 @@
+'use strict';
+
+/*
+ * A tool for deleting pads from the CLI, because sometimes a brick is required
+ * to fix a window.
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const settings = require('../tests/container/loadSettings').loadSettings();
+const path = require('path');
+const fs = require('fs');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+if (process.argv.length !== 3) throw new Error('Use: node deletePad.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+// get the API Key
+const filePath = path.join(__dirname, '../../APIKEY.txt');
+const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+
+(async () => {
+  let apiVersion = await api.get('/api/');
+  apiVersion = apiVersion.body.currentVersion;
+  if (!apiVersion) throw new Error('No version set in API');
+
+  // Now we know the latest API version, let's delete pad
+  const uri = `/api/${apiVersion}/deletePad?apikey=${apikey}&padID=${padId}`;
+  const deleteAttempt = await api.post(uri);
+  if (deleteAttempt.body.code === 1) throw new Error(`Error deleting pad ${deleteAttempt.body}`);
+  console.log('Deleted pad', deleteAttempt.body);
+})();

src/bin/dirty-db-cleaner.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env PYTHONUNBUFFERED=1 python
+#
+# Created by Bjarni R. Einarsson, placed in the public domain. Go wild!
+#
+import json
+import os
+import sys
+
+try:
+    dirtydb_input = sys.argv[1]
+    dirtydb_output = '%s.new' % dirtydb_input
+    assert(os.path.exists(dirtydb_input))
+    assert(not os.path.exists(dirtydb_output))
+except:
+    print()
+    print('Usage: %s /path/to/dirty.db' % sys.argv[0])
+    print()
+    print('Note: Will create a file named dirty.db.new in the same folder,')
+    print('      please make sure permissions are OK and a file by that')
+    print('      name does not exist already. This script works by omitting')
+    print('      duplicate lines from the dirty.db file, keeping only the')
+    print('      last (latest) instance. No revision data should be lost,')
+    print('      but be careful, make backups. If it breaks you get to keep')
+    print('      both pieces!')
+    print()
+    sys.exit(1)
+
+dirtydb = {}
+lines = 0
+with open(dirtydb_input, 'r') as fd:
+    print('Reading %s' % dirtydb_input)
+    for line in fd:
+        lines += 1
+        try:
+            data = json.loads(line)
+            dirtydb[data['key']] = line
+        except:
+            print("Skipping invalid JSON!")
+        if lines % 10000 == 0:
+            sys.stderr.write('.')
+print()
+print('OK, found %d unique keys in %d lines' % (len(dirtydb), lines))
+
+with open(dirtydb_output, 'w') as fd:
+    for data in list(dirtydb.values()):
+        fd.write(data)
+
+print('Wrote data to %s. All done!' % dirtydb_output)

src/bin/doc/LICENSE

@@ -0,0 +1,18 @@
+Copyright Joyent, Inc. and other Node contributors. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

src/bin/doc/README.md

@@ -0,0 +1,76 @@
+Here's how the node docs work.
+
+Each type of heading has a description block.
+
+
+    ## module
+
+        Stability: 3 - Stable
+
+    description and examples.
+
+    ### module.property
+
+    * Type
+
+    description of the property.
+
+    ### module.someFunction(x, y, [z=100])
+
+    * `x` {String} the description of the string
+    * `y` {Boolean} Should I stay or should I go?
+    * `z` {Number} How many zebras to bring.
+
+    A description of the function.
+
+    ### Event: 'blerg'
+
+    * Argument: SomeClass object.
+
+    Modules don't usually raise events on themselves.  `cluster` is the
+    only exception.
+
+    ## Class: SomeClass
+
+    description of the class.
+
+    ### Class Method: SomeClass.classMethod(anArg)
+
+    * `anArg` {Object}  Just an argument
+      * `field` {String} anArg can have this field.
+      * `field2` {Boolean}  Another field.  Default: `false`.
+    * Return: {Boolean} `true` if it worked.
+
+    Description of the method for humans.
+
+    ### someClass.nextSibling()
+
+    * Return: {SomeClass object | null}  The next someClass in line.
+
+    ### someClass.someProperty
+
+    * String
+
+    The indication of what someProperty is.
+
+    ### Event: 'grelb'
+
+    * `isBlerg` {Boolean}
+
+    This event is emitted on instances of SomeClass, not on the module itself.
+
+
+* Modules have (description, Properties, Functions, Classes, Examples)
+* Properties have (type, description)
+* Functions have (list of arguments, description)
+* Classes have (description, Properties, Methods, Events)
+* Events have (list of arguments, description)
+* Methods have (list of arguments, description)
+* Properties have (type, description)
+
+# CLI usage
+
+Run the following from the etherpad-lite root directory:
+```sh
+$ node bin/doc/generate doc/index.md --format=html --template=doc/template.html > out.html
+```

src/bin/doc/generate.js

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+
+'use strict';
+
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+const fs = require('fs');
+const path = require('path');
+
+// parse the args.
+// Don't use nopt or whatever for this.  It's simple enough.
+
+const args = process.argv.slice(2);
+let format = 'json';
+let template = null;
+let inputFile = null;
+
+args.forEach((arg) => {
+  if (!arg.match(/^--/)) {
+    inputFile = arg;
+  } else if (arg.match(/^--format=/)) {
+    format = arg.replace(/^--format=/, '');
+  } else if (arg.match(/^--template=/)) {
+    template = arg.replace(/^--template=/, '');
+  }
+});
+
+
+if (!inputFile) {
+  throw new Error('No input file specified');
+}
+
+
+console.error('Input file = %s', inputFile);
+fs.readFile(inputFile, 'utf8', (er, input) => {
+  if (er) throw er;
+  // process the input for @include lines
+  processIncludes(inputFile, input, next);
+});
+
+
+const includeExpr = /^@include\s+([A-Za-z0-9-_/]+)(?:\.)?([a-zA-Z]*)$/gmi;
+const includeData = {};
+const processIncludes = (inputFile, input, cb) => {
+  const includes = input.match(includeExpr);
+  if (includes == null) return cb(null, input);
+  let errState = null;
+  console.error(includes);
+  let incCount = includes.length;
+  if (incCount === 0) cb(null, input);
+
+  includes.forEach((include) => {
+    let fname = include.replace(/^@include\s+/, '');
+    if (!fname.match(/\.md$/)) fname += '.md';
+
+    if (Object.prototype.hasOwnProperty.call(includeData, fname)) {
+      input = input.split(include).join(includeData[fname]);
+      incCount--;
+      if (incCount === 0) {
+        return cb(null, input);
+      }
+    }
+
+    const fullFname = path.resolve(path.dirname(inputFile), fname);
+    fs.readFile(fullFname, 'utf8', (er, inc) => {
+      if (errState) return;
+      if (er) return cb(errState = er);
+      processIncludes(fullFname, inc, (er, inc) => {
+        if (errState) return;
+        if (er) return cb(errState = er);
+        incCount--;
+        includeData[fname] = inc;
+        input = input.split(include).join(includeData[fname]);
+        if (incCount === 0) {
+          return cb(null, input);
+        }
+      });
+    });
+  });
+};
+
+
+const next = (er, input) => {
+  if (er) throw er;
+  switch (format) {
+    case 'json':
+      require('./json.js')(input, inputFile, (er, obj) => {
+        console.log(JSON.stringify(obj, null, 2));
+        if (er) throw er;
+      });
+      break;
+
+    case 'html':
+      require('./html.js')(input, inputFile, template, (er, html) => {
+        if (er) throw er;
+        console.log(html);
+      });
+      break;
+
+    default:
+      throw new Error(`Invalid format: ${format}`);
+  }
+};

src/bin/doc/html.js

@@ -0,0 +1,172 @@
+'use strict';
+
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+const fs = require('fs');
+const marked = require('marked');
+const path = require('path');
+
+
+const toHTML = (input, filename, template, cb) => {
+  const lexed = marked.lexer(input);
+  fs.readFile(template, 'utf8', (er, template) => {
+    if (er) return cb(er);
+    render(lexed, filename, template, cb);
+  });
+};
+module.exports = toHTML;
+
+const render = (lexed, filename, template, cb) => {
+  // get the section
+  const section = getSection(lexed);
+
+  filename = path.basename(filename, '.md');
+
+  lexed = parseLists(lexed);
+
+  // generate the table of contents.
+  // this mutates the lexed contents in-place.
+  buildToc(lexed, filename, (er, toc) => {
+    if (er) return cb(er);
+
+    template = template.replace(/__FILENAME__/g, filename);
+    template = template.replace(/__SECTION__/g, section);
+    template = template.replace(/__TOC__/g, toc);
+
+    // content has to be the last thing we do with
+    // the lexed tokens, because it's destructive.
+    const content = marked.parser(lexed);
+    template = template.replace(/__CONTENT__/g, content);
+
+    cb(null, template);
+  });
+};
+
+
+// just update the list item text in-place.
+// lists that come right after a heading are what we're after.
+const parseLists = (input) => {
+  let state = null;
+  let depth = 0;
+  const output = [];
+  output.links = input.links;
+  input.forEach((tok) => {
+    if (state == null) {
+      if (tok.type === 'heading') {
+        state = 'AFTERHEADING';
+      }
+      output.push(tok);
+      return;
+    }
+    if (state === 'AFTERHEADING') {
+      if (tok.type === 'list_start') {
+        state = 'LIST';
+        if (depth === 0) {
+          output.push({type: 'html', text: '<div class="signature">'});
+        }
+        depth++;
+        output.push(tok);
+        return;
+      }
+      state = null;
+      output.push(tok);
+      return;
+    }
+    if (state === 'LIST') {
+      if (tok.type === 'list_start') {
+        depth++;
+        output.push(tok);
+        return;
+      }
+      if (tok.type === 'list_end') {
+        depth--;
+        if (depth === 0) {
+          state = null;
+          output.push({type: 'html', text: '</div>'});
+        }
+        output.push(tok);
+        return;
+      }
+      if (tok.text) {
+        tok.text = parseListItem(tok.text);
+      }
+    }
+    output.push(tok);
+  });
+
+  return output;
+};
+
+
+const parseListItem = (text) => {
+  text = text.replace(/\{([^}]+)\}/, '<span class="type">$1</span>');
+  // XXX maybe put more stuff here?
+  return text;
+};
+
+
+// section is just the first heading
+const getSection = (lexed) => {
+  for (let i = 0, l = lexed.length; i < l; i++) {
+    const tok = lexed[i];
+    if (tok.type === 'heading') return tok.text;
+  }
+  return '';
+};
+
+
+const buildToc = (lexed, filename, cb) => {
+  let toc = [];
+  let depth = 0;
+  lexed.forEach((tok) => {
+    if (tok.type !== 'heading') return;
+    if (tok.depth - depth > 1) {
+      return cb(new Error(`Inappropriate heading level\n${
+        JSON.stringify(tok)}`));
+    }
+
+    depth = tok.depth;
+    const id = getId(`${filename}_${tok.text.trim()}`);
+    toc.push(`${new Array((depth - 1) * 2 + 1).join(' ')
+    }* <a href="#${id}">${
+      tok.text}</a>`);
+    tok.text += `<span><a class="mark" href="#${id}" ` +
+                `id="${id}">#</a></span>`;
+  });
+
+  toc = marked.parse(toc.join('\n'));
+  cb(null, toc);
+};
+
+const idCounters = {};
+const getId = (text) => {
+  text = text.toLowerCase();
+  text = text.replace(/[^a-z0-9]+/g, '_');
+  text = text.replace(/^_+|_+$/, '');
+  text = text.replace(/^([^a-z])/, '_$1');
+  if (Object.prototype.hasOwnProperty.call(idCounters, text)) {
+    text += `_${++idCounters[text]}`;
+  } else {
+    idCounters[text] = 0;
+  }
+  return text;
+};

src/bin/doc/json.js

@@ -0,0 +1,556 @@
+'use strict';
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module.exports = doJSON;
+
+// Take the lexed input, and return a JSON-encoded object
+// A module looks like this: https://gist.github.com/1777387
+
+const marked = require('marked');
+
+const doJSON = (input, filename, cb) => {
+  const root = {source: filename};
+  const stack = [root];
+  let depth = 0;
+  let current = root;
+  let state = null;
+  const lexed = marked.lexer(input);
+  lexed.forEach((tok) => {
+    const type = tok.type;
+    let text = tok.text;
+
+    // <!-- type = module -->
+    // This is for cases where the markdown semantic structure is lacking.
+    if (type === 'paragraph' || type === 'html') {
+      const metaExpr = /<!--([^=]+)=([^-]+)-->\n*/g;
+      text = text.replace(metaExpr, (_0, k, v) => {
+        current[k.trim()] = v.trim();
+        return '';
+      });
+      text = text.trim();
+      if (!text) return;
+    }
+
+    if (type === 'heading' &&
+        !text.trim().match(/^example/i)) {
+      if (tok.depth - depth > 1) {
+        return cb(new Error(`Inappropriate heading level\n${
+          JSON.stringify(tok)}`));
+      }
+
+      // Sometimes we have two headings with a single
+      // blob of description.  Treat as a clone.
+      if (current &&
+          state === 'AFTERHEADING' &&
+          depth === tok.depth) {
+        const clone = current;
+        current = newSection(tok);
+        current.clone = clone;
+        // don't keep it around on the stack.
+        stack.pop();
+      } else {
+        // if the level is greater than the current depth,
+        // then it's a child, so we should just leave the stack
+        // as it is.
+        // However, if it's a sibling or higher, then it implies
+        // the closure of the other sections that came before.
+        // root is always considered the level=0 section,
+        // and the lowest heading is 1, so this should always
+        // result in having a valid parent node.
+        let d = tok.depth;
+        while (d <= depth) {
+          finishSection(stack.pop(), stack[stack.length - 1]);
+          d++;
+        }
+        current = newSection(tok);
+      }
+
+      depth = tok.depth;
+      stack.push(current);
+      state = 'AFTERHEADING';
+      return;
+    } // heading
+
+    // Immediately after a heading, we can expect the following
+    //
+    // { type: 'code', text: 'Stability: ...' },
+    //
+    // a list: starting with list_start, ending with list_end,
+    // maybe containing other nested lists in each item.
+    //
+    // If one of these isn't found, then anything that comes between
+    // here and the next heading should be parsed as the desc.
+    let stability;
+    if (state === 'AFTERHEADING') {
+      if (type === 'code' &&
+          (stability = text.match(/^Stability: ([0-5])(?:\s*-\s*)?(.*)$/))) {
+        current.stability = parseInt(stability[1], 10);
+        current.stabilityText = stability[2].trim();
+        return;
+      } else if (type === 'list_start' && !tok.ordered) {
+        state = 'AFTERHEADING_LIST';
+        current.list = current.list || [];
+        current.list.push(tok);
+        current.list.level = 1;
+      } else {
+        current.desc = current.desc || [];
+        if (!Array.isArray(current.desc)) {
+          current.shortDesc = current.desc;
+          current.desc = [];
+        }
+        current.desc.push(tok);
+        state = 'DESC';
+      }
+      return;
+    }
+
+    if (state === 'AFTERHEADING_LIST') {
+      current.list.push(tok);
+      if (type === 'list_start') {
+        current.list.level++;
+      } else if (type === 'list_end') {
+        current.list.level--;
+      }
+      if (current.list.level === 0) {
+        state = 'AFTERHEADING';
+        processList(current);
+      }
+      return;
+    }
+
+    current.desc = current.desc || [];
+    current.desc.push(tok);
+  });
+
+  // finish any sections left open
+  while (root !== (current = stack.pop())) {
+    finishSection(current, stack[stack.length - 1]);
+  }
+
+  return cb(null, root);
+};
+
+
+// go from something like this:
+// [ { type: 'list_item_start' },
+//   { type: 'text',
+//     text: '`settings` Object, Optional' },
+//   { type: 'list_start', ordered: false },
+//   { type: 'list_item_start' },
+//   { type: 'text',
+//     text: 'exec: String, file path to worker file.  Default: `__filename`' },
+//   { type: 'list_item_end' },
+//   { type: 'list_item_start' },
+//   { type: 'text',
+//     text: 'args: Array, string arguments passed to worker.' },
+//   { type: 'text',
+//     text: 'Default: `process.argv.slice(2)`' },
+//   { type: 'list_item_end' },
+//   { type: 'list_item_start' },
+//   { type: 'text',
+//     text: 'silent: Boolean, whether or not to send output to parent\'s stdio.' },
+//   { type: 'text', text: 'Default: `false`' },
+//   { type: 'space' },
+//   { type: 'list_item_end' },
+//   { type: 'list_end' },
+//   { type: 'list_item_end' },
+//   { type: 'list_end' } ]
+// to something like:
+// [ { name: 'settings',
+//     type: 'object',
+//     optional: true,
+//     settings:
+//      [ { name: 'exec',
+//          type: 'string',
+//          desc: 'file path to worker file',
+//          default: '__filename' },
+//        { name: 'args',
+//          type: 'array',
+//          default: 'process.argv.slice(2)',
+//          desc: 'string arguments passed to worker.' },
+//        { name: 'silent',
+//          type: 'boolean',
+//          desc: 'whether or not to send output to parent\'s stdio.',
+//          default: 'false' } ] } ]
+
+const processList = (section) => {
+  const list = section.list;
+  const values = [];
+  let current;
+  const stack = [];
+
+  // for now, *just* build the hierarchical list
+  list.forEach((tok) => {
+    const type = tok.type;
+    if (type === 'space') return;
+    if (type === 'list_item_start') {
+      if (!current) {
+        const n = {};
+        values.push(n);
+        current = n;
+      } else {
+        current.options = current.options || [];
+        stack.push(current);
+        const n = {};
+        current.options.push(n);
+        current = n;
+      }
+      return;
+    } else if (type === 'list_item_end') {
+      if (!current) {
+        throw new Error(`invalid list - end without current item\n${
+          JSON.stringify(tok)}\n${
+          JSON.stringify(list)}`);
+      }
+      current = stack.pop();
+    } else if (type === 'text') {
+      if (!current) {
+        throw new Error(`invalid list - text without current item\n${
+          JSON.stringify(tok)}\n${
+          JSON.stringify(list)}`);
+      }
+      current.textRaw = current.textRaw || '';
+      current.textRaw += `${tok.text} `;
+    }
+  });
+
+  // shove the name in there for properties, since they are always
+  // just going to be the value etc.
+  if (section.type === 'property' && values[0]) {
+    values[0].textRaw = `\`${section.name}\` ${values[0].textRaw}`;
+  }
+
+  // now pull the actual values out of the text bits.
+  values.forEach(parseListItem);
+
+  // Now figure out what this list actually means.
+  // depending on the section type, the list could be different things.
+
+  switch (section.type) {
+    case 'ctor':
+    case 'classMethod':
+    case 'method': {
+      // each item is an argument, unless the name is 'return',
+      // in which case it's the return value.
+      section.signatures = section.signatures || [];
+      const sig = {};
+      section.signatures.push(sig);
+      sig.params = values.filter((v) => {
+        if (v.name === 'return') {
+          sig.return = v;
+          return false;
+        }
+        return true;
+      });
+      parseSignature(section.textRaw, sig);
+      break;
+    }
+    case 'property': {
+      // there should be only one item, which is the value.
+      // copy the data up to the section.
+      const value = values[0] || {};
+      delete value.name;
+      section.typeof = value.type;
+      delete value.type;
+      Object.keys(value).forEach((k) => {
+        section[k] = value[k];
+      });
+      break;
+    }
+
+    case 'event': {
+      // event: each item is an argument.
+      section.params = values;
+      break;
+    }
+  }
+
+  delete section.list;
+};
+
+
+// textRaw = "someobject.someMethod(a, [b=100], [c])"
+const parseSignature = (text, sig) => {
+  let params = text.match(paramExpr);
+  if (!params) return;
+  params = params[1];
+  // the ] is irrelevant. [ indicates optionalness.
+  params = params.replace(/\]/g, '');
+  params = params.split(/,/);
+  params.forEach((p, i, _) => {
+    p = p.trim();
+    if (!p) return;
+    let param = sig.params[i];
+    let optional = false;
+    let def;
+    // [foo] -> optional
+    if (p.charAt(0) === '[') {
+      optional = true;
+      p = p.substr(1);
+    }
+    const eq = p.indexOf('=');
+    if (eq !== -1) {
+      def = p.substr(eq + 1);
+      p = p.substr(0, eq);
+    }
+    if (!param) {
+      param = sig.params[i] = {name: p};
+    }
+    // at this point, the name should match.
+    if (p !== param.name) {
+      console.error('Warning: invalid param "%s"', p);
+      console.error(` > ${JSON.stringify(param)}`);
+      console.error(` > ${text}`);
+    }
+    if (optional) param.optional = true;
+    if (def !== undefined) param.default = def;
+  });
+};
+
+
+const parseListItem = (item) => {
+  if (item.options) item.options.forEach(parseListItem);
+  if (!item.textRaw) return;
+
+  // the goal here is to find the name, type, default, and optional.
+  // anything left over is 'desc'
+  let text = item.textRaw.trim();
+  // text = text.replace(/^(Argument|Param)s?\s*:?\s*/i, '');
+
+  text = text.replace(/^, /, '').trim();
+  const retExpr = /^returns?\s*:?\s*/i;
+  const ret = text.match(retExpr);
+  if (ret) {
+    item.name = 'return';
+    text = text.replace(retExpr, '');
+  } else {
+    const nameExpr = /^['`"]?([^'`": {]+)['`"]?\s*:?\s*/;
+    const name = text.match(nameExpr);
+    if (name) {
+      item.name = name[1];
+      text = text.replace(nameExpr, '');
+    }
+  }
+
+  text = text.trim();
+  const defaultExpr = /\(default\s*[:=]?\s*['"`]?([^, '"`]*)['"`]?\)/i;
+  const def = text.match(defaultExpr);
+  if (def) {
+    item.default = def[1];
+    text = text.replace(defaultExpr, '');
+  }
+
+  text = text.trim();
+  const typeExpr = /^\{([^}]+)\}/;
+  const type = text.match(typeExpr);
+  if (type) {
+    item.type = type[1];
+    text = text.replace(typeExpr, '');
+  }
+
+  text = text.trim();
+  const optExpr = /^Optional\.|(?:, )?Optional$/;
+  const optional = text.match(optExpr);
+  if (optional) {
+    item.optional = true;
+    text = text.replace(optExpr, '');
+  }
+
+  text = text.replace(/^\s*-\s*/, '');
+  text = text.trim();
+  if (text) item.desc = text;
+};
+
+
+const finishSection = (section, parent) => {
+  if (!section || !parent) {
+    throw new Error(`Invalid finishSection call\n${
+      JSON.stringify(section)}\n${
+      JSON.stringify(parent)}`);
+  }
+
+  if (!section.type) {
+    section.type = 'module';
+    if (parent && (parent.type === 'misc')) {
+      section.type = 'misc';
+    }
+    section.displayName = section.name;
+    section.name = section.name.toLowerCase()
+        .trim().replace(/\s+/g, '_');
+  }
+
+  if (section.desc && Array.isArray(section.desc)) {
+    section.desc.links = section.desc.links || [];
+    section.desc = marked.parser(section.desc);
+  }
+
+  if (!section.list) section.list = [];
+  processList(section);
+
+  // classes sometimes have various 'ctor' children
+  // which are actually just descriptions of a constructor
+  // class signature.
+  // Merge them into the parent.
+  if (section.type === 'class' && section.ctors) {
+    section.signatures = section.signatures || [];
+    const sigs = section.signatures;
+    section.ctors.forEach((ctor) => {
+      ctor.signatures = ctor.signatures || [{}];
+      ctor.signatures.forEach((sig) => {
+        sig.desc = ctor.desc;
+      });
+      sigs.push(...ctor.signatures);
+    });
+    delete section.ctors;
+  }
+
+  // properties are a bit special.
+  // their "type" is the type of object, not "property"
+  if (section.properties) {
+    section.properties.forEach((p) => {
+      if (p.typeof) p.type = p.typeof;
+      else delete p.type;
+      delete p.typeof;
+    });
+  }
+
+  // handle clones
+  if (section.clone) {
+    const clone = section.clone;
+    delete section.clone;
+    delete clone.clone;
+    deepCopy(section, clone);
+    finishSection(clone, parent);
+  }
+
+  let plur;
+  if (section.type.slice(-1) === 's') {
+    plur = `${section.type}es`;
+  } else if (section.type.slice(-1) === 'y') {
+    plur = section.type.replace(/y$/, 'ies');
+  } else {
+    plur = `${section.type}s`;
+  }
+
+  // if the parent's type is 'misc', then it's just a random
+  // collection of stuff, like the "globals" section.
+  // Make the children top-level items.
+  if (section.type === 'misc') {
+    Object.keys(section).forEach((k) => {
+      switch (k) {
+        case 'textRaw':
+        case 'name':
+        case 'type':
+        case 'desc':
+        case 'miscs':
+          return;
+        default:
+          if (parent.type === 'misc') {
+            return;
+          }
+          if (Array.isArray(k) && parent[k]) {
+            parent[k] = parent[k].concat(section[k]);
+          } else if (!parent[k]) {
+            parent[k] = section[k];
+          } else {
+            // parent already has, and it's not an array.
+            return;
+          }
+      }
+    });
+  }
+
+  parent[plur] = parent[plur] || [];
+  parent[plur].push(section);
+};
+
+
+// Not a general purpose deep copy.
+// But sufficient for these basic things.
+const deepCopy = (src, dest) => {
+  Object.keys(src).filter((k) => !Object.prototype.hasOwnProperty.call(dest, k)).forEach((k) => {
+    dest[k] = deepCopy_(src[k]);
+  });
+};
+
+const deepCopy_ = (src) => {
+  if (!src) return src;
+  if (Array.isArray(src)) {
+    const c = new Array(src.length);
+    src.forEach((v, i) => {
+      c[i] = deepCopy_(v);
+    });
+    return c;
+  }
+  if (typeof src === 'object') {
+    const c = {};
+    Object.keys(src).forEach((k) => {
+      c[k] = deepCopy_(src[k]);
+    });
+    return c;
+  }
+  return src;
+};
+
+
+// these parse out the contents of an H# tag
+const eventExpr = /^Event(?::|\s)+['"]?([^"']+).*$/i;
+const classExpr = /^Class:\s*([^ ]+).*?$/i;
+const propExpr = /^(?:property:?\s*)?[^.]+\.([^ .()]+)\s*?$/i;
+const braceExpr = /^(?:property:?\s*)?[^.[]+(\[[^\]]+\])\s*?$/i;
+const classMethExpr =
+  /^class\s*method\s*:?[^.]+\.([^ .()]+)\([^)]*\)\s*?$/i;
+const methExpr =
+  /^(?:method:?\s*)?(?:[^.]+\.)?([^ .()]+)\([^)]*\)\s*?$/i;
+const newExpr = /^new ([A-Z][a-z]+)\([^)]*\)\s*?$/;
+const paramExpr = /\((.*)\);?$/;
+
+const newSection = (tok) => {
+  const section = {};
+  // infer the type from the text.
+  const text = section.textRaw = tok.text;
+  if (text.match(eventExpr)) {
+    section.type = 'event';
+    section.name = text.replace(eventExpr, '$1');
+  } else if (text.match(classExpr)) {
+    section.type = 'class';
+    section.name = text.replace(classExpr, '$1');
+  } else if (text.match(braceExpr)) {
+    section.type = 'property';
+    section.name = text.replace(braceExpr, '$1');
+  } else if (text.match(propExpr)) {
+    section.type = 'property';
+    section.name = text.replace(propExpr, '$1');
+  } else if (text.match(classMethExpr)) {
+    section.type = 'classMethod';
+    section.name = text.replace(classMethExpr, '$1');
+  } else if (text.match(methExpr)) {
+    section.type = 'method';
+    section.name = text.replace(methExpr, '$1');
+  } else if (text.match(newExpr)) {
+    section.type = 'ctor';
+    section.name = text.replace(newExpr, '$1');
+  } else {
+    section.name = text;
+  }
+  return section;
+};

src/bin/doc/package-lock.json

@@ -0,0 +1,13 @@
+{
+  "name": "node-doc-generator",
+  "version": "0.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "marked": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.2.tgz",
+      "integrity": "sha512-EGwzEeCcLniFX51DhTpmTom+dSA/MG/OBUDjnWtHbEnjAH180VzUeAw+oE4+Zv+CoYBWyRlYOTR0N8SO9R1PVw=="
+    }
+  }
+}

src/bin/doc/package.json

@@ -0,0 +1,15 @@
+{
+  "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)",
+  "name": "node-doc-generator",
+  "description": "Internal tool for generating Node.js API docs",
+  "version": "0.0.0",
+  "engines": {
+    "node": ">=10.17.0"
+  },
+  "dependencies": {
+    "marked": "0.8.2"
+  },
+  "devDependencies": {},
+  "optionalDependencies": {},
+  "bin": "./generate.js"
+}

src/bin/extractPadData.js

@@ -0,0 +1,64 @@
+'use strict';
+
+/*
+ * This is a debug tool. It helps to extract all datas of a pad and move it from
+ * a productive environment and to a develop environment to reproduce bugs
+ * there. It outputs a dirtydb file
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 3) throw new Error('Use: node extractPadData.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load extra modules
+  const dirtyDB = require('dirty');
+  const padManager = require('../node/db/PadManager');
+
+  // initialize output database
+  const dirty = dirtyDB(`${padId}.db`);
+
+  // Promise wrapped get and set function
+  const wrapped = db.db.db.wrappedDB;
+  const get = util.promisify(wrapped.get.bind(wrapped));
+  const set = util.promisify(dirty.set.bind(dirty));
+
+  // array in which required key values will be accumulated
+  const neededDBValues = [`pad:${padId}`];
+
+  // get the actual pad object
+  const pad = await padManager.getPad(padId);
+
+  // add all authors
+  neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`));
+
+  // add all revisions
+  for (let rev = 0; rev <= pad.head; ++rev) {
+    neededDBValues.push(`pad:${padId}:revs:${rev}`);
+  }
+
+  // add all chat values
+  for (let chat = 0; chat <= pad.chatHead; ++chat) {
+    neededDBValues.push(`pad:${padId}:chat:${chat}`);
+  }
+
+  for (const dbkey of neededDBValues) {
+    let dbvalue = await get(dbkey);
+    if (dbvalue && typeof dbvalue !== 'object') {
+      dbvalue = JSON.parse(dbvalue);
+    }
+    await set(dbkey, dbvalue);
+  }
+
+  console.log('finished');
+})();

src/bin/fastRun.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Run Etherpad directly, assuming all the dependencies are already installed.
+#
+# Useful for developers, or users that know what they are doing. If you just
+# upgraded Etherpad version, installed a new dependency, or are simply unsure
+# of what to do, please execute bin/installDeps.sh once before running this
+# script.
+
+set -eu
+
+# source: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself#246128
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+# Source constants and useful functions
+. ${DIR}/../bin/functions.sh
+
+echo "Running directly, without checking/installing dependencies"
+
+# move to the base Etherpad directory. This will be necessary until Etherpad
+# learns to run from arbitrary CWDs.
+cd "${DIR}/.."
+
+# run Etherpad main class
+node $(compute_node_args) "${DIR}/../node_modules/ep_etherpad-lite/node/server.js" "$@"

src/bin/functions.sh

@@ -0,0 +1,74 @@
+# minimum required node version
+REQUIRED_NODE_MAJOR=10
+REQUIRED_NODE_MINOR=13
+
+# minimum required npm version
+REQUIRED_NPM_MAJOR=5
+REQUIRED_NPM_MINOR=5
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
+
+get_program_version() {
+  PROGRAM="$1"
+  KIND="${2:-full}"
+  PROGRAM_VERSION_STRING=$($PROGRAM --version)
+  PROGRAM_VERSION_STRING=${PROGRAM_VERSION_STRING#"v"}
+
+  DETECTED_MAJOR=$(pecho "$PROGRAM_VERSION_STRING" | cut -s -d "." -f 1)
+  [ -n "$DETECTED_MAJOR" ] || fatal "Cannot extract $PROGRAM major version from version string \"$PROGRAM_VERSION_STRING\""
+  case "$DETECTED_MAJOR" in
+      ''|*[!0-9]*)
+        fatal "$PROGRAM_LABEL major version from \"$VERSION_STRING\" is not a number. Detected: \"$DETECTED_MAJOR\""
+        ;;
+  esac
+
+  DETECTED_MINOR=$(pecho "$PROGRAM_VERSION_STRING" | cut -s -d "." -f 2)
+  [ -n "$DETECTED_MINOR" ] || fatal "Cannot extract $PROGRAM minor version from version string \"$PROGRAM_VERSION_STRING\""
+  case "$DETECTED_MINOR" in
+      ''|*[!0-9]*)
+        fatal "$PROGRAM_LABEL minor version from \"$VERSION_STRING\" is not a number. Detected: \"$DETECTED_MINOR\""
+  esac
+
+  case $KIND in
+    major)
+      echo $DETECTED_MAJOR
+      exit;;
+    minor)
+      echo $DETECTED_MINOR
+      exit;;
+    *)
+      echo $DETECTED_MAJOR.$DETECTED_MINOR
+      exit;;
+  esac
+
+  echo $VERSION
+}
+
+
+compute_node_args() {
+  ARGS=""
+
+  NODE_MAJOR=$(get_program_version "node" "major")
+  [ "$NODE_MAJOR" -eq "10" ] && ARGS="$ARGS --experimental-worker"
+
+  echo $ARGS
+}
+
+
+require_minimal_version() {
+  PROGRAM_LABEL="$1"
+  VERSION="$2"
+  REQUIRED_MAJOR="$3"
+  REQUIRED_MINOR="$4"
+
+  VERSION_MAJOR=$(pecho "$VERSION" | cut -s -d "." -f 1)
+  VERSION_MINOR=$(pecho "$VERSION" | cut -s -d "." -f 2)
+
+  [ "$VERSION_MAJOR" -gt "$REQUIRED_MAJOR" ] || ([ "$VERSION_MAJOR" -eq "$REQUIRED_MAJOR" ] && [ "$VERSION_MINOR" -ge "$REQUIRED_MINOR" ]) \
+    || fatal "Your $PROGRAM_LABEL version \"$VERSION_MAJOR.$VERSION_MINOR\" is too old. $PROGRAM_LABEL $REQUIRED_MAJOR.$REQUIRED_MINOR.x or higher is required."
+}

src/bin/importSqlFile.js

@@ -0,0 +1,100 @@
+'use strict';
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const startTime = Date.now();
+
+const log = (str) => {
+  console.log(`${(Date.now() - startTime) / 1000}\t${str}`);
+};
+
+const unescape = (val) => {
+  // value is a string
+  if (val.substr(0, 1) === "'") {
+    val = val.substr(0, val.length - 1).substr(1);
+
+    return val.replace(/\\[0nrbtZ\\'"]/g, (s) => {
+      switch (s) {
+        case '\\0': return '\0';
+        case '\\n': return '\n';
+        case '\\r': return '\r';
+        case '\\b': return '\b';
+        case '\\t': return '\t';
+        case '\\Z': return '\x1a';
+        default: return s.substr(1);
+      }
+    });
+  }
+
+  // value is a boolean or NULL
+  if (val === 'NULL') {
+    return null;
+  }
+  if (val === 'true') {
+    return true;
+  }
+  if (val === 'false') {
+    return false;
+  }
+
+  // value is a number
+  return val;
+};
+
+(async () => {
+  const fs = require('fs');
+  const log4js = require('log4js');
+  const settings = require('../node/utils/Settings');
+  const ueberDB = require('ueberdb2');
+
+  const dbWrapperSettings = {
+    cache: 0,
+    writeInterval: 100,
+    json: false, // data is already json encoded
+  };
+  const db = new ueberDB.database( // eslint-disable-line new-cap
+      settings.dbType,
+      settings.dbSettings,
+      dbWrapperSettings,
+      log4js.getLogger('ueberDB'));
+
+  const sqlFile = process.argv[2];
+
+  // stop if the settings file is not set
+  if (!sqlFile) throw new Error('Use: node importSqlFile.js $SQLFILE');
+
+  log('initializing db');
+  await util.promisify(db.init.bind(db))();
+  log('done');
+
+  log('open output file...');
+  const lines = fs.readFileSync(sqlFile, 'utf8').split('\n');
+
+  const count = lines.length;
+  let keyNo = 0;
+
+  process.stdout.write(`Start importing ${count} keys...\n`);
+  lines.forEach((l) => {
+    if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') {
+      const pos = l.indexOf("', '");
+      const key = l.substr(28, pos - 28);
+      let value = l.substr(pos + 3);
+      value = value.substr(0, value.length - 2);
+      console.log(`key: ${key} val: ${value}`);
+      console.log(`unval: ${unescape(value)}`);
+      db.set(key, unescape(value), null);
+      keyNo++;
+      if (keyNo % 1000 === 0) {
+        process.stdout.write(` ${keyNo}/${count}\n`);
+      }
+    }
+  });
+  process.stdout.write('\n');
+  process.stdout.write('done. waiting for db to finish transaction. ' +
+                       'depended on dbms this may take some time..\n');
+
+  await util.promisify(db.close.bind(db))();
+  log(`finished, imported ${keyNo} keys.`);
+})();

src/bin/installDeps.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+# Is node installed?
+# Not checking io.js, default installation creates a symbolic link to node
+is_cmd node || fatal "Please install node.js ( https://nodejs.org )"
+
+# Is npm installed?
+is_cmd npm || fatal "Please install npm ( https://npmjs.org )"
+
+# Check npm version
+require_minimal_version "npm" $(get_program_version "npm") "$REQUIRED_NPM_MAJOR" "$REQUIRED_NPM_MINOR"
+
+# Check node version
+require_minimal_version "nodejs" $(get_program_version "node") "$REQUIRED_NODE_MAJOR" "$REQUIRED_NODE_MINOR"
+
+# Get the name of the settings file
+settings="settings.json"
+a='';
+for arg in "$@"; do
+  if [ "$a" = "--settings" ] || [ "$a" = "-s" ]; then settings=$arg; fi
+  a=$arg
+done
+
+# Does a $settings exist? if not copy the template
+if [ ! -f "$settings" ]; then
+  log "Copy the settings template to $settings..."
+  cp settings.json.template "$settings" || exit 1
+fi
+
+log "Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient."
+(
+  mkdir -p node_modules
+  cd node_modules
+  [ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
+  cd ep_etherpad-lite
+  npm ci
+) || {
+  rm -rf src/node_modules
+  exit 1
+}
+
+# Remove all minified data to force node creating it new
+log "Clearing minified cache..."
+rm -f var/minified*
+
+exit 0

src/bin/installOnWindows.bat

@@ -0,0 +1,34 @@
+@echo off
+
+:: Change directory to etherpad-lite root
+cd /D "%~dp0\.."
+
+:: Is node installed?
+cmd /C node -e "" || ( echo "Please install node.js ( https://nodejs.org )" && exit /B 1 )
+
+echo _
+echo Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient.
+
+mkdir node_modules
+cd /D node_modules
+mklink /D "ep_etherpad-lite" "..\src"
+
+cd /D "ep_etherpad-lite"
+cmd /C npm ci || exit /B 1
+
+cd /D "%~dp0\.."
+
+echo _
+echo Clearing cache...
+del /S var\minified*
+
+echo _
+echo Setting up settings.json...
+IF NOT EXIST settings.json (
+  echo Can't find settings.json.
+  echo Copying settings.json.template...
+  cmd /C copy settings.json.template settings.json || exit /B 1
+)
+
+echo _
+echo Installed Etherpad!  To run Etherpad type start.bat

src/bin/migrateDirtyDBtoRealDB.js

@@ -0,0 +1,56 @@
+'use strict';
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+(async () => {
+  // This script requires that you have modified your settings.json file
+  // to work with a real database.  Please make a backup of your dirty.db
+  // file before using this script, just to be safe.
+
+  // It might be necessary to run the script using more memory:
+  // `node --max-old-space-size=4096 bin/migrateDirtyDBtoRealDB.js`
+
+  const dirtyDb = require('dirty');
+  const log4js = require('log4js');
+  const settings = require('../node/utils/Settings');
+  const ueberDB = require('ueberdb2');
+  const util = require('util');
+
+  const dbWrapperSettings = {
+    cache: '0', // The cache slows things down when you're mostly writing.
+    writeInterval: 0, // Write directly to the database, don't buffer
+  };
+  const db = new ueberDB.database( // eslint-disable-line new-cap
+      settings.dbType,
+      settings.dbSettings,
+      dbWrapperSettings,
+      log4js.getLogger('ueberDB'));
+  await db.init();
+
+  console.log('Waiting for dirtyDB to parse its file.');
+  const dirty = dirtyDb(`${__dirname}/../../var/dirty.db`);
+  const length = await new Promise((resolve) => { dirty.once('load', resolve); });
+
+  console.log(`Found ${length} records, processing now.`);
+  const p = [];
+  let numWritten = 0;
+  dirty.forEach((key, value) => {
+    let bcb, wcb;
+    p.push(new Promise((resolve, reject) => {
+      bcb = (err) => { if (err != null) return reject(err); };
+      wcb = (err) => {
+        if (err != null) return reject(err);
+        if (++numWritten % 100 === 0) console.log(`Wrote record ${numWritten} of ${length}`);
+        resolve();
+      };
+    }));
+    db.set(key, value, bcb, wcb);
+  });
+  await Promise.all(p);
+  console.log(`Wrote all ${numWritten} records`);
+
+  await util.promisify(db.close.bind(db))();
+  console.log('Finished.');
+})();

src/bin/plugins/README.md

@@ -0,0 +1,53 @@
+The files in this folder are for Plugin developers.
+
+# Get suggestions to improve your Plugin
+
+This code will check your plugin for known usual issues and some suggestions for
+improvements. No changes will be made to your project.
+
+```
+node bin/plugins/checkPlugin.js $PLUGIN_NAME$
+```
+
+# Basic Example:
+
+```
+node bin/plugins/checkPlugin.js ep_webrtc
+```
+
+## Autofixing - will autofix any issues it can
+
+```
+node bin/plugins/checkPlugin.js ep_whatever autofix
+```
+
+## Autocommitting, push, npm minor patch and npm publish (highly dangerous)
+
+```
+node bin/plugins/checkPlugin.js ep_whatever autocommit
+```
+
+# All the plugins
+
+Replace johnmclear with your github username
+
+```
+# Clones
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+# autofixes and autocommits /pushes & npm publishes
+for dir in node_modules/ep_*; do
+  dir=${dir#node_modules/}
+  [ "$dir" != ep_etherpad-lite ] || continue
+  node bin/plugins/checkPlugin.js "$dir" autocommit
+done
+```
+
+# Automating update of ether organization plugins
+
+```
+getCorePlugins.sh
+updateCorePlugins.sh
+```

src/bin/plugins/checkPlugin.js

@@ -0,0 +1,476 @@
+'use strict';
+
+/*
+ * Usage -- see README.md
+ *
+ * Normal usage:                node bin/plugins/checkPlugin.js ep_whatever
+ * Auto fix the things it can:  node bin/plugins/checkPlugin.js ep_whatever autofix
+ * Auto commit, push and publish to npm (highly dangerous):
+ *                              node bin/plugins/checkPlugin.js ep_whatever autocommit
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const fs = require('fs');
+const childProcess = require('child_process');
+
+// get plugin name & path from user input
+const pluginName = process.argv[2];
+
+if (!pluginName) throw new Error('no plugin name specified');
+
+const pluginPath = `node_modules/${pluginName}`;
+
+console.log(`Checking the plugin: ${pluginName}`);
+
+const optArgs = process.argv.slice(3);
+const autoCommit = optArgs.indexOf('autocommit') !== -1;
+const autoFix = autoCommit || optArgs.indexOf('autofix') !== -1;
+
+const execSync = (cmd, opts = {}) => (childProcess.execSync(cmd, {
+  cwd: `${pluginPath}/`,
+  ...opts,
+}) || '').toString().replace(/\n+$/, '');
+
+const writePackageJson = (obj) => {
+  let s = JSON.stringify(obj, null, 2);
+  if (s.length && s.slice(s.length - 1) !== '\n') s += '\n';
+  return fs.writeFileSync(`${pluginPath}/package.json`, s);
+};
+
+const updateDeps = (parsedPackageJson, key, wantDeps) => {
+  const {[key]: deps = {}} = parsedPackageJson;
+  let changed = false;
+  for (const [pkg, verInfo] of Object.entries(wantDeps)) {
+    const {ver, overwrite = true} = typeof verInfo === 'string' ? {ver: verInfo} : verInfo;
+    if (deps[pkg] === ver) continue;
+    if (deps[pkg] == null) {
+      console.warn(`Missing dependency in ${key}: '${pkg}': '${ver}'`);
+    } else {
+      if (!overwrite) continue;
+      console.warn(`Dependency mismatch in ${key}: '${pkg}': '${ver}' (current: ${deps[pkg]})`);
+    }
+    if (autoFix) {
+      deps[pkg] = ver;
+      changed = true;
+    }
+  }
+  if (changed) {
+    parsedPackageJson[key] = deps;
+    writePackageJson(parsedPackageJson);
+  }
+};
+
+const prepareRepo = () => {
+  let branch = execSync('git symbolic-ref HEAD');
+  if (branch !== 'refs/heads/master' && branch !== 'refs/heads/main') {
+    throw new Error('master/main must be checked out');
+  }
+  branch = branch.replace(/^refs\/heads\//, '');
+  execSync('git rev-parse --verify -q HEAD^0 || ' +
+           `{ echo "Error: no commits on ${branch}" >&2; exit 1; }`);
+  execSync('git rev-parse --verify @{u}'); // Make sure there's a remote tracking branch.
+  const modified = execSync('git diff-files --name-status');
+  if (modified !== '') throw new Error(`working directory has modifications:\n${modified}`);
+  const untracked = execSync('git ls-files -o --exclude-standard');
+  if (untracked !== '') throw new Error(`working directory has untracked files:\n${untracked}`);
+  const indexStatus = execSync('git diff-index --cached --name-status HEAD');
+  if (indexStatus !== '') throw new Error(`uncommitted staged changes to files:\n${indexStatus}`);
+  execSync('git pull --ff-only', {stdio: 'inherit'});
+  if (execSync('git rev-list @{u}...') !== '') throw new Error('repo contains unpushed commits');
+  if (autoCommit) {
+    execSync('git config --get user.name');
+    execSync('git config --get user.email');
+  }
+};
+
+if (autoCommit) {
+  console.warn('Auto commit is enabled, I hope you know what you are doing...');
+}
+
+fs.readdir(pluginPath, (err, rootFiles) => {
+  // handling error
+  if (err) {
+    return console.log(`Unable to scan directory: ${err}`);
+  }
+
+  // rewriting files to lower case
+  const files = [];
+
+  // some files we need to know the actual file name.  Not compulsory but might help in the future.
+  let readMeFileName;
+  let repository;
+
+  for (let i = 0; i < rootFiles.length; i++) {
+    if (rootFiles[i].toLowerCase().indexOf('readme') !== -1) readMeFileName = rootFiles[i];
+    files.push(rootFiles[i].toLowerCase());
+  }
+
+  if (files.indexOf('.git') === -1) throw new Error('No .git folder, aborting');
+  prepareRepo();
+
+  try {
+    const path = `${pluginPath}/.github/workflows/npmpublish.yml`;
+    if (!fs.existsSync(path)) {
+      console.log('no .github/workflows/npmpublish.yml');
+      console.log('create one and set npm secret to auto publish to npm on commit');
+      if (autoFix) {
+        const npmpublish =
+            fs.readFileSync('bin/plugins/lib/npmpublish.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, npmpublish);
+        console.log("If you haven't already, setup autopublish for this plugin https://github.com/ether/etherpad-lite/wiki/Plugins:-Automatically-publishing-to-npm-on-commit-to-Github-Repo");
+      } else {
+        console.log('Setup autopublish for this plugin https://github.com/ether/etherpad-lite/wiki/Plugins:-Automatically-publishing-to-npm-on-commit-to-Github-Repo');
+      }
+    } else {
+      // autopublish exists, we should check the version..
+      // checkVersion takes two file paths and checks for a version string in them.
+      const currVersionFile = fs.readFileSync(path, {encoding: 'utf8', flag: 'r'});
+      const existingConfigLocation = currVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const existingValue = parseInt(
+          currVersionFile.substr(existingConfigLocation + 17, existingConfigLocation.length));
+
+      const reqVersionFile =
+          fs.readFileSync('bin/plugins/lib/npmpublish.yml', {encoding: 'utf8', flag: 'r'});
+      const reqConfigLocation = reqVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const reqValue =
+          parseInt(reqVersionFile.substr(reqConfigLocation + 17, reqConfigLocation.length));
+
+      if (!existingValue || (reqValue > existingValue)) {
+        const npmpublish =
+            fs.readFileSync('bin/plugins/lib/npmpublish.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, npmpublish);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+  }
+
+
+  try {
+    const path = `${pluginPath}/.github/workflows/backend-tests.yml`;
+    if (!fs.existsSync(path)) {
+      console.log('no .github/workflows/backend-tests.yml');
+      console.log('create one and set npm secret to auto publish to npm on commit');
+      if (autoFix) {
+        const backendTests =
+            fs.readFileSync('bin/plugins/lib/backend-tests.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, backendTests);
+      }
+    } else {
+      // autopublish exists, we should check the version..
+      // checkVersion takes two file paths and checks for a version string in them.
+      const currVersionFile = fs.readFileSync(path, {encoding: 'utf8', flag: 'r'});
+      const existingConfigLocation = currVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const existingValue = parseInt(
+          currVersionFile.substr(existingConfigLocation + 17, existingConfigLocation.length));
+
+      const reqVersionFile =
+          fs.readFileSync('bin/plugins/lib/backend-tests.yml', {encoding: 'utf8', flag: 'r'});
+      const reqConfigLocation = reqVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const reqValue =
+          parseInt(reqVersionFile.substr(reqConfigLocation + 17, reqConfigLocation.length));
+
+      if (!existingValue || (reqValue > existingValue)) {
+        const backendTests =
+            fs.readFileSync('bin/plugins/lib/backend-tests.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, backendTests);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+  }
+
+  if (files.indexOf('package.json') === -1) {
+    console.warn('no package.json, please create');
+  }
+
+  if (files.indexOf('package.json') !== -1) {
+    const packageJSON =
+        fs.readFileSync(`${pluginPath}/package.json`, {encoding: 'utf8', flag: 'r'});
+    const parsedPackageJSON = JSON.parse(packageJSON);
+    if (autoFix) {
+      let updatedPackageJSON = false;
+      if (!parsedPackageJSON.funding) {
+        updatedPackageJSON = true;
+        parsedPackageJSON.funding = {
+          type: 'individual',
+          url: 'https://etherpad.org/',
+        };
+      }
+      if (updatedPackageJSON) {
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+
+    if (packageJSON.toLowerCase().indexOf('repository') === -1) {
+      console.warn('No repository in package.json');
+      if (autoFix) {
+        console.warn('Repository not detected in package.json.  Add repository section.');
+      }
+    } else {
+      // useful for creating README later.
+      repository = parsedPackageJSON.repository.url;
+    }
+
+    updateDeps(parsedPackageJSON, 'devDependencies', {
+      'eslint': '^7.18.0',
+      'eslint-config-etherpad': '^1.0.24',
+      'eslint-plugin-eslint-comments': '^3.2.0',
+      'eslint-plugin-mocha': '^8.0.0',
+      'eslint-plugin-node': '^11.1.0',
+      'eslint-plugin-prefer-arrow': '^1.2.3',
+      'eslint-plugin-promise': '^4.2.1',
+      'eslint-plugin-you-dont-need-lodash-underscore': '^6.10.0',
+    });
+
+    updateDeps(parsedPackageJSON, 'peerDependencies', {
+      // Some plugins require a newer version of Etherpad so don't overwrite if already set.
+      'ep_etherpad-lite': {ver: '>=1.8.6', overwrite: false},
+    });
+
+    if (packageJSON.toLowerCase().indexOf('eslintconfig') === -1) {
+      console.warn('No esLintConfig in package.json');
+      if (autoFix) {
+        const eslintConfig = {
+          root: true,
+          extends: 'etherpad/plugin',
+        };
+        parsedPackageJSON.eslintConfig = eslintConfig;
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+
+    if (packageJSON.toLowerCase().indexOf('scripts') === -1) {
+      console.warn('No scripts in package.json');
+      if (autoFix) {
+        const scripts = {
+          'lint': 'eslint .',
+          'lint:fix': 'eslint --fix .',
+        };
+        parsedPackageJSON.scripts = scripts;
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+
+    if ((packageJSON.toLowerCase().indexOf('engines') === -1) || !parsedPackageJSON.engines.node) {
+      console.warn('No engines or node engine in package.json');
+      if (autoFix) {
+        const engines = {
+          node: '^10.17.0 || >=11.14.0',
+        };
+        parsedPackageJSON.engines = engines;
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+  }
+
+  if (files.indexOf('package-lock.json') === -1) {
+    console.warn('package-lock.json not found');
+    if (!autoFix) {
+      console.warn('Run npm install in the plugin folder and commit the package-lock.json file.');
+    }
+  }
+
+  if (files.indexOf('readme') === -1 && files.indexOf('readme.md') === -1) {
+    console.warn('README.md file not found, please create');
+    if (autoFix) {
+      console.log('Autofixing missing README.md file');
+      console.log('please edit the README.md file further to include plugin specific details.');
+      let readme = fs.readFileSync('bin/plugins/lib/README.md', {encoding: 'utf8', flag: 'r'});
+      readme = readme.replace(/\[plugin_name\]/g, pluginName);
+      if (repository) {
+        const org = repository.split('/')[3];
+        const name = repository.split('/')[4];
+        readme = readme.replace(/\[org_name\]/g, org);
+        readme = readme.replace(/\[repo_url\]/g, name);
+        fs.writeFileSync(`${pluginPath}/README.md`, readme);
+      } else {
+        console.warn('Unable to find repository in package.json, aborting.');
+      }
+    }
+  }
+
+  if (files.indexOf('contributing') === -1 && files.indexOf('contributing.md') === -1) {
+    console.warn('CONTRIBUTING.md file not found, please create');
+    if (autoFix) {
+      console.log('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md ' +
+                  'file further to include plugin specific details.');
+      let contributing =
+          fs.readFileSync('bin/plugins/lib/CONTRIBUTING.md', {encoding: 'utf8', flag: 'r'});
+      contributing = contributing.replace(/\[plugin_name\]/g, pluginName);
+      fs.writeFileSync(`${pluginPath}/CONTRIBUTING.md`, contributing);
+    }
+  }
+
+
+  if (files.indexOf('readme') !== -1 && files.indexOf('readme.md') !== -1) {
+    const readme =
+        fs.readFileSync(`${pluginPath}/${readMeFileName}`, {encoding: 'utf8', flag: 'r'});
+    if (readme.toLowerCase().indexOf('license') === -1) {
+      console.warn('No license section in README');
+      if (autoFix) {
+        console.warn('Please add License section to README manually.');
+      }
+    }
+  }
+
+  if (files.indexOf('license') === -1 && files.indexOf('license.md') === -1) {
+    console.warn('LICENSE.md file not found, please create');
+    if (autoFix) {
+      console.log('Autofixing missing LICENSE.md file, including Apache 2 license.');
+      let license = fs.readFileSync('bin/plugins/lib/LICENSE.md', {encoding: 'utf8', flag: 'r'});
+      license = license.replace('[yyyy]', new Date().getFullYear());
+      license = license.replace('[name of copyright owner]', execSync('git config user.name'));
+      fs.writeFileSync(`${pluginPath}/LICENSE.md`, license);
+    }
+  }
+
+  let travisConfig = fs.readFileSync('bin/plugins/lib/travis.yml', {encoding: 'utf8', flag: 'r'});
+  travisConfig = travisConfig.replace(/\[plugin_name\]/g, pluginName);
+
+  if (files.indexOf('.travis.yml') === -1) {
+    console.warn('.travis.yml file not found, please create.  ' +
+                 '.travis.yml is used for automatically CI testing Etherpad.  ' +
+                 'It is useful to know if your plugin breaks another feature for example.');
+    // TODO: Make it check version of the .travis file to see if it needs an update.
+    if (autoFix) {
+      console.log('Autofixing missing .travis.yml file');
+      fs.writeFileSync(`${pluginPath}/.travis.yml`, travisConfig);
+      console.log('Travis file created, please sign into travis and enable this repository');
+    }
+  }
+  if (autoFix) {
+    // checks the file versioning of .travis and updates it to the latest.
+    const existingConfig =
+        fs.readFileSync(`${pluginPath}/.travis.yml`, {encoding: 'utf8', flag: 'r'});
+    const existingConfigLocation = existingConfig.indexOf('##ETHERPAD_TRAVIS_V=');
+    const existingValue =
+        parseInt(existingConfig.substr(existingConfigLocation + 20, existingConfig.length));
+
+    const newConfigLocation = travisConfig.indexOf('##ETHERPAD_TRAVIS_V=');
+    const newValue = parseInt(travisConfig.substr(newConfigLocation + 20, travisConfig.length));
+    if (existingConfigLocation === -1) {
+      console.warn('no previous .travis.yml version found so writing new.');
+      // we will write the newTravisConfig to the location.
+      fs.writeFileSync(`${pluginPath}/.travis.yml`, travisConfig);
+    } else if (newValue > existingValue) {
+      console.log('updating .travis.yml');
+      fs.writeFileSync(`${pluginPath}/.travis.yml`, travisConfig);
+    }//
+  }
+
+  if (files.indexOf('.gitignore') === -1) {
+    console.warn('.gitignore file not found, please create.  .gitignore files are useful to ' +
+                 "ensure files aren't incorrectly commited to a repository.");
+    if (autoFix) {
+      console.log('Autofixing missing .gitignore file');
+      const gitignore = fs.readFileSync('bin/plugins/lib/gitignore', {encoding: 'utf8', flag: 'r'});
+      fs.writeFileSync(`${pluginPath}/.gitignore`, gitignore);
+    }
+  } else {
+    let gitignore =
+        fs.readFileSync(`${pluginPath}/.gitignore`, {encoding: 'utf8', flag: 'r'});
+    if (gitignore.indexOf('node_modules/') === -1) {
+      console.warn('node_modules/ missing from .gitignore');
+      if (autoFix) {
+        gitignore += 'node_modules/';
+        fs.writeFileSync(`${pluginPath}/.gitignore`, gitignore);
+      }
+    }
+  }
+
+  // if we include templates but don't have translations...
+  if (files.indexOf('templates') !== -1 && files.indexOf('locales') === -1) {
+    console.warn('Translations not found, please create.  ' +
+                 'Translation files help with Etherpad accessibility.');
+  }
+
+
+  if (files.indexOf('.ep_initialized') !== -1) {
+    console.warn(
+        '.ep_initialized found, please remove.  .ep_initialized should never be commited to git ' +
+        'and should only exist once the plugin has been executed one time.');
+    if (autoFix) {
+      console.log('Autofixing incorrectly existing .ep_initialized file');
+      fs.unlinkSync(`${pluginPath}/.ep_initialized`);
+    }
+  }
+
+  if (files.indexOf('npm-debug.log') !== -1) {
+    console.warn('npm-debug.log found, please remove.  npm-debug.log should never be commited to ' +
+                 'your repository.');
+    if (autoFix) {
+      console.log('Autofixing incorrectly existing npm-debug.log file');
+      fs.unlinkSync(`${pluginPath}/npm-debug.log`);
+    }
+  }
+
+  if (files.indexOf('static') !== -1) {
+    fs.readdir(`${pluginPath}/static`, (errRead, staticFiles) => {
+      if (staticFiles.indexOf('tests') === -1) {
+        console.warn('Test files not found, please create tests.  https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin');
+      }
+    });
+  } else {
+    console.warn('Test files not found, please create tests.  https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin');
+  }
+
+  // Install dependencies so we can run ESLint. This should also create or update package-lock.json
+  // if autoFix is enabled.
+  const npmInstall = `npm install${autoFix ? '' : ' --no-package-lock'}`;
+  execSync(npmInstall, {stdio: 'inherit'});
+  // The ep_etherpad-lite peer dep must be installed last otherwise `npm install` will nuke it. An
+  // absolute path to etherpad-lite/src is used here so that pluginPath can be a symlink.
+  execSync(
+      `${npmInstall} --no-save ep_etherpad-lite@file:${__dirname}/../../`, {stdio: 'inherit'});
+
+  // linting begins
+  try {
+    console.log('Linting...');
+    const lintCmd = autoFix ? 'npx eslint --fix .' : 'npx eslint';
+    execSync(lintCmd, {stdio: 'inherit'});
+  } catch (e) {
+    // it is gonna throw an error anyway
+    console.log('Manual linting probably required, check with: npm run lint');
+  }
+  // linting ends.
+
+  if (autoFix) {
+    const unchanged = JSON.parse(execSync(
+        'untracked=$(git ls-files -o --exclude-standard) || exit 1; ' +
+        'git diff-files --quiet && [ -z "$untracked" ] && echo true || echo false'));
+    if (!unchanged) {
+      // Display a diff of changes. Git doesn't diff untracked files, so they must be added to the
+      // index. Use a temporary index file to avoid modifying Git's default index file.
+      execSync('git read-tree HEAD; git add -A && git diff-index -p --cached HEAD && echo ""', {
+        env: {...process.env, GIT_INDEX_FILE: '.git/checkPlugin.index'},
+        stdio: 'inherit',
+      });
+      fs.unlinkSync(`${pluginPath}/.git/checkPlugin.index`);
+
+      const cmd = [
+        'git add -A',
+        'git commit -m "autofixes from Etherpad checkPlugin.js"',
+        'git push',
+      ].join(' && ');
+      if (autoCommit) {
+        console.log('Attempting autocommit and auto publish to npm');
+        execSync(cmd, {stdio: 'inherit'});
+      } else {
+        console.log('Fixes applied. Check the above git diff then run the following command:');
+        console.log(`(cd node_modules/${pluginName} && ${cmd})`);
+      }
+    } else {
+      console.log('No changes.');
+    }
+  }
+
+  console.log('Finished');
+});

src/bin/plugins/getCorePlugins.sh

@@ -0,0 +1,4 @@
+cd node_modules/
+GHUSER=ether; curl "https://api.github.com/users/$GHUSER/repos?per_page=100" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=ether; curl "https://api.github.com/users/$GHUSER/repos?per_page=100&page=2&" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=ether; curl "https://api.github.com/users/$GHUSER/repos?per_page=100&page=3&" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone

src/bin/plugins/lib/CONTRIBUTING.md

@@ -0,0 +1,133 @@
+# Contributor Guidelines
+(Please talk to people on the mailing list before you change this page, see our section on [how to get in touch](https://github.com/ether/etherpad-lite#get-in-touch))
+
+## Pull requests
+
+* the commit series in the PR should be _linear_ (it **should not contain merge commits**). This is necessary because we want to be able to [bisect](https://en.wikipedia.org/wiki/Bisection_(software_engineering)) bugs easily. Rewrite history/perform a rebase if necessary
+* PRs should be issued against the **develop** branch: we never pull directly into **master**
+* PRs **should not have conflicts** with develop. If there are, please resolve them rebasing and force-pushing
+* when preparing your PR, please make sure that you have included the relevant **changes to the documentation** (preferably with usage examples)
+* contain meaningful and detailed **commit messages** in the form:
+  ```
+  submodule: description
+
+  longer description of the change you have made, eventually mentioning the
+  number of the issue that is being fixed, in the form: Fixes #someIssueNumber
+  ```
+* if the PR is a **bug fix**:
+  * the first commit in the series must be a test that shows the failure
+  * subsequent commits will fix the bug and make the test pass
+  * the final commit message should include the text `Fixes: #xxx` to link it to its bug report
+* think about stability: code has to be backwards compatible as much as possible. Always **assume your code will be run with an older version of the DB/config file**
+* if you want to remove a feature, **deprecate it instead**:
+  * write an issue with your deprecation plan
+  * output a `WARN` in the log informing that the feature is going to be removed
+  * remove the feature in the next version
+* if you want to add a new feature, put it under a **feature flag**:
+  * once the new feature has reached a minimal level of stability, do a PR for it, so it can be integrated early
+  * expose a mechanism for enabling/disabling the feature
+  * the new feature should be **disabled** by default. With the feature disabled, the code path should be exactly the same as before your contribution. This is a __necessary condition__ for early integration
+* think of the PR not as something that __you wrote__, but as something that __someone else is going to read__. The commit series in the PR should tell a novice developer the story of your thoughts when developing it
+
+## How to write a bug report
+
+* Please be polite, we all are humans and problems can occur.
+* Please add as much information as possible, for example
+  * client os(s) and version(s)
+    * browser(s) and version(s), is the problem reproducible on different clients
+    * special environments like firewalls or antivirus
+  * host os and version
+    * npm and nodejs version
+    * Logfiles if available
+  * steps to reproduce
+  * what you expected to happen
+  * what actually happened
+* Please format logfiles and code examples with markdown see github Markdown help below the issue textarea for more information.
+
+If you send logfiles, please set the loglevel switch DEBUG in your settings.json file:
+
+```
+/* The log level we are using, can be: DEBUG, INFO, WARN, ERROR */
+  "loglevel": "DEBUG",
+```
+
+The logfile location is defined in startup script or the log is directly shown in the commandline after you have started etherpad.
+
+## General goals of Etherpad
+To make sure everybody is going in the same direction:
+* easy to install for admins and easy to use for people
+* easy to integrate into other apps, but also usable as standalone
+* lightweight and scalable
+* extensible, as much functionality should be extendable with plugins so changes don't have to be done in core.
+Also, keep it maintainable. We don't wanna end up as the monster Etherpad was!
+
+## How to work with git?
+* Don't work in your master branch.
+* Make a new branch for every feature you're working on. (This ensures that you can work you can do lots of small, independent pull requests instead of one big one with complete different features)
+* Don't use the online edit function of github (this only creates ugly and not working commits!)
+* Try to make clean commits that are easy readable (including descriptive commit messages!)
+* Test before you push. Sounds easy, it isn't!
+* Don't check in stuff that gets generated during build or runtime
+* Make small pull requests that are easy to review but make sure they do add value by themselves / individually
+
+## Coding style
+* Do write comments. (You don't have to comment every line, but if you come up with something that's a bit complex/weird, just leave a comment. Bear in mind that you will probably leave the project at some point and that other people will read your code. Undocumented huge amounts of code are worthless!)
+* Never ever use tabs
+* Indentation: JS/CSS: 2 spaces; HTML: 4 spaces
+* Don't overengineer. Don't try to solve any possible problem in one step, but try to solve problems as easy as possible and improve the solution over time!
+* Do generalize sooner or later! (if an old solution, quickly hacked together, poses more problems than it solves today, refactor it!)
+* Keep it compatible. Do not introduce changes to the public API, db schema or configurations too lightly. Don't make incompatible changes without good reasons!
+* If you do make changes, document them! (see below)
+* Use protocol independent urls "//"
+
+## Branching model / git workflow
+see git flow http://nvie.com/posts/a-successful-git-branching-model/
+
+### `master` branch
+* the stable
+* This is the branch everyone should use for production stuff
+
+### `develop`branch
+* everything that is READY to go into master at some point in time
+* This stuff is tested and ready to go out
+
+### release branches
+* stuff that should go into master very soon
+* only bugfixes go into these (see http://nvie.com/posts/a-successful-git-branching-model/ for why)
+* we should not be blocking new features to develop, just because we feel that we should be releasing it to master soon. This is the situation that release branches solve/handle.
+
+### hotfix branches
+* fixes for bugs in master
+
+### feature branches (in your own repos)
+* these are the branches where you develop your features in
+* If it's ready to go out, it will be merged into develop
+
+Over the time we pull features from feature branches into the develop branch. Every month we pull from develop into master. Bugs in master get fixed in hotfix branches. These branches will get merged into master AND develop. There should never be commits in master that aren't in develop
+
+## Documentation
+The docs are in the `doc/` folder in the git repository, so people can easily find the suitable docs for the current git revision.
+
+Documentation should be kept up-to-date. This means, whenever you add a new API method, add a new hook or change the database model, pack the relevant changes to the docs in the same pull request.
+
+You can build the docs e.g. produce html, using `make docs`. At some point in the future we will provide an online documentation. The current documentation in the github wiki should always reflect the state of `master` (!), since there are no docs in master, yet.
+
+## Testing
+Front-end tests are found in the `tests/frontend/` folder in the repository. Run them by pointing your browser to `<yourdomainhere>/tests/frontend`.
+
+Back-end tests can be run from the `src` directory, via `npm test`.
+
+## Things you can help with
+Etherpad is much more than software.  So if you aren't a developer then worry not, there is still a LOT you can do!  A big part of what we do is community engagement.  You can help in the following ways
+ * Triage bugs (applying labels) and confirming their existence
+ * Testing fixes (simply applying them and seeing if it fixes your issue or not) - Some git experience required
+ * Notifying large site admins of new releases
+ * Writing Changelogs for releases
+ * Creating Windows packages
+ * Creating releases
+ * Bumping dependencies periodically and checking they don't break anything
+ * Write proposals for grants
+ * Co-Author and Publish CVEs
+ * Work with SFC to maintain legal side of project
+ * Maintain TODO page - https://github.com/ether/etherpad-lite/wiki/TODO#IMPORTANT_TODOS
+

src/bin/plugins/lib/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

src/bin/plugins/lib/README.md

@@ -0,0 +1,29 @@
+[![Travis (.com)](https://api.travis-ci.com/[org_name]/[repo_url].svg?branch=develop)](https://travis-ci.com/github/[org_name]/[repo_url])
+
+# My awesome plugin README example
+Explain what your plugin does and who it's useful for.
+
+## Example animated gif of usage if appropriate
+![screenshot](https://user-images.githubusercontent.com/220864/99979953-97841d80-2d9f-11eb-9782-5f65817c58f4.PNG)
+
+## Installing
+npm install [plugin_name]
+
+or Use the Etherpad ``/admin`` interface.
+
+## Settings
+Document settings if any
+
+## Testing
+Document how to run backend / frontend tests.
+
+### Frontend
+
+Visit http://whatever/tests/frontend/ to run the frontend tests.
+
+### backend
+
+Type ``cd src && npm run test`` to run the backend tests.
+
+## LICENSE
+Apache 2.0

src/bin/plugins/lib/backend-tests.yml

@@ -0,0 +1,51 @@
+# You need to change lines 38 and 46 in case the plugin's name on npmjs.com is different
+# from the repository name
+
+name: "Backend tests"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  withplugins:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: with Plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Install libreoffice
+      run: |
+        sudo add-apt-repository -y ppa:libreoffice/ppa
+        sudo apt update
+        sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+
+    # clone etherpad-lite
+    - name: Install etherpad core
+      uses: actions/checkout@v2
+      with:
+        repository: ether/etherpad-lite
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    # clone this repository into node_modules/ep_plugin-name
+    - name: Checkout plugin repository
+      uses: actions/checkout@v2
+      with:
+        path: ./node_modules/${{github.event.repository.name}}
+
+    - name: Install plugin dependencies
+      run: |
+        cd node_modules/${{github.event.repository.name}}
+        npm ci
+
+    # configures some settings and runs npm run test
+    - name: Run the backend tests
+      run: tests/frontend/travis/runnerBackend.sh
+
+##ETHERPAD_NPM_V=1
+## NPM configuration automatically created using bin/plugins/updateAllPluginsScript.sh

src/bin/plugins/lib/gitignore

@@ -0,0 +1,5 @@
+.ep_initialized
+.DS_Store
+node_modules/
+node_modules
+npm-debug.log

src/bin/plugins/lib/npmpublish.yml

@@ -0,0 +1,83 @@
+# This workflow will run tests using node and then publish a package to the npm registry when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: Node.js Package
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      # Clone ether/etherpad-lite to ../etherpad-lite so that ep_etherpad-lite
+      # can be "installed" in this plugin's node_modules. The checkout v2 action
+      # doesn't support cloning outside of $GITHUB_WORKSPACE (see
+      # https://github.com/actions/checkout/issues/197), so the repo is first
+      # cloned to etherpad-lite then moved to ../etherpad-lite. To avoid
+      # conflicts with this plugin's clone, etherpad-lite must be cloned and
+      # moved out before this plugin's repo is cloned to $GITHUB_WORKSPACE.
+      - uses: actions/checkout@v2
+        with:
+          repository: ether/etherpad-lite
+          path: etherpad-lite
+      - run: mv etherpad-lite ..
+      # etherpad-lite has been moved outside of $GITHUB_WORKSPACE, so it is now
+      # safe to clone this plugin's repo to $GITHUB_WORKSPACE.
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12
+      # All of ep_etherpad-lite's devDependencies are installed because the
+      # plugin might do `require('ep_etherpad-lite/node_modules/${devDep}')`.
+      # Eventually it would be nice to create an ESLint plugin that prohibits
+      # Etherpad plugins from piggybacking off of ep_etherpad-lite's
+      # devDependencies. If we had that, we could change this line to only
+      # install production dependencies.
+      - run: cd ../etherpad-lite/src && npm ci
+      - run: npm ci
+      # This runs some sanity checks and creates a symlink at
+      # node_modules/ep_etherpad-lite that points to ../../etherpad-lite/src.
+      # This step must be done after `npm ci` installs the plugin's dependencies
+      # because npm "helpfully" cleans up such symlinks. :( Installing
+      # ep_etherpad-lite in the plugin's node_modules prevents lint errors and
+      # unit test failures if the plugin does `require('ep_etherpad-lite/foo')`.
+      - run: npm install --no-save ep_etherpad-lite@file:../etherpad-lite/src
+      - run: npm test
+      - run: npm run lint
+
+  publish-npm:
+    if: github.event_name == 'push'
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12
+          registry-url: https://registry.npmjs.org/
+      - run: git config user.name 'github-actions[bot]'
+      - run: git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+      - run: npm ci
+      - run: npm version patch
+      - run: git push --follow-tags
+      # `npm publish` must come after `git push` otherwise there is a race
+      # condition: If two PRs are merged back-to-back then master/main will be
+      # updated with the commits from the second PR before the first PR's
+      # workflow has a chance to push the commit generated by `npm version
+      # patch`. This causes the first PR's `git push` step to fail after the
+      # package has already been published, which in turn will cause all future
+      # workflow runs to fail because they will all attempt to use the same
+      # already-used version number. By running `npm publish` after `git push`,
+      # back-to-back merges will cause the first merge's workflow to fail but
+      # the second's will succeed.
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
+
+##ETHERPAD_NPM_V=2
+## NPM configuration automatically created using bin/plugins/updateAllPluginsScript.sh

src/bin/plugins/lib/travis.yml

@@ -0,0 +1,70 @@
+language: node_js
+
+node_js:
+  - "lts/*"
+
+cache: false
+
+services:
+  - docker
+
+install:
+  - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+
+#script:
+#  - "tests/frontend/travis/runner.sh"
+
+env:
+  global:
+    - secure: "WMGxFkOeTTlhWB+ChMucRtIqVmMbwzYdNHuHQjKCcj8HBEPdZLfCuK/kf4rG\nVLcLQiIsyllqzNhBGVHG1nyqWr0/LTm8JRqSCDDVIhpyzp9KpCJQQJG2Uwjk\n6/HIJJh/wbxsEdLNV2crYU/EiVO3A4Bq0YTHUlbhUqG3mSCr5Ec="
+    - secure: "gejXUAHYscbR6Bodw35XexpToqWkv2ifeECsbeEmjaLkYzXmUUNWJGknKSu7\nEUsSfQV8w+hxApr1Z+jNqk9aX3K1I4btL3cwk2trnNI8XRAvu1c1Iv60eerI\nkE82Rsd5lwUaMEh+/HoL8ztFCZamVndoNgX7HWp5J/NRZZMmh4g="
+
+jobs:
+  include:
+    - name: "Lint test package-lock"
+      install:
+        - "npm install lockfile-lint"
+      script:
+        - npx lockfile-lint --path package-lock.json --validate-https --allowed-hosts npm
+    - name: "Run the Backend tests"
+      before_install:
+        - sudo add-apt-repository -y ppa:libreoffice/ppa
+        - sudo apt-get update
+        - sudo apt-get -y install libreoffice
+        - sudo apt-get -y install libreoffice-pdfimport
+      install:
+        - "npm install"
+        - "mkdir [plugin_name]"
+        - "mv !([plugin_name]) [plugin_name]"
+        - "git clone https://github.com/ether/etherpad-lite.git etherpad"
+        - "cd etherpad"
+        - "mkdir -p node_modules"
+        - "mv ../[plugin_name] node_modules"
+        - "bin/installDeps.sh"
+        - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+        - "cd src && npm install && cd -"
+      script:
+        - "tests/frontend/travis/runnerBackend.sh"
+    - name: "Test the Frontend"
+      before_script:
+        - "tests/frontend/travis/sauce_tunnel.sh"
+      install:
+        - "npm install"
+        - "mkdir [plugin_name]"
+        - "mv !([plugin_name]) [plugin_name]"
+        - "git clone https://github.com/ether/etherpad-lite.git etherpad"
+        - "cd etherpad"
+        - "mkdir -p node_modules"
+        - "mv ../[plugin_name] node_modules"
+        - "bin/installDeps.sh"
+        - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+      script:
+        - "tests/frontend/travis/runner.sh"
+
+notifications:
+  irc:
+    channels:
+      - "irc.freenode.org#etherpad-lite-dev"
+
+##ETHERPAD_TRAVIS_V=9
+## Travis configuration automatically created using bin/plugins/updateAllPluginsScript.sh

src/bin/plugins/reTestAllPlugins.sh

@@ -0,0 +1,14 @@
+echo "herp";
+for dir in `ls node_modules`;
+do
+  echo $dir
+  if [[ $dir == *"ep_"* ]]; then
+    if [[ $dir != "ep_etherpad-lite" ]]; then
+      # node bin/plugins/checkPlugin.js $dir autofix autocommit autoupdate
+      cd node_modules/$dir
+      git commit -m "Automatic update: bump update to re-run latest Etherpad tests" --allow-empty
+      git push origin master
+      cd ../..
+    fi
+  fi
+done

src/bin/plugins/updateAllPluginsScript.sh

@@ -0,0 +1,17 @@
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=2" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=3" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=4" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+for dir in `ls node_modules`;
+do
+  # echo $0
+  if [[ $dir == *"ep_"* ]]; then
+    if [[ $dir != "ep_etherpad-lite" ]]; then
+      node bin/plugins/checkPlugin.js $dir autofix autocommit autoupdate
+    fi
+  fi
+  # echo $dir
+done

src/bin/plugins/updateCorePlugins.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+for dir in node_modules/ep_*; do
+  dir=${dir#node_modules/}
+  [ "$dir" != ep_etherpad-lite ] || continue
+  node bin/plugins/checkPlugin.js "$dir" autofix autocommit autoupdate
+done

src/bin/rebuildPad.js

@@ -0,0 +1,84 @@
+'use strict';
+
+/*
+  This is a repair tool. It rebuilds an old pad at a new pad location up to a
+  known "good" revision.
+*/
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 4 && process.argv.length !== 5) {
+  throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]');
+}
+
+const padId = process.argv[2];
+const newRevHead = process.argv[3];
+const newPadId = process.argv[4] || `${padId}-rebuilt`;
+
+(async () => {
+  const db = require('../node/db/DB');
+  await db.init();
+
+  const PadManager = require('../node/db/PadManager');
+  const Pad = require('../node/db/Pad').Pad;
+  // Validate the newPadId if specified and that a pad with that ID does
+  // not already exist to avoid overwriting it.
+  if (!PadManager.isValidPadId(newPadId)) {
+    throw new Error('Cannot create a pad with that id as it is invalid');
+  }
+  const exists = await PadManager.doesPadExist(newPadId);
+  if (exists) throw new Error('Cannot create a pad with that id as it already exists');
+
+  const oldPad = await PadManager.getPad(padId);
+  const newPad = new Pad(newPadId);
+
+  // Clone all Chat revisions
+  const chatHead = oldPad.chatHead;
+  await Promise.all([...Array(chatHead + 1).keys()].map(async (i) => {
+    const chat = await db.get(`pad:${padId}:chat:${i}`);
+    await db.set(`pad:${newPadId}:chat:${i}`, chat);
+    console.log(`Created: Chat Revision: pad:${newPadId}:chat:${i}`);
+  }));
+
+  // Rebuild Pad from revisions up to and including the new revision head
+  const AuthorManager = require('../node/db/AuthorManager');
+  const Changeset = require('../static/js/Changeset');
+  // Author attributes are derived from changesets, but there can also be
+  // non-author attributes with specific mappings that changesets depend on
+  // and, AFAICT, cannot be recreated any other way
+  newPad.pool.numToAttrib = oldPad.pool.numToAttrib;
+  for (let curRevNum = 0; curRevNum <= newRevHead; curRevNum++) {
+    const rev = await db.get(`pad:${padId}:revs:${curRevNum}`);
+    if (!rev || !rev.meta) throw new Error('The specified revision number could not be found.');
+    const newRevNum = ++newPad.head;
+    const newRevId = `pad:${newPad.id}:revs:${newRevNum}`;
+    await Promise.all([
+      db.set(newRevId, rev),
+      AuthorManager.addPad(rev.meta.author, newPad.id),
+    ]);
+    newPad.atext = Changeset.applyToAText(rev.changeset, newPad.atext, newPad.pool);
+    console.log(`Created: Revision: pad:${newPad.id}:revs:${newRevNum}`);
+  }
+
+  // Add saved revisions up to the new revision head
+  console.log(newPad.head);
+  const newSavedRevisions = [];
+  for (const savedRev of oldPad.savedRevisions) {
+    if (savedRev.revNum <= newRevHead) {
+      newSavedRevisions.push(savedRev);
+      console.log(`Added: Saved Revision: ${savedRev.revNum}`);
+    }
+  }
+  newPad.savedRevisions = newSavedRevisions;
+
+  // Save the source pad
+  await db.set(`pad:${newPadId}`, newPad);
+
+  console.log(`Created: Source Pad: pad:${newPadId}`);
+  await newPad.saveToDatabase();
+
+  await db.shutdown();
+  console.info('finished');
+})();

src/bin/release.js

@@ -0,0 +1,74 @@
+'use strict';
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const fs = require('fs');
+const childProcess = require('child_process');
+const semver = require('semver');
+
+/*
+
+Usage
+
+node bin/release.js patch
+
+*/
+const usage = 'node bin/release.js [patch/minor/major] -- example: "node bin/release.js patch"';
+
+const release = process.argv[2];
+
+if (!release) {
+  console.log(usage);
+  throw new Error('No release type included');
+}
+
+const changelog = fs.readFileSync('CHANGELOG.md', {encoding: 'utf8', flag: 'r'});
+let packageJson = fs.readFileSync('./src/package.json', {encoding: 'utf8', flag: 'r'});
+packageJson = JSON.parse(packageJson);
+const currentVersion = packageJson.version;
+
+const newVersion = semver.inc(currentVersion, release);
+if (!newVersion) {
+  console.log(usage);
+  throw new Error('Unable to generate new version from input');
+}
+
+const changelogIncludesVersion = changelog.indexOf(newVersion) !== -1;
+
+if (!changelogIncludesVersion) {
+  throw new Error('No changelog record for ', newVersion, ' - please create changelog record');
+}
+
+console.log('Okay looks good, lets create the package.json and package-lock.json');
+
+packageJson.version = newVersion;
+
+fs.writeFileSync('src/package.json', JSON.stringify(packageJson, null, 2));
+
+// run npm version `release` where release is patch, minor or major
+childProcess.execSync('npm install --package-lock-only', {cwd: 'src/'});
+// run npm install --package-lock-only <-- required???
+
+childProcess.execSync(`git checkout -b release/${newVersion}`);
+childProcess.execSync('git add src/package.json');
+childProcess.execSync('git add src/package-lock.json');
+childProcess.execSync('git commit -m "bump version"');
+childProcess.execSync(`git push origin release/${newVersion}`);
+
+
+childProcess.execSync('make docs');
+childProcess.execSync('git clone git@github.com:ether/ether.github.com.git');
+childProcess.execSync(`cp -R out/doc/ ether.github.com/doc/v${newVersion}`);
+
+console.log('Once merged into master please run the following commands');
+console.log(`git tag -a ${newVersion} -m ${newVersion} && git push origin master`);
+console.log(`cd ether.github.com && git add . && git commit -m '${newVersion} docs'`);
+console.log('Build the windows zip');
+console.log('Visit https://github.com/ether/etherpad-lite/releases/new and create a new release ' +
+            `with 'master' as the target and the version is ${newVersion}.  Include the windows ` +
+            'zip as an asset');
+console.log(`Once the new docs are uploaded then modify the download
+   link on etherpad.org and then pull master onto develop`);
+console.log('Finally go public with an announcement via our comms channels :)');

src/bin/repairPad.js

@@ -0,0 +1,56 @@
+'use strict';
+
+/*
+ * This is a repair tool. It extracts all datas of a pad, removes and inserts them again.
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+console.warn('WARNING: This script must not be used while etherpad is running!');
+
+if (process.argv.length !== 3) throw new Error('Use: node bin/repairPad.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+let valueCount = 0;
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // get the pad
+  const padManager = require('../node/db/PadManager');
+  const pad = await padManager.getPad(padId);
+
+  // accumulate the required keys
+  const neededDBValues = [`pad:${padId}`];
+
+  // add all authors
+  neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`));
+
+  // add all revisions
+  for (let rev = 0; rev <= pad.head; ++rev) {
+    neededDBValues.push(`pad:${padId}:revs:${rev}`);
+  }
+
+  // add all chat values
+  for (let chat = 0; chat <= pad.chatHead; ++chat) {
+    neededDBValues.push(`pad:${padId}:chat:${chat}`);
+  }
+  // now fetch and reinsert every key
+  for (const key of neededDBValues) {
+    const value = await db.get(key);
+    // if it isn't a globalAuthor value which we want to ignore..
+    // console.log(`Key: ${key}, value: ${JSON.stringify(value)}`);
+    await db.remove(key);
+    await db.set(key, value);
+    valueCount++;
+  }
+
+  console.info(`Finished: Replaced ${valueCount} values in the database`);
+})();

src/bin/run.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+ignoreRoot=0
+for ARG in "$@"; do
+  if [ "$ARG" = "--root" ]; then
+    ignoreRoot=1
+  fi
+done
+
+# Stop the script if it's started as root
+if [ "$(id -u)" -eq 0 ] && [ "$ignoreRoot" -eq 0 ]; then
+  cat <<EOF >&2
+You shouldn't start Etherpad as root!
+Please type 'Etherpad rocks my socks' (or restart with the '--root'
+argument) if you still want to start it as root:
+EOF
+  printf "> " >&2
+  read rocks
+  [ "$rocks" = "Etherpad rocks my socks" ] || fatal "Your input was incorrect"
+fi
+
+# Prepare the environment
+bin/installDeps.sh "$@" || exit 1
+
+# Move to the node folder and start
+log "Starting Etherpad..."
+
+SCRIPTPATH=$(pwd -P)
+exec node $(compute_node_args) "$SCRIPTPATH/node_modules/ep_etherpad-lite/node/server.js" "$@"

src/bin/safeRun.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+# This script ensures that ep-lite is automatically restarting after
+# an error happens
+
+# Handling Errors
+#   0 silent
+#   1 email
+ERROR_HANDLING=0
+# Your email address which should receive the error messages
+EMAIL_ADDRESS="no-reply@example.com"
+# Sets the minimum amount of time between the sending of error emails.
+# This ensures you do not get spammed during an endless reboot loop
+# It's the time in seconds
+TIME_BETWEEN_EMAILS=600 # 10 minutes
+
+# DON'T EDIT AFTER THIS LINE
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+
+LAST_EMAIL_SEND=0
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Check if a logfile parameter is set
+LOG="$1"
+[ -n "${LOG}" ] || fatal "Set a logfile as the first parameter"
+shift
+
+while true; do
+  # Try to touch the file if it doesn't exist
+  [ -f "${LOG}" ] || touch "${LOG}" || fatal "Logfile '${LOG}' is not writeable"
+
+  # Check if the file is writeable
+  [ -w "${LOG}" ] || fatal "Logfile '${LOG}' is not writeable"
+
+  # Start the application
+  bin/run.sh "$@" >>${LOG} 2>>${LOG}
+
+  TIME_FMT=$(date +%Y-%m-%dT%H:%M:%S%z)
+
+  # Send email
+  if [ "$ERROR_HANDLING" = 1 ]; then
+    TIME_NOW=$(date +%s)
+    TIME_SINCE_LAST_SEND=$(($TIME_NOW - $LAST_EMAIL_SEND))
+
+    if [ "$TIME_SINCE_LAST_SEND" -gt "$TIME_BETWEEN_EMAILS" ]; then
+      {
+        cat <<EOF
+Server was restarted at: ${TIME_FMT}
+The last 50 lines of the log before the server exited:
+
+EOF
+        tail -n 50 "${LOG}"
+      } | mail -s "Etherpad restarted" "$EMAIL_ADDRESS"
+
+      LAST_EMAIL_SEND=$TIME_NOW
+    fi
+  fi
+
+  pecho "RESTART! ${TIME_FMT}" >>${LOG}
+
+  # Sleep 10 seconds before restart
+  sleep 10
+done

src/bin/updatePlugins.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+#Move to the folder where ep-lite is installed
+cd $(dirname $0)
+
+#Was this script started in the bin folder? if yes move out
+if [ -d "../bin" ]; then
+  cd "../"
+fi
+
+# npm outdated --depth=0 | grep -v "^Package" | awk '{print $1}' | xargs npm install $1 --save-dev
+OUTDATED=$(npm outdated --depth=0 | grep -v "^Package" | awk '{print $1}')
+# echo $OUTDATED
+if test -n "$OUTDATED"; then
+  echo "Plugins require update, doing this now..."
+  echo "Updating $OUTDATED"
+  npm install $OUTDATED --save-dev
+else
+  echo "Plugins are all up to date"
+fi

src/package.json

@@ -104,7 +104,8 @@
       "/static/js/gritter.js",
       "/static/js/html10n.js",
       "/static/js/jquery.js",
-      "/static/js/vendors/nice-select.js"
+      "/static/js/vendors/nice-select.js",
+      "/tests/frontend/lib/"
     ],
     "overrides": [
       {
@@ -118,7 +119,8 @@
           "**/*"
         ],
         "excludedFiles": [
-          "**/.eslintrc.js"
+          "**/.eslintrc.js",
+          "tests/frontend/**/*"
         ],
         "extends": "etherpad/node"
       },
@@ -133,6 +135,74 @@
         "env": {
           "shared-node-browser": true
         }
+      },
+      {
+        "files": [
+          "tests/**/*"
+        ],
+        "excludedFiles": [
+          "**/.eslintrc.js",
+          "tests/frontend/travis/**/*",
+          "tests/ratelimit/**/*"
+        ],
+        "extends": "etherpad/tests",
+        "rules": {
+          "mocha/no-exports": "off",
+          "mocha/no-top-level-hooks": "off"
+        }
+      },
+      {
+        "files": [
+          "tests/backend/**/*"
+        ],
+        "excludedFiles": [
+          "**/.eslintrc.js"
+        ],
+        "extends": "etherpad/tests/backend",
+        "overrides": [
+          {
+            "files": [
+              "tests/backend/**/*"
+            ],
+            "excludedFiles": [
+              "tests/backend/specs/**/*"
+            ],
+            "rules": {
+              "mocha/no-exports": "off",
+              "mocha/no-top-level-hooks": "off"
+            }
+          }
+        ]
+      },
+      {
+        "files": [
+          "tests/frontend/**/*"
+        ],
+        "excludedFiles": [
+          "**/.eslintrc.js",
+          "tests/frontend/travis/**/*"
+        ],
+        "extends": "etherpad/tests/frontend",
+        "overrides": [
+          {
+            "files": [
+              "tests/frontend/**/*"
+            ],
+            "excludedFiles": [
+              "tests/frontend/specs/**/*"
+            ],
+            "rules": {
+              "mocha/no-exports": "off",
+              "mocha/no-top-level-hooks": "off"
+            }
+          }
+        ]
+      },
+      {
+        "files": [
+          "tests/frontend/travis/**/*"
+        ],
+        "extends": "etherpad/node"
       }
     ],
     "root": true

src/tests

@@ -1 +0,0 @@
-../tests

src/tests/README.md

@@ -0,0 +1,11 @@
+# About this folder: Tests
+
+Before running the tests, start an Etherpad instance on your machine.
+
+## Frontend
+
+To run the frontend tests, point your browser to `<yourdomainhere>/tests/frontend`
+
+## Backend
+
+To run the backend tests, run `cd src` and then `npm test`

src/tests/backend/common.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const apiHandler = require('../../node/handler/APIHandler');
+const log4js = require('log4js');
+const process = require('process');
+const server = require('../../node/server');
+const settings = require('../../node/utils/Settings');
+const supertest = require('supertest');
+const webaccess = require('../../node/hooks/express/webaccess');
+
+const backups = {};
+let inited = false;
+
+exports.apiKey = apiHandler.exportedForTestingOnly.apiKey;
+exports.agent = null;
+exports.baseUrl = null;
+exports.httpServer = null;
+exports.logger = log4js.getLogger('test');
+
+const logLevel = exports.logger.level;
+
+// Mocha doesn't monitor unhandled Promise rejections, so convert them to uncaught exceptions.
+// https://github.com/mochajs/mocha/issues/2640
+process.on('unhandledRejection', (reason, promise) => { throw reason; });
+
+exports.init = async function () {
+  if (inited) return exports.agent;
+  inited = true;
+
+  if (!logLevel.isLessThanOrEqualTo(log4js.levels.DEBUG)) {
+    exports.logger.warn('Disabling non-test logging for the duration of the test. ' +
+                        'To enable non-test logging, change the loglevel setting to DEBUG.');
+    log4js.setGlobalLogLevel(log4js.levels.OFF);
+    exports.logger.setLevel(logLevel);
+  }
+
+  // Note: This is only a shallow backup.
+  backups.settings = Object.assign({}, settings);
+  // Start the Etherpad server on a random unused port.
+  settings.port = 0;
+  settings.ip = 'localhost';
+  exports.httpServer = await server.start();
+  exports.baseUrl = `http://localhost:${exports.httpServer.address().port}`;
+  exports.logger.debug(`HTTP server at ${exports.baseUrl}`);
+  // Create a supertest user agent for the HTTP server.
+  exports.agent = supertest(exports.baseUrl);
+  // Speed up authn tests.
+  backups.authnFailureDelayMs = webaccess.authnFailureDelayMs;
+  webaccess.authnFailureDelayMs = 0;
+
+  after(async function () {
+    webaccess.authnFailureDelayMs = backups.authnFailureDelayMs;
+    // Note: This does not unset settings that were added.
+    Object.assign(settings, backups.settings);
+    log4js.setGlobalLogLevel(logLevel);
+    await server.exit();
+  });
+
+  return exports.agent;
+};

src/tests/backend/fuzzImportTest.js

@@ -0,0 +1,68 @@
+/*
+ * Fuzz testing the import endpoint
+ * Usage: node fuzzImportTest.js
+ */
+const common = require('./common');
+const host = `http://${settings.ip}:${settings.port}`;
+const request = require('request');
+const froth = require('mocha-froth');
+const settings = require('../container/loadSettings').loadSettings();
+
+const apiKey = common.apiKey;
+const apiVersion = 1;
+const testPadId = `TEST_fuzz${makeid()}`;
+
+const endPoint = function (point, version) {
+  version = version || apiVersion;
+  return `/api/${version}/${point}?apikey=${apiKey}`;
+};
+
+console.log('Testing against padID', testPadId);
+console.log(`To watch the test live visit ${host}/p/${testPadId}`);
+console.log('Tests will start in 5 seconds, click the URL now!');
+
+setTimeout(() => {
+  for (let i = 1; i < 1000000; i++) { // 1M runs
+    setTimeout(() => {
+      runTest(i);
+    }, i * 100); // 100 ms
+  }
+}, 5000); // wait 5 seconds
+
+function runTest(number) {
+  request(`${host + endPoint('createPad')}&padID=${testPadId}`, (err, res, body) => {
+    const req = request.post(`${host}/p/${testPadId}/import`, (err, res, body) => {
+      if (err) {
+        throw new Error('FAILURE', err);
+      } else {
+        console.log('Success');
+      }
+    });
+
+    let fN = '/test.txt';
+    let cT = 'text/plain';
+
+    // To be more aggressive every other test we mess with Etherpad
+    // We provide a weird file name and also set a weird contentType
+    if (number % 2 == 0) {
+      fN = froth().toString();
+      cT = froth().toString();
+    }
+
+    const form = req.form();
+    form.append('file', froth().toString(), {
+      filename: fN,
+      contentType: cT,
+    });
+  });
+}
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/api.js

@@ -0,0 +1,75 @@
+/**
+ * API specs
+ *
+ * Tests for generic overarching HTTP API related features not related to any
+ * specific part of the data model or domain. For example: tests for versioning
+ * and openapi definitions.
+ */
+
+const common = require('../../common');
+const settings = require('../../../../node/utils/Settings');
+const supertest = require('supertest');
+const validateOpenAPI = require('openapi-schema-validation').validate;
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+const apiKey = common.apiKey;
+let apiVersion = 1;
+
+const testPadId = makeid();
+
+describe(__filename, function () {
+  describe('API Versioning', function () {
+    it('errors if can not connect', function (done) {
+      api
+          .get('/api/')
+          .expect((res) => {
+            apiVersion = res.body.currentVersion;
+            if (!res.body.currentVersion) throw new Error('No version set in API');
+            return;
+          })
+          .expect(200, done);
+    });
+  });
+
+  describe('OpenAPI definition', function () {
+    it('generates valid openapi definition document', function (done) {
+      api
+          .get('/api/openapi.json')
+          .expect((res) => {
+            const {valid, errors} = validateOpenAPI(res.body, 3);
+            if (!valid) {
+              const prettyErrors = JSON.stringify(errors, null, 2);
+              throw new Error(`Document is not valid OpenAPI. ${errors.length} validation errors:\n${prettyErrors}`);
+            }
+            return;
+          })
+          .expect(200, done);
+    });
+  });
+
+  describe('jsonp support', function () {
+    it('supports jsonp calls', function (done) {
+      api
+          .get(`${endPoint('createPad')}&jsonp=jsonp_1&padID=${testPadId}`)
+          .expect((res) => {
+            if (!res.text.match('jsonp_1')) throw new Error('no jsonp call seen');
+          })
+          .expect('Content-Type', /javascript/)
+          .expect(200, done);
+    });
+  });
+});
+
+var endPoint = function (point) {
+  return `/api/${apiVersion}/${point}?apikey=${apiKey}`;
+};
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/characterEncoding.js

@@ -0,0 +1,111 @@
+'use strict';
+
+/*
+ * This file is copied & modified from <basedir>/tests/backend/specs/api/pad.js
+ *
+ * TODO: maybe unify those two files and merge in a single one.
+ */
+
+const common = require('../../common');
+const fs = require('fs');
+const settings = require('../../../../node/utils/Settings');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+const apiKey = common.apiKey;
+let apiVersion = 1;
+const testPadId = makeid();
+
+describe(__filename, function () {
+  describe('Connectivity For Character Encoding', function () {
+    it('can connect', function (done) {
+      api.get('/api/')
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('API Versioning', function () {
+    it('finds the version tag', function (done) {
+      api.get('/api/')
+          .expect((res) => {
+            apiVersion = res.body.currentVersion;
+            if (!res.body.currentVersion) throw new Error('No version set in API');
+            return;
+          })
+          .expect(200, done);
+    });
+  });
+
+  describe('Permission', function () {
+    it('errors with invalid APIKey', function (done) {
+      // This is broken because Etherpad doesn't handle HTTP codes properly see #2343
+      // If your APIKey is password you deserve to fail all tests anyway
+      const permErrorURL = `/api/${apiVersion}/createPad?apikey=password&padID=test`;
+      api.get(permErrorURL)
+          .expect(401, done);
+    });
+  });
+
+  describe('createPad', function () {
+    it('creates a new Pad', function (done) {
+      api.get(`${endPoint('createPad')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to create new Pad');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setHTML', function () {
+    it('Sets the HTML of a Pad attempting to weird utf8 encoded content', function (done) {
+      fs.readFile('../tests/backend/specs/api/emojis.html', 'utf8', (err, html) => {
+        api.post(endPoint('setHTML'))
+            .send({
+              padID: testPadId,
+              html,
+            })
+            .expect((res) => {
+              if (res.body.code !== 0) throw new Error("Can't set HTML properly");
+            })
+            .expect('Content-Type', /json/)
+            .expect(200, done);
+      });
+    });
+  });
+
+  describe('getHTML', function () {
+    it('get the HTML of Pad with emojis', function (done) {
+      api.get(`${endPoint('getHTML')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.html.indexOf('&#127484') === -1) {
+              throw new Error('Unable to get the HTML');
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+});
+
+/*
+
+  End of test
+
+*/
+
+var endPoint = function (point, version) {
+  version = version || apiVersion;
+  return `/api/${version}/${point}?apikey=${apiKey}`;
+};
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 10; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/chat.js

@@ -0,0 +1,110 @@
+const common = require('../../common');
+const settings = require('../../../../node/utils/Settings');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+const apiKey = common.apiKey;
+let apiVersion = 1;
+let authorID = '';
+const padID = makeid();
+const timestamp = Date.now();
+
+describe(__filename, function () {
+  describe('API Versioning', function () {
+    it('errors if can not connect', function (done) {
+      api.get('/api/')
+          .expect((res) => {
+            apiVersion = res.body.currentVersion;
+            if (!res.body.currentVersion) throw new Error('No version set in API');
+            return;
+          })
+          .expect(200, done);
+    });
+  });
+
+  // BEGIN GROUP AND AUTHOR TESTS
+  // ///////////////////////////////////
+  // ///////////////////////////////////
+
+  /* Tests performed
+  -> createPad(padID)
+   -> createAuthor([name]) -- should return an authorID
+    -> appendChatMessage(padID, text, authorID, time)
+     -> getChatHead(padID)
+      -> getChatHistory(padID)
+  */
+
+  describe('createPad', function () {
+    it('creates a new Pad', function (done) {
+      api.get(`${endPoint('createPad')}&padID=${padID}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to create new Pad');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('createAuthor', function () {
+    it('Creates an author with a name set', function (done) {
+      api.get(endPoint('createAuthor'))
+          .expect((res) => {
+            if (res.body.code !== 0 || !res.body.data.authorID) throw new Error('Unable to create author');
+            authorID = res.body.data.authorID; // we will be this author for the rest of the tests
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('appendChatMessage', function () {
+    it('Adds a chat message to the pad', function (done) {
+      api.get(`${endPoint('appendChatMessage')}&padID=${padID}&text=blalblalbha&authorID=${authorID}&time=${timestamp}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to create chat message');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+
+  describe('getChatHead', function () {
+    it('Gets the head of chat', function (done) {
+      api.get(`${endPoint('getChatHead')}&padID=${padID}`)
+          .expect((res) => {
+            if (res.body.data.chatHead !== 0) throw new Error('Chat Head Length is wrong');
+
+            if (res.body.code !== 0) throw new Error('Unable to get chat head');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getChatHistory', function () {
+    it('Gets Chat History of a Pad', function (done) {
+      api.get(`${endPoint('getChatHistory')}&padID=${padID}`)
+          .expect((res) => {
+            if (res.body.data.messages.length !== 1) throw new Error('Chat History Length is wrong');
+            if (res.body.code !== 0) throw new Error('Unable to get chat history');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+});
+
+var endPoint = function (point) {
+  return `/api/${apiVersion}/${point}?apikey=${apiKey}`;
+};
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/fuzzImportTest.js

@@ -0,0 +1,71 @@
+/*
+ * Fuzz testing the import endpoint
+ */
+/*
+const common = require('../../common');
+const froth = require('mocha-froth');
+const request = require('request');
+const settings = require('../../../container/loadSettings.js').loadSettings();
+
+const host = "http://" + settings.ip + ":" + settings.port;
+
+const apiKey = common.apiKey;
+var apiVersion = 1;
+var testPadId = "TEST_fuzz" + makeid();
+
+var endPoint = function(point, version){
+  version = version || apiVersion;
+  return '/api/'+version+'/'+point+'?apikey='+apiKey;
+}
+
+//console.log("Testing against padID", testPadId);
+//console.log("To watch the test live visit " + host + "/p/" + testPadId);
+//console.log("Tests will start in 5 seconds, click the URL now!");
+
+setTimeout(function(){
+  for (let i=1; i<5; i++) { // 5000 runs
+    setTimeout( function timer(){
+      runTest(i);
+    }, i*100 ); // 100 ms
+  }
+  process.exit(0);
+},5000); // wait 5 seconds
+
+function runTest(number){
+  request(host + endPoint('createPad') + '&padID=' + testPadId, function(err, res, body){
+    var req = request.post(host + '/p/'+testPadId+'/import', function (err, res, body) {
+      if (err) {
+        throw new Error("FAILURE", err);
+      }else{
+        console.log("Success");
+      }
+    });
+
+    var fN = '/tmp/fuzztest.txt';
+    var cT = 'text/plain';
+
+    if (number % 2 == 0) {
+      fN = froth().toString();
+      cT = froth().toString();
+    }
+
+    let form = req.form();
+
+    form.append('file', froth().toString(), {
+      filename: fN,
+      contentType: cT
+    });
+console.log("here");
+  });
+}
+
+function makeid() {
+  var text = "";
+  var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+
+  for( var i=0; i < 5; i++ ){
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}
+*/

src/tests/backend/specs/api/importexport.js

@@ -0,0 +1,319 @@
+'use strict';
+/*
+ * ACHTUNG: there is a copied & modified version of this file in
+ * <basedir>/tests/container/spacs/api/pad.js
+ *
+ * TODO: unify those two files, and merge in a single one.
+ */
+
+const common = require('../../common');
+const settings = require('../../../container/loadSettings.js').loadSettings();
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+const apiKey = common.apiKey;
+const apiVersion = 1;
+
+const testImports = {
+  'malformed': {
+    input: '<html><body><li>wtf</ul></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>wtf<br><br></body></html>',
+    wantText: 'wtf\n\n',
+    disabled: true,
+  },
+  'nonelistiteminlist #3620': {
+    input: '<html><body><ul>test<li>FOO</li></ul></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ul class="bullet">test<li>FOO</ul><br></body></html>',
+    wantText: '\ttest\n\t* FOO\n\n',
+    disabled: true,
+  },
+  'whitespaceinlist #3620': {
+    input: '<html><body><ul> <li>FOO</li></ul></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ul class="bullet"><li>FOO</ul><br></body></html>',
+    wantText: '\t* FOO\n\n',
+  },
+  'prefixcorrectlinenumber': {
+    input: '<html><body><ol><li>should be 1</li><li>should be 2</li></ol></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ol start="1" class="number"><li>should be 1</li><li>should be 2</ol><br></body></html>',
+    wantText: '\t1. should be 1\n\t2. should be 2\n\n',
+  },
+  'prefixcorrectlinenumbernested': {
+    input: '<html><body><ol><li>should be 1</li><ol><li>foo</li></ol><li>should be 2</li></ol></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ol start="1" class="number"><li>should be 1<ol start="2" class="number"><li>foo</ol><li>should be 2</ol><br></body></html>',
+    wantText: '\t1. should be 1\n\t\t1.1. foo\n\t2. should be 2\n\n',
+  },
+
+  /*
+  "prefixcorrectlinenumber when introduced none list item - currently not supported see #3450": {
+    input: '<html><body><ol><li>should be 1</li>test<li>should be 2</li></ol></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ol start="1" class="number"><li>should be 1</li>test<li>should be 2</li></ol><br></body></html>',
+    wantText: '\t1. should be 1\n\ttest\n\t2. should be 2\n\n',
+  }
+  ,
+  "newlinesshouldntresetlinenumber #2194": {
+    input: '<html><body><ol><li>should be 1</li>test<li>should be 2</li></ol></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ol class="number"><li>should be 1</li>test<li>should be 2</li></ol><br></body></html>',
+    wantText: '\t1. should be 1\n\ttest\n\t2. should be 2\n\n',
+  }
+  */
+  'ignoreAnyTagsOutsideBody': {
+    description: 'Content outside body should be ignored',
+    input: '<html><head><title>title</title><style></style></head><body>empty<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>empty<br><br></body></html>',
+    wantText: 'empty\n\n',
+  },
+  'indentedListsAreNotBullets': {
+    description: 'Indented lists are represented with tabs and without bullets',
+    input: '<html><body><ul class="indent"><li>indent</li><li>indent</ul></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><ul class="indent"><li>indent</li><li>indent</ul><br></body></html>',
+    wantText: '\tindent\n\tindent\n\n',
+  },
+  'lineWithMultipleSpaces': {
+    description: 'Multiple spaces should be collapsed',
+    input: '<html><body>Text with  more   than    one space.<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Text with more than one space.<br><br></body></html>',
+    wantText: 'Text with more than one space.\n\n',
+  },
+  'lineWithMultipleNonBreakingAndNormalSpaces': {
+    // XXX the HTML between "than" and "one" looks strange
+    description: 'non-breaking space should be preserved, but can be replaced when it',
+    input: '<html><body>Text&nbsp;with&nbsp; more&nbsp;&nbsp;&nbsp;than   &nbsp;one space.<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Text with&nbsp; more&nbsp;&nbsp; than&nbsp; one space.<br><br></body></html>',
+    wantText: 'Text with  more   than  one space.\n\n',
+  },
+  'multiplenbsp': {
+    description: 'Multiple non-breaking space should be preserved',
+    input: '<html><body>&nbsp;&nbsp;<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>&nbsp;&nbsp;<br><br></body></html>',
+    wantText: '  \n\n',
+  },
+  'multipleNonBreakingSpaceBetweenWords': {
+    description: 'A normal space is always inserted before a word',
+    input: '<html><body>&nbsp;&nbsp;word1&nbsp;&nbsp;word2&nbsp;&nbsp;&nbsp;word3<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>&nbsp; word1&nbsp; word2&nbsp;&nbsp; word3<br><br></body></html>',
+    wantText: '  word1  word2   word3\n\n',
+  },
+  'nonBreakingSpacePreceededBySpaceBetweenWords': {
+    description: 'A non-breaking space preceded by a normal space',
+    input: '<html><body> &nbsp;word1 &nbsp;word2 &nbsp;word3<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>&nbsp;word1&nbsp; word2&nbsp; word3<br><br></body></html>',
+    wantText: ' word1  word2  word3\n\n',
+  },
+  'nonBreakingSpaceFollowededBySpaceBetweenWords': {
+    description: 'A non-breaking space followed by a normal space',
+    input: '<html><body>&nbsp; word1&nbsp; word2&nbsp; word3<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>&nbsp; word1&nbsp; word2&nbsp; word3<br><br></body></html>',
+    wantText: '  word1  word2  word3\n\n',
+  },
+  'spacesAfterNewline': {
+    description: 'Collapse spaces that follow a newline',
+    input: '<!doctype html><html><body>something<br>             something<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>something<br>something<br><br></body></html>',
+    wantText: 'something\nsomething\n\n',
+  },
+  'spacesAfterNewlineP': {
+    description: 'Collapse spaces that follow a paragraph',
+    input: '<!doctype html><html><body>something<p></p>             something<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>something<br><br>something<br><br></body></html>',
+    wantText: 'something\n\nsomething\n\n',
+  },
+  'spacesAtEndOfLine': {
+    description: 'Collapse spaces that preceed/follow a newline',
+    input: '<html><body>something            <br>             something<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>something<br>something<br><br></body></html>',
+    wantText: 'something\nsomething\n\n',
+  },
+  'spacesAtEndOfLineP': {
+    description: 'Collapse spaces that preceed/follow a paragraph',
+    input: '<html><body>something            <p></p>             something<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>something<br><br>something<br><br></body></html>',
+    wantText: 'something\n\nsomething\n\n',
+  },
+  'nonBreakingSpacesAfterNewlines': {
+    description: 'Don\'t collapse non-breaking spaces that follow a newline',
+    input: '<html><body>something<br>&nbsp;&nbsp;&nbsp;something<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>something<br>&nbsp;&nbsp; something<br><br></body></html>',
+    wantText: 'something\n   something\n\n',
+  },
+  'nonBreakingSpacesAfterNewlinesP': {
+    description: 'Don\'t collapse non-breaking spaces that follow a paragraph',
+    input: '<html><body>something<p></p>&nbsp;&nbsp;&nbsp;something<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>something<br><br>&nbsp;&nbsp; something<br><br></body></html>',
+    wantText: 'something\n\n   something\n\n',
+  },
+  'collapseSpacesInsideElements': {
+    description: 'Preserve only one space when multiple are present',
+    input: '<html><body>Need <span> more </span> space<i>  s </i> !<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Need more space<em> s </em>!<br><br></body></html>',
+    wantText: 'Need more space s !\n\n',
+  },
+  'collapseSpacesAcrossNewlines': {
+    description: 'Newlines and multiple spaces across newlines should be collapsed',
+    input: `
+      <html><body>Need
+          <span> more </span>
+          space
+          <i>  s </i>
+          !<br></body></html>`,
+    wantHTML: '<!DOCTYPE HTML><html><body>Need more space <em>s </em>!<br><br></body></html>',
+    wantText: 'Need more space s !\n\n',
+  },
+  'multipleNewLinesAtBeginning': {
+    description: 'Multiple new lines and paragraphs at the beginning should be preserved',
+    input: '<html><body><br><br><p></p><p></p>first line<br><br>second line<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body><br><br><br><br>first line<br><br>second line<br><br></body></html>',
+    wantText: '\n\n\n\nfirst line\n\nsecond line\n\n',
+  },
+  'multiLineParagraph': {
+    description: 'A paragraph with multiple lines should not loose spaces when lines are combined',
+    input: `<html><body>
+    <p>
+      а б в г ґ д е є ж з и і ї й к л м н о
+      п р с т у ф х ц ч ш щ ю я ь
+    </p>
+</body></html>`,
+    wantHTML: '<!DOCTYPE HTML><html><body>&#1072; &#1073; &#1074; &#1075; &#1169; &#1076; &#1077; &#1108; &#1078; &#1079; &#1080; &#1110; &#1111; &#1081; &#1082; &#1083; &#1084; &#1085; &#1086; &#1087; &#1088; &#1089; &#1090; &#1091; &#1092; &#1093; &#1094; &#1095; &#1096; &#1097; &#1102; &#1103; &#1100;<br><br></body></html>',
+    wantText: 'а б в г ґ д е є ж з и і ї й к л м н о п р с т у ф х ц ч ш щ ю я ь\n\n',
+  },
+  'multiLineParagraphWithPre': {
+    // XXX why is there &nbsp; before "in"?
+    description: 'lines in preformatted text should be kept intact',
+    input: `<html><body>
+    <p>
+        а б в г ґ д е є ж з и і ї й к л м н о<pre>multiple
+   lines
+ in
+      pre
+</pre></p><p>п р с т у ф х ц ч ш щ ю я
+ь</p>
+</body></html>`,
+    wantHTML: '<!DOCTYPE HTML><html><body>&#1072; &#1073; &#1074; &#1075; &#1169; &#1076; &#1077; &#1108; &#1078; &#1079; &#1080; &#1110; &#1111; &#1081; &#1082; &#1083; &#1084; &#1085; &#1086;<br>multiple<br>&nbsp;&nbsp; lines<br>&nbsp;in<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; pre<br><br>&#1087; &#1088; &#1089; &#1090; &#1091; &#1092; &#1093; &#1094; &#1095; &#1096; &#1097; &#1102; &#1103; &#1100;<br><br></body></html>',
+    wantText: 'а б в г ґ д е є ж з и і ї й к л м н о\nmultiple\n   lines\n in\n      pre\n\nп р с т у ф х ц ч ш щ ю я ь\n\n',
+  },
+  'preIntroducesASpace': {
+    description: 'pre should be on a new line not preceded by a space',
+    input: `<html><body><p>
+    1
+<pre>preline
+</pre></p></body></html>`,
+    wantHTML: '<!DOCTYPE HTML><html><body>1<br>preline<br><br><br></body></html>',
+    wantText: '1\npreline\n\n\n',
+  },
+  'dontDeleteSpaceInsideElements': {
+    description: 'Preserve spaces inside elements',
+    input: '<html><body>Need<span> more </span>space<i> s </i>!<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Need more space<em> s </em>!<br><br></body></html>',
+    wantText: 'Need more space s !\n\n',
+  },
+  'dontDeleteSpaceOutsideElements': {
+    description: 'Preserve spaces outside elements',
+    input: '<html><body>Need <span>more</span> space <i>s</i> !<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Need more space <em>s</em> !<br><br></body></html>',
+    wantText: 'Need more space s !\n\n',
+  },
+  'dontDeleteSpaceAtEndOfElement': {
+    description: 'Preserve spaces at the end of an element',
+    input: '<html><body>Need <span>more </span>space <i>s </i>!<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Need more space <em>s </em>!<br><br></body></html>',
+    wantText: 'Need more space s !\n\n',
+  },
+  'dontDeleteSpaceAtBeginOfElements': {
+    description: 'Preserve spaces at the start of an element',
+    input: '<html><body>Need<span> more</span> space<i> s</i> !<br></body></html>',
+    wantHTML: '<!DOCTYPE HTML><html><body>Need more space<em> s</em> !<br><br></body></html>',
+    wantText: 'Need more space s !\n\n',
+  },
+};
+
+describe(__filename, function () {
+  Object.keys(testImports).forEach((testName) => {
+    describe(testName, function () {
+      const testPadId = makeid();
+      const test = testImports[testName];
+      if (test.disabled) {
+        return xit(`DISABLED: ${testName}`, function (done) {
+          done();
+        });
+      }
+      it('createPad', function (done) {
+        api.get(`${endPoint('createPad')}&padID=${testPadId}`)
+            .expect((res) => {
+              if (res.body.code !== 0) throw new Error('Unable to create new Pad');
+            })
+            .expect('Content-Type', /json/)
+            .expect(200, done);
+      });
+
+      it('setHTML', function (done) {
+        api.get(`${endPoint('setHTML')}&padID=${testPadId}&html=${encodeURIComponent(test.input)}`)
+            .expect((res) => {
+              if (res.body.code !== 0) throw new Error(`Error:${testName}`);
+            })
+            .expect('Content-Type', /json/)
+            .expect(200, done);
+      });
+
+      it('getHTML', function (done) {
+        api.get(`${endPoint('getHTML')}&padID=${testPadId}`)
+            .expect((res) => {
+              const gotHtml = res.body.data.html;
+              if (gotHtml !== test.wantHTML) {
+                throw new Error(`HTML received from export is not the one we were expecting.
+             Test Name:
+             ${testName}
+
+             Got:
+             ${JSON.stringify(gotHtml)}
+
+             Want:
+             ${JSON.stringify(test.wantHTML)}
+
+             Which is a different version of the originally imported one:
+             ${test.input}`);
+              }
+            })
+            .expect('Content-Type', /json/)
+            .expect(200, done);
+      });
+
+      it('getText', function (done) {
+        api.get(`${endPoint('getText')}&padID=${testPadId}`)
+            .expect((res) => {
+              const gotText = res.body.data.text;
+              if (gotText !== test.wantText) {
+                throw new Error(`Text received from export is not the one we were expecting.
+             Test Name:
+             ${testName}
+
+             Got:
+             ${JSON.stringify(gotText)}
+
+             Want:
+             ${JSON.stringify(test.wantText)}
+
+             Which is a different version of the originally imported one:
+             ${test.input}`);
+              }
+            })
+            .expect('Content-Type', /json/)
+            .expect(200, done);
+      });
+    });
+  });
+});
+
+
+function endPoint(point, version) {
+  version = version || apiVersion;
+  return `/api/${version}/${point}?apikey=${apiKey}`;
+}
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/importexportGetPost.js

@@ -0,0 +1,369 @@
+'use strict';
+
+/*
+ * Import and Export tests for the /p/whateverPadId/import and /p/whateverPadId/export endpoints.
+ */
+
+const assert = require('assert').strict;
+const common = require('../../common');
+const fs = require('fs');
+const settings = require('../../../../node/utils/Settings');
+const superagent = require('superagent');
+const padManager = require('../../../../node/db/PadManager');
+const plugins = require('../../../../static/js/pluginfw/plugin_defs');
+
+const padText = fs.readFileSync('../tests/backend/specs/api/test.txt');
+const etherpadDoc = fs.readFileSync('../tests/backend/specs/api/test.etherpad');
+const wordDoc = fs.readFileSync('../tests/backend/specs/api/test.doc');
+const wordXDoc = fs.readFileSync('../tests/backend/specs/api/test.docx');
+const odtDoc = fs.readFileSync('../tests/backend/specs/api/test.odt');
+const pdfDoc = fs.readFileSync('../tests/backend/specs/api/test.pdf');
+
+let agent;
+const apiKey = common.apiKey;
+const apiVersion = 1;
+const testPadId = makeid();
+const testPadIdEnc = encodeURIComponent(testPadId);
+
+describe(__filename, function () {
+  before(async function () { agent = await common.init(); });
+
+  describe('Connectivity', function () {
+    it('can connect', async function () {
+      await agent.get('/api/')
+          .expect(200)
+          .expect('Content-Type', /json/);
+    });
+  });
+
+  describe('API Versioning', function () {
+    it('finds the version tag', async function () {
+      await agent.get('/api/')
+          .expect(200)
+          .expect((res) => assert(res.body.currentVersion));
+    });
+  });
+
+  /*
+  Tests
+  -----
+
+  Test.
+    / Create a pad
+    / Set pad contents
+    / Try export pad in various formats
+    / Get pad contents and ensure it matches imported contents
+
+  Test.
+    / Try to export a pad that doesn't exist // Expect failure
+
+  Test.
+    / Try to import an unsupported file to a pad that exists
+
+  -- TODO: Test.
+    Try to import to a file and abort it half way through
+
+  Test.
+    Try to import to files of varying size.
+
+  Example Curl command for testing import URI:
+    curl -s -v --form file=@/home/jose/test.txt http://127.0.0.1:9001/p/foo/import
+  */
+
+  describe('Imports and Exports', function () {
+    const backups = {};
+
+    beforeEach(async function () {
+      backups.hooks = {};
+      for (const hookName of ['preAuthorize', 'authenticate', 'authorize']) {
+        backups.hooks[hookName] = plugins.hooks[hookName];
+        plugins.hooks[hookName] = [];
+      }
+      // Note: This is a shallow copy.
+      backups.settings = Object.assign({}, settings);
+      settings.requireAuthentication = false;
+      settings.requireAuthorization = false;
+      settings.users = {user: {password: 'user-password'}};
+    });
+
+    afterEach(async function () {
+      Object.assign(plugins.hooks, backups.hooks);
+      // Note: This does not unset settings that were added.
+      Object.assign(settings, backups.settings);
+    });
+
+    it('creates a new Pad, imports content to it, checks that content', async function () {
+      await agent.get(`${endPoint('createPad')}&padID=${testPadId}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => assert.equal(res.body.code, 0));
+      await agent.post(`/p/${testPadId}/import`)
+          .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+          .expect(200);
+      await agent.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect(200)
+          .expect((res) => assert.equal(res.body.data.text, padText.toString()));
+    });
+
+    it('gets read only pad Id and exports the html and text for this pad', async function () {
+      const ro = await agent.get(`${endPoint('getReadOnlyID')}&padID=${testPadId}`)
+          .expect(200)
+          .expect((res) => assert.ok(JSON.parse(res.text).data.readOnlyID));
+      const readOnlyId = JSON.parse(ro.text).data.readOnlyID;
+
+      await agent.get(`/p/${readOnlyId}/export/html`)
+          .expect(200)
+          .expect((res) => assert(res.text.indexOf('This is the') !== -1));
+
+      await agent.get(`/p/${readOnlyId}/export/txt`)
+          .expect(200)
+          .expect((res) => assert(res.text.indexOf('This is the') !== -1));
+    });
+
+
+    describe('Import/Export tests requiring AbiWord/LibreOffice', function () {
+      before(async function () {
+        if ((!settings.abiword || settings.abiword.indexOf('/') === -1) &&
+            (!settings.soffice || settings.soffice.indexOf('/') === -1)) {
+          this.skip();
+        }
+      });
+
+      // For some reason word import does not work in testing..
+      // TODO: fix support for .doc files..
+      it('Tries to import .doc that uses soffice or abiword', async function () {
+        await agent.post(`/p/${testPadId}/import`)
+            .attach('file', wordDoc, {filename: '/test.doc', contentType: 'application/msword'})
+            .expect(200)
+            .expect(/FrameCall\('undefined', 'ok'\);/);
+      });
+
+      it('exports DOC', async function () {
+        await agent.get(`/p/${testPadId}/export/doc`)
+            .buffer(true).parse(superagent.parse['application/octet-stream'])
+            .expect(200)
+            .expect((res) => assert(res.body.length >= 9000));
+      });
+
+      it('Tries to import .docx that uses soffice or abiword', async function () {
+        await agent.post(`/p/${testPadId}/import`)
+            .attach('file', wordXDoc, {
+              filename: '/test.docx',
+              contentType:
+                  'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            })
+            .expect(200)
+            .expect(/FrameCall\('undefined', 'ok'\);/);
+      });
+
+      it('exports DOC from imported DOCX', async function () {
+        await agent.get(`/p/${testPadId}/export/doc`)
+            .buffer(true).parse(superagent.parse['application/octet-stream'])
+            .expect(200)
+            .expect((res) => assert(res.body.length >= 9100));
+      });
+
+      it('Tries to import .pdf that uses soffice or abiword', async function () {
+        await agent.post(`/p/${testPadId}/import`)
+            .attach('file', pdfDoc, {filename: '/test.pdf', contentType: 'application/pdf'})
+            .expect(200)
+            .expect(/FrameCall\('undefined', 'ok'\);/);
+      });
+
+      it('exports PDF', async function () {
+        await agent.get(`/p/${testPadId}/export/pdf`)
+            .buffer(true).parse(superagent.parse['application/octet-stream'])
+            .expect(200)
+            .expect((res) => assert(res.body.length >= 1000));
+      });
+
+      it('Tries to import .odt that uses soffice or abiword', async function () {
+        await agent.post(`/p/${testPadId}/import`)
+            .attach('file', odtDoc, {filename: '/test.odt', contentType: 'application/odt'})
+            .expect(200)
+            .expect(/FrameCall\('undefined', 'ok'\);/);
+      });
+
+      it('exports ODT', async function () {
+        await agent.get(`/p/${testPadId}/export/odt`)
+            .buffer(true).parse(superagent.parse['application/octet-stream'])
+            .expect(200)
+            .expect((res) => assert(res.body.length >= 7000));
+      });
+    }); // End of AbiWord/LibreOffice tests.
+
+    it('Tries to import .etherpad', async function () {
+      await agent.post(`/p/${testPadId}/import`)
+          .attach('file', etherpadDoc, {
+            filename: '/test.etherpad',
+            contentType: 'application/etherpad',
+          })
+          .expect(200)
+          .expect(/FrameCall\('true', 'ok'\);/);
+    });
+
+    it('exports Etherpad', async function () {
+      await agent.get(`/p/${testPadId}/export/etherpad`)
+          .buffer(true).parse(superagent.parse.text)
+          .expect(200)
+          .expect(/hello/);
+    });
+
+    it('exports HTML for this Etherpad file', async function () {
+      await agent.get(`/p/${testPadId}/export/html`)
+          .expect(200)
+          .expect('content-type', 'text/html; charset=utf-8')
+          .expect(/<ul class="bullet"><li><ul class="bullet"><li>hello<\/ul><\/li><\/ul>/);
+    });
+
+    it('Tries to import unsupported file type', async function () {
+      settings.allowUnknownFileEnds = false;
+      await agent.post(`/p/${testPadId}/import`)
+          .attach('file', padText, {filename: '/test.xasdasdxx', contentType: 'weirdness/jobby'})
+          .expect(200)
+          .expect((res) => assert.doesNotMatch(res.text, /FrameCall\('undefined', 'ok'\);/));
+    });
+
+    describe('Import authorization checks', function () {
+      let authorize;
+
+      const deleteTestPad = async () => {
+        if (await padManager.doesPadExist(testPadId)) {
+          const pad = await padManager.getPad(testPadId);
+          await pad.remove();
+        }
+      };
+
+      const createTestPad = async (text) => {
+        const pad = await padManager.getPad(testPadId);
+        if (text) await pad.setText(text);
+        return pad;
+      };
+
+      beforeEach(async function () {
+        await deleteTestPad();
+        settings.requireAuthorization = true;
+        authorize = () => true;
+        plugins.hooks.authorize = [{hook_fn: (hookName, {req}, cb) => cb([authorize(req)])}];
+      });
+
+      afterEach(async function () {
+        await deleteTestPad();
+      });
+
+      it('!authn !exist -> create', async function () {
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(200);
+        assert(await padManager.doesPadExist(testPadId));
+        const pad = await padManager.getPad(testPadId);
+        assert.equal(pad.text(), padText.toString());
+      });
+
+      it('!authn exist -> replace', async function () {
+        const pad = await createTestPad('before import');
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(200);
+        assert(await padManager.doesPadExist(testPadId));
+        assert.equal(pad.text(), padText.toString());
+      });
+
+      it('authn anonymous !exist -> fail', async function () {
+        settings.requireAuthentication = true;
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(401);
+        assert(!(await padManager.doesPadExist(testPadId)));
+      });
+
+      it('authn anonymous exist -> fail', async function () {
+        settings.requireAuthentication = true;
+        const pad = await createTestPad('before import\n');
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(401);
+        assert.equal(pad.text(), 'before import\n');
+      });
+
+      it('authn user create !exist -> create', async function () {
+        settings.requireAuthentication = true;
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .auth('user', 'user-password')
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(200);
+        assert(await padManager.doesPadExist(testPadId));
+        const pad = await padManager.getPad(testPadId);
+        assert.equal(pad.text(), padText.toString());
+      });
+
+      it('authn user modify !exist -> fail', async function () {
+        settings.requireAuthentication = true;
+        authorize = () => 'modify';
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .auth('user', 'user-password')
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(403);
+        assert(!(await padManager.doesPadExist(testPadId)));
+      });
+
+      it('authn user readonly !exist -> fail', async function () {
+        settings.requireAuthentication = true;
+        authorize = () => 'readOnly';
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .auth('user', 'user-password')
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(403);
+        assert(!(await padManager.doesPadExist(testPadId)));
+      });
+
+      it('authn user create exist -> replace', async function () {
+        settings.requireAuthentication = true;
+        const pad = await createTestPad('before import\n');
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .auth('user', 'user-password')
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(200);
+        assert.equal(pad.text(), padText.toString());
+      });
+
+      it('authn user modify exist -> replace', async function () {
+        settings.requireAuthentication = true;
+        authorize = () => 'modify';
+        const pad = await createTestPad('before import\n');
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .auth('user', 'user-password')
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(200);
+        assert.equal(pad.text(), padText.toString());
+      });
+
+      it('authn user readonly exist -> fail', async function () {
+        const pad = await createTestPad('before import\n');
+        settings.requireAuthentication = true;
+        authorize = () => 'readOnly';
+        await agent.post(`/p/${testPadIdEnc}/import`)
+            .auth('user', 'user-password')
+            .attach('file', padText, {filename: '/test.txt', contentType: 'text/plain'})
+            .expect(403);
+        assert.equal(pad.text(), 'before import\n');
+      });
+    });
+  });
+}); // End of tests.
+
+
+const endPoint = (point, version) => {
+  version = version || apiVersion;
+  return `/api/${version}/${point}?apikey=${apiKey}`;
+};
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/instance.js

@@ -0,0 +1,51 @@
+/*
+ * Tests for the instance-level APIs
+ *
+ * Section "GLOBAL FUNCTIONS" in src/node/db/API.js
+ */
+const common = require('../../common');
+const settings = require('../../../../node/utils/Settings');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+const apiKey = common.apiKey;
+const apiVersion = '1.2.14';
+
+describe(__filename, function () {
+  describe('Connectivity for instance-level API tests', function () {
+    it('can connect', function (done) {
+      api.get('/api/')
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getStats', function () {
+    it('Gets the stats of a running instance', function (done) {
+      api.get(endPoint('getStats'))
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('getStats() failed');
+
+            if (!(('totalPads' in res.body.data) && (typeof res.body.data.totalPads === 'number'))) {
+              throw new Error(`Response to getStats() does not contain field totalPads, or it's not a number: ${JSON.stringify(res.body.data)}`);
+            }
+
+            if (!(('totalSessions' in res.body.data) && (typeof res.body.data.totalSessions === 'number'))) {
+              throw new Error(`Response to getStats() does not contain field totalSessions, or it's not a number: ${JSON.stringify(res.body.data)}`);
+            }
+
+            if (!(('totalActivePads' in res.body.data) && (typeof res.body.data.totalActivePads === 'number'))) {
+              throw new Error(`Response to getStats() does not contain field totalActivePads, or it's not a number: ${JSON.stringify(res.body.data)}`);
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+});
+
+var endPoint = function (point, version) {
+  version = version || apiVersion;
+  return `/api/${version}/${point}?apikey=${apiKey}`;
+};

src/tests/backend/specs/api/pad.js

@@ -0,0 +1,845 @@
+/*
+ * ACHTUNG: there is a copied & modified version of this file in
+ * <basedir>/tests/container/specs/api/pad.js
+ *
+ * TODO: unify those two files, and merge in a single one.
+ */
+
+const async = require('async');
+const common = require('../../common');
+const settings = require('../../../../node/utils/Settings');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+const apiKey = common.apiKey;
+let apiVersion = 1;
+const testPadId = makeid();
+let lastEdited = '';
+const text = generateLongText();
+
+/*
+ * Html document with nested lists of different types, to test its import and
+ * verify it is exported back correctly
+ */
+const ulHtml = '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two</li><li>0</li><li>1</li><li>2<ul class="bullet"><li>3</li><li>4</li></ul></li></ul><ol class="number"><li>item<ol class="number"><li>item1</li><li>item2</li></ol></li></ol></body></html>';
+
+/*
+ * When exported back, Etherpad produces an html which is not exactly the same
+ * textually, but at least it remains standard compliant and has an equal DOM
+ * structure.
+ */
+const expectedHtml = '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two</li><li>0</li><li>1</li><li>2<ul class="bullet"><li>3</li><li>4</ul></li></ul><ol start="1" class="number"><li>item<ol start="2" class="number"><li>item1</li><li>item2</ol></li></ol></body></html>';
+
+/*
+ * Html document with space between list items, to test its import and
+ * verify it is exported back correctly
+ */
+const ulSpaceHtml = '<!doctype html><html><body><ul class="bullet"> <li>one</li></ul></body></html>';
+
+/*
+ * When exported back, Etherpad produces an html which is not exactly the same
+ * textually, but at least it remains standard compliant and has an equal DOM
+ * structure.
+ */
+const expectedSpaceHtml = '<!doctype html><html><body><ul class="bullet"><li>one</ul></body></html>';
+
+describe(__filename, function () {
+  describe('Connectivity', function () {
+    it('can connect', function (done) {
+      api.get('/api/')
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('API Versioning', function () {
+    it('finds the version tag', function (done) {
+      api.get('/api/')
+          .expect((res) => {
+            apiVersion = res.body.currentVersion;
+            if (!res.body.currentVersion) throw new Error('No version set in API');
+            return;
+          })
+          .expect(200, done);
+    });
+  });
+
+  describe('Permission', function () {
+    it('errors with invalid APIKey', function (done) {
+      // This is broken because Etherpad doesn't handle HTTP codes properly see #2343
+      // If your APIKey is password you deserve to fail all tests anyway
+      const permErrorURL = `/api/${apiVersion}/createPad?apikey=password&padID=test`;
+      api.get(permErrorURL)
+          .expect(401, done);
+    });
+  });
+
+  /* Pad Tests Order of execution
+  -> deletePad -- This gives us a guaranteed clear environment
+   -> createPad
+    -> getRevisions -- Should be 0
+     -> getSavedRevisionsCount(padID) -- Should be 0
+      -> listSavedRevisions(padID) -- Should be an empty array
+       -> getHTML -- Should be the default pad text in HTML format
+        -> deletePad -- Should just delete a pad
+         -> getHTML -- Should return an error
+          -> createPad(withText)
+           -> getText -- Should have the text specified above as the pad text
+            -> setText
+             -> getText -- Should be the text set before
+              -> getRevisions -- Should be 0 still?
+               -> saveRevision
+                -> getSavedRevisionsCount(padID) -- Should be 0 still?
+                 -> listSavedRevisions(padID) -- Should be an empty array still ?
+                  -> padUsersCount -- Should be 0
+                   -> getReadOnlyId -- Should be a value
+                    -> listAuthorsOfPad(padID) -- should be empty array?
+                     -> getLastEdited(padID) -- Should be when pad was made
+                      -> setText(padId)
+                       -> getLastEdited(padID) -- Should be when setText was performed
+                        -> padUsers(padID) -- Should be when setText was performed
+
+                         -> setText(padId, "hello world")
+                          -> getLastEdited(padID) -- Should be when pad was made
+                           -> getText(padId) -- Should be "hello world"
+                            -> movePad(padID, newPadId) -- Should provide consistent pad data
+                             -> getText(newPadId) -- Should be "hello world"
+                              -> movePad(newPadID, originalPadId) -- Should provide consistent pad data
+                               -> getText(originalPadId) -- Should be "hello world"
+                                -> getLastEdited(padID) -- Should not be 0
+                                -> appendText(padID, "hello")
+                                -> getText(padID) -- Should be "hello worldhello"
+                                 -> setHTML(padID) -- Should fail on invalid HTML
+                                  -> setHTML(padID) *3 -- Should fail on invalid HTML
+                                   -> getHTML(padID) -- Should return HTML close to posted HTML
+                                    -> createPad -- Tries to create pads with bad url characters
+
+  */
+
+  describe('deletePad', function () {
+    it('deletes a Pad', function (done) {
+      api.get(`${endPoint('deletePad')}&padID=${testPadId}`)
+          .expect('Content-Type', /json/)
+          .expect(200, done); // @TODO: we shouldn't expect 200 here since the pad may not exist
+    });
+  });
+
+  describe('createPad', function () {
+    it('creates a new Pad', function (done) {
+      api.get(`${endPoint('createPad')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to create new Pad');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getRevisionsCount', function () {
+    it('gets revision count of Pad', function (done) {
+      api.get(`${endPoint('getRevisionsCount')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to get Revision Count');
+            if (res.body.data.revisions !== 0) throw new Error('Incorrect Revision Count');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getSavedRevisionsCount', function () {
+    it('gets saved revisions count of Pad', function (done) {
+      api.get(`${endPoint('getSavedRevisionsCount')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to get Saved Revisions Count');
+            if (res.body.data.savedRevisions !== 0) throw new Error('Incorrect Saved Revisions Count');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('listSavedRevisions', function () {
+    it('gets saved revision list of Pad', function (done) {
+      api.get(`${endPoint('listSavedRevisions')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to get Saved Revisions List');
+            if (!res.body.data.savedRevisions.equals([])) throw new Error('Incorrect Saved Revisions List');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getHTML', function () {
+    it('get the HTML of Pad', function (done) {
+      api.get(`${endPoint('getHTML')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.html.length <= 1) throw new Error('Unable to get the HTML');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('listAllPads', function () {
+    it('list all pads', function (done) {
+      api.get(endPoint('listAllPads'))
+          .expect((res) => {
+            if (res.body.data.padIDs.includes(testPadId) !== true) {
+              throw new Error('Unable to find pad in pad list');
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('deletePad', function () {
+    it('deletes a Pad', function (done) {
+      api.get(`${endPoint('deletePad')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Deletion failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('listAllPads', function () {
+    it('list all pads', function (done) {
+      api.get(endPoint('listAllPads'))
+          .expect((res) => {
+            if (res.body.data.padIDs.includes(testPadId) !== false) {
+              throw new Error('Test pad should not be in pads list');
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getHTML', function () {
+    it('get the HTML of a Pad -- Should return a failure', function (done) {
+      api.get(`${endPoint('getHTML')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 1) throw new Error('Pad deletion failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('createPad', function () {
+    it('creates a new Pad with text', function (done) {
+      api.get(`${endPoint('createPad')}&padID=${testPadId}&text=testText`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Creation failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it('gets the Pad text and expect it to be testText with \n which is a line break', function (done) {
+      api.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.text !== 'testText\n') throw new Error('Pad Creation with text');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setText', function () {
+    it('creates a new Pad with text', function (done) {
+      api.post(endPoint('setText'))
+          .send({
+            padID: testPadId,
+            text: 'testTextTwo',
+          })
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad setting text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it('gets the Pad text', function (done) {
+      api.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.text !== 'testTextTwo\n') throw new Error('Setting Text');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getRevisionsCount', function () {
+    it('gets Revision Count of a Pad', function (done) {
+      api.get(`${endPoint('getRevisionsCount')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.revisions !== 1) throw new Error('Unable to get text revision count');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('saveRevision', function () {
+    it('saves Revision', function (done) {
+      api.get(`${endPoint('saveRevision')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to save Revision');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getSavedRevisionsCount', function () {
+    it('gets saved revisions count of Pad', function (done) {
+      api.get(`${endPoint('getSavedRevisionsCount')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to get Saved Revisions Count');
+            if (res.body.data.savedRevisions !== 1) throw new Error('Incorrect Saved Revisions Count');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('listSavedRevisions', function () {
+    it('gets saved revision list of Pad', function (done) {
+      api.get(`${endPoint('listSavedRevisions')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Unable to get Saved Revisions List');
+            if (!res.body.data.savedRevisions.equals([1])) throw new Error('Incorrect Saved Revisions List');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+  describe('padUsersCount', function () {
+    it('gets User Count of a Pad', function (done) {
+      api.get(`${endPoint('padUsersCount')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.padUsersCount !== 0) throw new Error('Incorrect Pad User count');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getReadOnlyID', function () {
+    it('Gets the Read Only ID of a Pad', function (done) {
+      api.get(`${endPoint('getReadOnlyID')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (!res.body.data.readOnlyID) throw new Error('No Read Only ID for Pad');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('listAuthorsOfPad', function () {
+    it('Get Authors of the Pad', function (done) {
+      api.get(`${endPoint('listAuthorsOfPad')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.authorIDs.length !== 0) throw new Error('# of Authors of pad is not 0');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getLastEdited', function () {
+    it('Get When Pad was left Edited', function (done) {
+      api.get(`${endPoint('getLastEdited')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (!res.body.data.lastEdited) {
+              throw new Error('# of Authors of pad is not 0');
+            } else {
+              lastEdited = res.body.data.lastEdited;
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setText', function () {
+    it('creates a new Pad with text', function (done) {
+      api.post(endPoint('setText'))
+          .send({
+            padID: testPadId,
+            text: 'testTextTwo',
+          })
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad setting text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getLastEdited', function () {
+    it('Get When Pad was left Edited', function (done) {
+      api.get(`${endPoint('getLastEdited')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.lastEdited <= lastEdited) {
+              throw new Error('Editing A Pad is not updating when it was last edited');
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('padUsers', function () {
+    it('gets User Count of a Pad', function (done) {
+      api.get(`${endPoint('padUsers')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.padUsers.length !== 0) throw new Error('Incorrect Pad Users');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('deletePad', function () {
+    it('deletes a Pad', function (done) {
+      api.get(`${endPoint('deletePad')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Deletion failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  const originalPadId = testPadId;
+  const newPadId = makeid();
+  const copiedPadId = makeid();
+
+  describe('createPad', function () {
+    it('creates a new Pad with text', function (done) {
+      api.get(`${endPoint('createPad')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Creation failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setText', function () {
+    it('Sets text on a pad Id', function (done) {
+      api.post(`${endPoint('setText')}&padID=${testPadId}`)
+          .field({text})
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Set Text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it('Gets text on a pad Id', function (done) {
+      api.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Get Text failed');
+            if (res.body.data.text !== `${text}\n`) throw new Error('Pad Text not set properly');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setText', function () {
+    it('Sets text on a pad Id including an explicit newline', function (done) {
+      api.post(`${endPoint('setText')}&padID=${testPadId}`)
+          .field({text: `${text}\n`})
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Set Text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it("Gets text on a pad Id and doesn't have an excess newline", function (done) {
+      api.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Get Text failed');
+            if (res.body.data.text !== `${text}\n`) throw new Error('Pad Text not set properly');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getLastEdited', function () {
+    it('Gets when pad was last edited', function (done) {
+      api.get(`${endPoint('getLastEdited')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.lastEdited === 0) throw new Error('Get Last Edited Failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('movePad', function () {
+    it('Move a Pad to a different Pad ID', function (done) {
+      api.get(`${endPoint('movePad')}&sourceID=${testPadId}&destinationID=${newPadId}&force=true`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Moving Pad Failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it('Gets text on a pad Id', function (done) {
+      api.get(`${endPoint('getText')}&padID=${newPadId}`)
+          .expect((res) => {
+            if (res.body.data.text !== `${text}\n`) throw new Error('Pad Get Text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('movePad', function () {
+    it('Move a Pad to a different Pad ID', function (done) {
+      api.get(`${endPoint('movePad')}&sourceID=${newPadId}&destinationID=${testPadId}&force=false`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Moving Pad Failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it('Gets text on a pad Id', function (done) {
+      api.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.data.text !== `${text}\n`) throw new Error('Pad Get Text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getLastEdited', function () {
+    it('Gets when pad was last edited', function (done) {
+      api.get(`${endPoint('getLastEdited')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.lastEdited === 0) throw new Error('Get Last Edited Failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('appendText', function () {
+    it('Append text to a pad Id', function (done) {
+      api.get(`${endPoint('appendText', '1.2.13')}&padID=${testPadId}&text=hello`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Append Text failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getText', function () {
+    it('Gets text on a pad Id', function (done) {
+      api.get(`${endPoint('getText')}&padID=${testPadId}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Pad Get Text failed');
+            if (res.body.data.text !== `${text}hello\n`) throw new Error('Pad Text not set properly');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+
+  describe('setHTML', function () {
+    it('Sets the HTML of a Pad attempting to pass ugly HTML', function (done) {
+      const html = '<div><b>Hello HTML</title></head></div>';
+      api.post(endPoint('setHTML'))
+          .send({
+            padID: testPadId,
+            html,
+          })
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error("Crappy HTML Can't be Imported[we weren't able to sanitize it']");
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setHTML', function () {
+    it('Sets the HTML of a Pad with complex nested lists of different types', function (done) {
+      api.post(endPoint('setHTML'))
+          .send({
+            padID: testPadId,
+            html: ulHtml,
+          })
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('List HTML cant be imported');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getHTML', function () {
+    it('Gets back the HTML of a Pad with complex nested lists of different types', function (done) {
+      api.get(`${endPoint('getHTML')}&padID=${testPadId}`)
+          .expect((res) => {
+            const receivedHtml = res.body.data.html.replace('<br></body>', '</body>').toLowerCase();
+
+            if (receivedHtml !== expectedHtml) {
+              throw new Error(`HTML received from export is not the one we were expecting.
+           Received:
+           ${receivedHtml}
+
+           Expected:
+           ${expectedHtml}
+
+           Which is a slightly modified version of the originally imported one:
+           ${ulHtml}`);
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('setHTML', function () {
+    it('Sets the HTML of a Pad with white space between list items', function (done) {
+      api.get(`${endPoint('setHTML')}&padID=${testPadId}&html=${ulSpaceHtml}`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('List HTML cant be imported');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('getHTML', function () {
+    it('Gets back the HTML of a Pad with complex nested lists of different types', function (done) {
+      api.get(`${endPoint('getHTML')}&padID=${testPadId}`)
+          .expect((res) => {
+            const receivedHtml = res.body.data.html.replace('<br></body>', '</body>').toLowerCase();
+            if (receivedHtml !== expectedSpaceHtml) {
+              throw new Error(`HTML received from export is not the one we were expecting.
+           Received:
+           ${receivedHtml}
+
+           Expected:
+           ${expectedSpaceHtml}
+
+           Which is a slightly modified version of the originally imported one:
+           ${ulSpaceHtml}`);
+            }
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('createPad', function () {
+    it('errors if pad can be created', function (done) {
+      const badUrlChars = ['/', '%23', '%3F', '%26'];
+      async.map(
+          badUrlChars,
+          (badUrlChar, cb) => {
+            api.get(`${endPoint('createPad')}&padID=${badUrlChar}`)
+                .expect((res) => {
+                  if (res.body.code !== 1) throw new Error('Pad with bad characters was created');
+                })
+                .expect('Content-Type', /json/)
+                .end(cb);
+          },
+          done);
+    });
+  });
+
+  describe('copyPad', function () {
+    it('copies the content of a existent pad', function (done) {
+      api.get(`${endPoint('copyPad')}&sourceID=${testPadId}&destinationID=${copiedPadId}&force=true`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Copy Pad Failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+  });
+
+  describe('copyPadWithoutHistory', function () {
+    const sourcePadId = makeid();
+    let newPad;
+
+    before(function (done) {
+      createNewPadWithHtml(sourcePadId, ulHtml, done);
+    });
+
+    beforeEach(function () {
+      newPad = makeid();
+    });
+
+    it('returns a successful response', function (done) {
+      api.get(`${endPoint('copyPadWithoutHistory')}&sourceID=${sourcePadId}&destinationID=${newPad}&force=false`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Copy Pad Without History Failed');
+          })
+          .expect('Content-Type', /json/)
+          .expect(200, done);
+    });
+
+    // this test validates if the source pad's text and attributes are kept
+    it('creates a new pad with the same content as the source pad', function (done) {
+      api.get(`${endPoint('copyPadWithoutHistory')}&sourceID=${sourcePadId}&destinationID=${newPad}&force=false`)
+          .expect((res) => {
+            if (res.body.code !== 0) throw new Error('Copy Pad Without History Failed');
+          })
+          .end(() => {
+            api.get(`${endPoint('getHTML')}&padID=${newPad}`)
+                .expect((res) => {
+                  const receivedHtml = res.body.data.html.replace('<br><br></body>', '</body>').toLowerCase();
+
+                  if (receivedHtml !== expectedHtml) {
+                    throw new Error(`HTML received from export is not the one we were expecting.
+                 Received:
+                 ${receivedHtml}
+
+                 Expected:
+                 ${expectedHtml}
+
+                 Which is a slightly modified version of the originally imported one:
+                 ${ulHtml}`);
+                  }
+                })
+                .expect(200, done);
+          });
+    });
+
+    context('when try copy a pad with a group that does not exist', function () {
+      const padId = makeid();
+      const padWithNonExistentGroup = `notExistentGroup$${padId}`;
+      it('throws an error', function (done) {
+        api.get(`${endPoint('copyPadWithoutHistory')}&sourceID=${sourcePadId}&destinationID=${padWithNonExistentGroup}&force=true`)
+            .expect((res) => {
+              // code 1, it means an error has happened
+              if (res.body.code !== 1) throw new Error('It should report an error');
+            })
+            .expect(200, done);
+      });
+    });
+
+    context('when try copy a pad and destination pad already exist', function () {
+      const padIdExistent = makeid();
+
+      before(function (done) {
+        createNewPadWithHtml(padIdExistent, ulHtml, done);
+      });
+
+      context('and force is false', function () {
+        it('throws an error', function (done) {
+          api.get(`${endPoint('copyPadWithoutHistory')}&sourceID=${sourcePadId}&destinationID=${padIdExistent}&force=false`)
+              .expect((res) => {
+                // code 1, it means an error has happened
+                if (res.body.code !== 1) throw new Error('It should report an error');
+              })
+              .expect(200, done);
+        });
+      });
+
+      context('and force is true', function () {
+        it('returns a successful response', function (done) {
+          api.get(`${endPoint('copyPadWithoutHistory')}&sourceID=${sourcePadId}&destinationID=${padIdExistent}&force=true`)
+              .expect((res) => {
+                // code 1, it means an error has happened
+                if (res.body.code !== 0) throw new Error('Copy pad without history with force true failed');
+              })
+              .expect(200, done);
+        });
+      });
+    });
+  });
+});
+
+/*
+                          -> movePadForce Test
+
+*/
+
+var createNewPadWithHtml = function (padId, html, cb) {
+  api.get(`${endPoint('createPad')}&padID=${padId}`)
+      .end(() => {
+        api.post(endPoint('setHTML'))
+            .send({
+              padID: padId,
+              html,
+            })
+            .end(cb);
+      });
+};
+
+var endPoint = function (point, version) {
+  version = version || apiVersion;
+  return `/api/${version}/${point}?apikey=${apiKey}`;
+};
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}
+
+function generateLongText() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 80000; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}
+
+// Need this to compare arrays (listSavedRevisions test)
+Array.prototype.equals = function (array) {
+  // if the other array is a falsy value, return
+  if (!array) return false;
+  // compare lengths - can save a lot of time
+  if (this.length != array.length) return false;
+  for (let i = 0, l = this.length; i < l; i++) {
+    // Check if we have nested arrays
+    if (this[i] instanceof Array && array[i] instanceof Array) {
+      // recurse into the nested arrays
+      if (!this[i].equals(array[i])) return false;
+    } else if (this[i] != array[i]) {
+      // Warning - two different object instances will never be equal: {x:20} != {x:20}
+      return false;
+    }
+  }
+  return true;
+};

src/tests/backend/specs/api/sessionsAndGroups.js

@@ -0,0 +1,377 @@
+const assert = require('assert').strict;
+const common = require('../../common');
+const settings = require('../../../../node/utils/Settings');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+const apiKey = common.apiKey;
+let apiVersion = 1;
+let groupID = '';
+let authorID = '';
+let sessionID = '';
+let padID = makeid();
+
+describe(__filename, function () {
+  describe('API Versioning', function () {
+    it('errors if can not connect', async function () {
+      await api.get('/api/')
+          .expect(200)
+          .expect((res) => {
+            assert(res.body.currentVersion);
+            apiVersion = res.body.currentVersion;
+          });
+    });
+  });
+
+  // BEGIN GROUP AND AUTHOR TESTS
+  // ///////////////////////////////////
+  // ///////////////////////////////////
+
+  /* Tests performed
+  -> createGroup() -- should return a groupID
+   -> listSessionsOfGroup(groupID) -- should be 0
+    -> deleteGroup(groupID)
+     -> createGroupIfNotExistsFor(groupMapper) -- should return a groupID
+
+      -> createAuthor([name]) -- should return an authorID
+       -> createAuthorIfNotExistsFor(authorMapper [, name]) -- should return an authorID
+        -> getAuthorName(authorID) -- should return a name IE "john"
+
+  -> createSession(groupID, authorID, validUntil)
+   -> getSessionInfo(sessionID)
+    -> listSessionsOfGroup(groupID) -- should be 1
+     -> deleteSession(sessionID)
+      -> getSessionInfo(sessionID) -- should have author id etc in
+
+  -> listPads(groupID) -- should be empty array
+   -> createGroupPad(groupID, padName [, text])
+    -> listPads(groupID) -- should be empty array
+     -> getPublicStatus(padId)
+      -> setPublicStatus(padId, status)
+       -> getPublicStatus(padId)
+
+  -> listPadsOfAuthor(authorID)
+  */
+
+  describe('API: Group creation and deletion', function () {
+    it('createGroup', async function () {
+      await api.get(endPoint('createGroup'))
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.groupID);
+            groupID = res.body.data.groupID;
+          });
+    });
+
+    it('listSessionsOfGroup for empty group', async function () {
+      await api.get(`${endPoint('listSessionsOfGroup')}&groupID=${groupID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data, null);
+          });
+    });
+
+    it('deleteGroup', async function () {
+      await api.get(`${endPoint('deleteGroup')}&groupID=${groupID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+          });
+    });
+
+    it('createGroupIfNotExistsFor', async function () {
+      await api.get(`${endPoint('createGroupIfNotExistsFor')}&groupMapper=management`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.groupID);
+          });
+    });
+
+    // Test coverage for https://github.com/ether/etherpad-lite/issues/4227
+    // Creates a group, creates 2 sessions, 2 pads and then deletes the group.
+    it('createGroup', async function () {
+      await api.get(endPoint('createGroup'))
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.groupID);
+            groupID = res.body.data.groupID;
+          });
+    });
+
+    it('createAuthor', async function () {
+      await api.get(endPoint('createAuthor'))
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.authorID);
+            authorID = res.body.data.authorID;
+          });
+    });
+
+    it('createSession', async function () {
+      await api.get(`${endPoint('createSession')
+      }&authorID=${authorID}&groupID=${groupID}&validUntil=999999999999`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.sessionID);
+            sessionID = res.body.data.sessionID;
+          });
+    });
+
+    it('createSession', async function () {
+      await api.get(`${endPoint('createSession')
+      }&authorID=${authorID}&groupID=${groupID}&validUntil=999999999999`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.sessionID);
+            sessionID = res.body.data.sessionID;
+          });
+    });
+
+    it('createGroupPad', async function () {
+      await api.get(`${endPoint('createGroupPad')}&groupID=${groupID}&padName=x1234567`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+          });
+    });
+
+    it('createGroupPad', async function () {
+      await api.get(`${endPoint('createGroupPad')}&groupID=${groupID}&padName=x12345678`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+          });
+    });
+
+    it('deleteGroup', async function () {
+      await api.get(`${endPoint('deleteGroup')}&groupID=${groupID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+          });
+    });
+    // End of coverage for https://github.com/ether/etherpad-lite/issues/4227
+
+  });
+
+  describe('API: Author creation', function () {
+    it('createGroup', async function () {
+      await api.get(endPoint('createGroup'))
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.groupID);
+            groupID = res.body.data.groupID;
+          });
+    });
+
+    it('createAuthor', async function () {
+      await api.get(endPoint('createAuthor'))
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.authorID);
+          });
+    });
+
+    it('createAuthor with name', async function () {
+      await api.get(`${endPoint('createAuthor')}&name=john`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.authorID);
+            authorID = res.body.data.authorID; // we will be this author for the rest of the tests
+          });
+    });
+
+    it('createAuthorIfNotExistsFor', async function () {
+      await api.get(`${endPoint('createAuthorIfNotExistsFor')}&authorMapper=chris`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.authorID);
+          });
+    });
+
+    it('getAuthorName', async function () {
+      await api.get(`${endPoint('getAuthorName')}&authorID=${authorID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data, 'john');
+          });
+    });
+  });
+
+  describe('API: Sessions', function () {
+    it('createSession', async function () {
+      await api.get(`${endPoint('createSession')
+      }&authorID=${authorID}&groupID=${groupID}&validUntil=999999999999`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.sessionID);
+            sessionID = res.body.data.sessionID;
+          });
+    });
+
+    it('getSessionInfo', async function () {
+      await api.get(`${endPoint('getSessionInfo')}&sessionID=${sessionID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert(res.body.data.groupID);
+            assert(res.body.data.authorID);
+            assert(res.body.data.validUntil);
+          });
+    });
+
+    it('listSessionsOfGroup', async function () {
+      await api.get(`${endPoint('listSessionsOfGroup')}&groupID=${groupID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(typeof res.body.data, 'object');
+          });
+    });
+
+    it('deleteSession', async function () {
+      await api.get(`${endPoint('deleteSession')}&sessionID=${sessionID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+          });
+    });
+
+    it('getSessionInfo of deleted session', async function () {
+      await api.get(`${endPoint('getSessionInfo')}&sessionID=${sessionID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 1);
+          });
+    });
+  });
+
+  describe('API: Group pad management', function () {
+    it('listPads', async function () {
+      await api.get(`${endPoint('listPads')}&groupID=${groupID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data.padIDs.length, 0);
+          });
+    });
+
+    it('createGroupPad', async function () {
+      await api.get(`${endPoint('createGroupPad')}&groupID=${groupID}&padName=${padID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            padID = res.body.data.padID;
+          });
+    });
+
+    it('listPads after creating a group pad', async function () {
+      await api.get(`${endPoint('listPads')}&groupID=${groupID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data.padIDs.length, 1);
+          });
+    });
+  });
+
+  describe('API: Pad security', function () {
+    it('getPublicStatus', async function () {
+      await api.get(`${endPoint('getPublicStatus')}&padID=${padID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data.publicStatus, false);
+          });
+    });
+
+    it('setPublicStatus', async function () {
+      await api.get(`${endPoint('setPublicStatus')}&padID=${padID}&publicStatus=true`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+          });
+    });
+
+    it('getPublicStatus after changing public status', async function () {
+      await api.get(`${endPoint('getPublicStatus')}&padID=${padID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data.publicStatus, true);
+          });
+    });
+  });
+
+  // NOT SURE HOW TO POPULAT THIS /-_-\
+  // /////////////////////////////////////
+  // /////////////////////////////////////
+
+  describe('API: Misc', function () {
+    it('listPadsOfAuthor', async function () {
+      await api.get(`${endPoint('listPadsOfAuthor')}&authorID=${authorID}`)
+          .expect(200)
+          .expect('Content-Type', /json/)
+          .expect((res) => {
+            assert.equal(res.body.code, 0);
+            assert.equal(res.body.data.padIDs.length, 0);
+          });
+    });
+  });
+});
+
+
+const endPoint = function (point) {
+  return `/api/${apiVersion}/${point}?apikey=${apiKey}`;
+};
+
+function makeid() {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length));
+  }
+  return text;
+}

src/tests/backend/specs/api/test.etherpad

@@ -0,0 +1 @@
+{"pad:Pd4b1Kgvv9qHZZtj8yzl":{"atext":{"text":"*hello\n","attribs":"*0*1*5*3*4+1*0+5|1+1"},"pool":{"numToAttrib":{"0":["author","a.ElbBWNTxmtRrfFqn"],"1":["insertorder","first"],"2":["list","bullet1"],"3":["lmkr","1"],"4":["start","1"],"5":["list","bullet2"]},"nextNum":6},"head":5,"chatHead":-1,"publicStatus":false,"passwordHash":null,"savedRevisions":[]},"globalAuthor:a.ElbBWNTxmtRrfFqn":{"colorId":50,"name":null,"timestamp":1595111151414,"padIDs":"Pd4b1Kgvv9qHZZtj8yzl"},"pad:Pd4b1Kgvv9qHZZtj8yzl:revs:0":{"changeset":"Z:1>bj|7+bj$Welcome to Etherpad!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nGet involved with Etherpad at https://etherpad.org\n\nWarning: DirtyDB is used. This is fine for testing but not recommended for production. -- To suppress these warning messages change suppressErrorsInPadText to true in your settings.json\n","meta":{"author":"","timestamp":1595111092400,"pool":{"numToAttrib":{},"attribToNum":{},"nextNum":0},"atext":{"text":"Welcome to Etherpad!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nGet involved with Etherpad at https://etherpad.org\n\nWarning: DirtyDB is used. This is fine for testing but not recommended for production. -- To suppress these warning messages change suppressErrorsInPadText to true in your settings.json\n\n","attribs":"|8+bk"}}},"pad:Pd4b1Kgvv9qHZZtj8yzl:revs:1":{"changeset":"Z:bk<bj|7-bj$","meta":{"author":"a.ElbBWNTxmtRrfFqn","timestamp":1595111112138}},"pad:Pd4b1Kgvv9qHZZtj8yzl:revs:2":{"changeset":"Z:1>1*0*1*2*3*4+1$*","meta":{"author":"a.ElbBWNTxmtRrfFqn","timestamp":1595111119434}},"pad:Pd4b1Kgvv9qHZZtj8yzl:revs:3":{"changeset":"Z:2>0*5*4=1$","meta":{"author":"a.ElbBWNTxmtRrfFqn","timestamp":1595111127471}},"pad:Pd4b1Kgvv9qHZZtj8yzl:revs:4":{"changeset":"Z:2>2=1*0+2$he","meta":{"author":"a.ElbBWNTxmtRrfFqn","timestamp":1595111128230}},"pad:Pd4b1Kgvv9qHZZtj8yzl:revs:5":{"changeset":"Z:4>3=3*0+3$llo","meta":{"author":"a.ElbBWNTxmtRrfFqn","timestamp":1595111128727}}}

src/tests/backend/specs/api/test.txt

@@ -0,0 +1 @@
+This is the contents we insert into pads when testing the import functions.  Thanks for using Etherpad!

src/tests/backend/specs/api/tidy.js

@@ -0,0 +1,71 @@
+'use strict';
+
+const assert = require('assert');
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+let TidyHtml;
+let Settings;
+const npm = require('npm/lib/npm.js');
+const nodeify = require('nodeify');
+
+describe(__filename, function () {
+  describe('tidyHtml', function () {
+    before(function (done) {
+      npm.load({}, (err) => {
+        assert.ok(!err);
+        TidyHtml = require('../../../../node/utils/TidyHtml');
+        Settings = require('../../../../node/utils/Settings');
+        return done();
+      });
+    });
+
+    const tidy = (file, callback) => nodeify(TidyHtml.tidy(file), callback);
+
+    it('Tidies HTML', function (done) {
+      // If the user hasn't configured Tidy, we skip this tests as it's required for this test
+      if (!Settings.tidyHtml) {
+        this.skip();
+      }
+
+      // Try to tidy up a bad HTML file
+      const tmpDir = os.tmpdir();
+
+      const tmpFile = path.join(tmpDir, `tmp_${Math.floor(Math.random() * 1000000)}.html`);
+      fs.writeFileSync(tmpFile, '<html><body><p>a paragraph</p><li>List without outer UL</li>trailing closing p</p></body></html>');
+      tidy(tmpFile, (err) => {
+        assert.ok(!err);
+
+        // Read the file again
+        const cleanedHtml = fs.readFileSync(tmpFile).toString();
+
+        const expectedHtml = [
+          '<title></title>',
+          '</head>',
+          '<body>',
+          '<p>a paragraph</p>',
+          '<ul>',
+          '<li>List without outer UL</li>',
+          '<li style="list-style: none">trailing closing p</li>',
+          '</ul>',
+          '</body>',
+          '</html>',
+        ].join('\n');
+        assert.notStrictEqual(cleanedHtml.indexOf(expectedHtml), -1);
+        done();
+      });
+    });
+
+    it('can deal with errors', function (done) {
+      // If the user hasn't configured Tidy, we skip this tests as it's required for this test
+      if (!Settings.tidyHtml) {
+        this.skip();
+      }
+
+      tidy('/some/none/existing/file.html', (err) => {
+        assert.ok(err);
+        done();
+      });
+    });
+  });
+});

src/tests/backend/specs/caching_middleware.js

@@ -0,0 +1,147 @@
+/**
+ * caching_middleware is responsible for serving everything under path `/javascripts/`
+ * That includes packages as defined in `src/node/utils/tar.json` and probably also plugin code
+ *
+ */
+
+const common = require('../common');
+const assert = require('assert').strict;
+const url = require('url');
+const queryString = require('querystring');
+const settings = require('../../../node/utils/Settings');
+
+let agent;
+
+/**
+ * Hack! Returns true if the resource is not plaintext
+ * The file should start with the callback method, so we need the
+ * URL.
+ *
+ * @param {string} fileContent the response body
+ * @param {URI} resource resource URI
+ * @returns {boolean} if it is plaintext
+ */
+function isPlaintextResponse(fileContent, resource) {
+  // callback=require.define&v=1234
+  const query = url.parse(resource).query;
+  // require.define
+  const jsonp = queryString.parse(query).callback;
+
+  // returns true if the first letters in fileContent equal the content of `jsonp`
+  return fileContent.substring(0, jsonp.length) === jsonp;
+}
+
+/**
+ * A hack to disable `superagent`'s auto unzip functionality
+ *
+ * @param {Request} request
+ */
+function disableAutoDeflate(request) {
+  request._shouldUnzip = function () {
+    return false;
+  };
+}
+
+describe(__filename, function () {
+  const backups = {};
+  const fantasyEncoding = 'brainwaves'; // non-working encoding until https://github.com/visionmedia/superagent/pull/1560 is resolved
+  const packages = [
+    '/javascripts/lib/ep_etherpad-lite/static/js/ace2_common.js?callback=require.define',
+    '/javascripts/lib/ep_etherpad-lite/static/js/ace2_inner.js?callback=require.define',
+    '/javascripts/lib/ep_etherpad-lite/static/js/pad.js?callback=require.define',
+    '/javascripts/lib/ep_etherpad-lite/static/js/timeslider.js?callback=require.define',
+  ];
+
+  before(async function () {
+    agent = await common.init();
+  });
+  beforeEach(async function () {
+    backups.settings = {};
+    backups.settings.minify = settings.minify;
+  });
+  afterEach(async function () {
+    Object.assign(settings, backups.settings);
+  });
+
+  context('when minify is false', function () {
+    before(async function () {
+      settings.minify = false;
+    });
+    it('gets packages uncompressed without Accept-Encoding gzip', async function () {
+      await Promise.all(packages.map(async (resource) => agent.get(resource)
+          .set('Accept-Encoding', fantasyEncoding)
+          .use(disableAutoDeflate)
+          .then((res) => {
+            assert.match(res.header['content-type'], /application\/javascript/);
+            assert.equal(res.header['content-encoding'], undefined);
+            assert.equal(isPlaintextResponse(res.text, resource), true);
+            return;
+          })));
+    });
+
+    it('gets packages compressed with Accept-Encoding gzip', async function () {
+      await Promise.all(packages.map(async (resource) => agent.get(resource)
+          .set('Accept-Encoding', 'gzip')
+          .use(disableAutoDeflate)
+          .then((res) => {
+            assert.match(res.header['content-type'], /application\/javascript/);
+            assert.equal(res.header['content-encoding'], 'gzip');
+            assert.equal(isPlaintextResponse(res.text, resource), false);
+            return;
+          })));
+    });
+
+    it('does not cache content-encoding headers', async function () {
+      await agent.get(packages[0])
+          .set('Accept-Encoding', fantasyEncoding)
+          .then((res) => assert.equal(res.header['content-encoding'], undefined));
+      await agent.get(packages[0])
+          .set('Accept-Encoding', 'gzip')
+          .then((res) => assert.equal(res.header['content-encoding'], 'gzip'));
+      await agent.get(packages[0])
+          .set('Accept-Encoding', fantasyEncoding)
+          .then((res) => assert.equal(res.header['content-encoding'], undefined));
+    });
+  });
+
+  context('when minify is true', function () {
+    before(async function () {
+      settings.minify = true;
+    });
+    it('gets packages uncompressed without Accept-Encoding gzip', async function () {
+      await Promise.all(packages.map(async (resource) => agent.get(resource)
+          .set('Accept-Encoding', fantasyEncoding)
+          .use(disableAutoDeflate)
+          .then((res) => {
+            assert.match(res.header['content-type'], /application\/javascript/);
+            assert.equal(res.header['content-encoding'], undefined);
+            assert.equal(isPlaintextResponse(res.text, resource), true);
+            return;
+          })));
+    });
+
+    it('gets packages compressed with Accept-Encoding gzip', async function () {
+      await Promise.all(packages.map(async (resource) => agent.get(resource)
+          .set('Accept-Encoding', 'gzip')
+          .use(disableAutoDeflate)
+          .then((res) => {
+            assert.match(res.header['content-type'], /application\/javascript/);
+            assert.equal(res.header['content-encoding'], 'gzip');
+            assert.equal(isPlaintextResponse(res.text, resource), false);
+            return;
+          })));
+    });
+
+    it('does not cache content-encoding headers', async function () {
+      await agent.get(packages[0])
+          .set('Accept-Encoding', fantasyEncoding)
+          .then((res) => assert.equal(res.header['content-encoding'], undefined));
+      await agent.get(packages[0])
+          .set('Accept-Encoding', 'gzip')
+          .then((res) => assert.equal(res.header['content-encoding'], 'gzip'));
+      await agent.get(packages[0])
+          .set('Accept-Encoding', fantasyEncoding)
+          .then((res) => assert.equal(res.header['content-encoding'], undefined));
+    });
+  });
+});

src/tests/backend/specs/contentcollector.js

@@ -0,0 +1,324 @@
+'use strict';
+
+/*
+ * While importexport tests target the `setHTML` API endpoint, which is nearly identical to what
+ * happens when a user manually imports a document via the UI, the contentcollector tests here don't
+ * use rehype to process the document. Rehype removes spaces and newĺines were applicable, so the
+ * expected results here can differ from importexport.js.
+ *
+ * If you add tests here, please also add them to importexport.js
+ */
+
+const AttributePool = require('../../../static/js/AttributePool');
+const assert = require('assert').strict;
+const cheerio = require('cheerio');
+const contentcollector = require('../../../static/js/contentcollector');
+
+const tests = {
+  nestedLi: {
+    description: 'Complex nested Li',
+    html: '<!doctype html><html><body><ol><li>one</li><li><ol><li>1.1</li></ol></li><li>two</li></ol></body></html>',
+    wantLineAttribs: [
+      '*0*1*2*3+1+3', '*0*4*2*5+1+3', '*0*1*2*5+1+3',
+    ],
+    wantText: [
+      '*one', '*1.1', '*two',
+    ],
+  },
+  complexNest: {
+    description: 'Complex list of different types',
+    html: '<!doctype html><html><body><ul class="bullet"><li>one</li><li>two</li><li>0</li><li>1</li><li>2<ul class="bullet"><li>3</li><li>4</li></ul></li></ul><ol class="number"><li>item<ol class="number"><li>item1</li><li>item2</li></ol></li></ol></body></html>',
+    wantLineAttribs: [
+      '*0*1*2+1+3',
+      '*0*1*2+1+3',
+      '*0*1*2+1+1',
+      '*0*1*2+1+1',
+      '*0*1*2+1+1',
+      '*0*3*2+1+1',
+      '*0*3*2+1+1',
+      '*0*4*2*5+1+4',
+      '*0*6*2*7+1+5',
+      '*0*6*2*7+1+5',
+    ],
+    wantText: [
+      '*one',
+      '*two',
+      '*0',
+      '*1',
+      '*2',
+      '*3',
+      '*4',
+      '*item',
+      '*item1',
+      '*item2',
+    ],
+  },
+  ul: {
+    description: 'Tests if uls properly get attributes',
+    html: '<html><body><ul><li>a</li><li>b</li></ul><div>div</div><p>foo</p></body></html>',
+    wantLineAttribs: ['*0*1*2+1+1', '*0*1*2+1+1', '+3', '+3'],
+    wantText: ['*a', '*b', 'div', 'foo'],
+  },
+  ulIndented: {
+    description: 'Tests if indented uls properly get attributes',
+    html: '<html><body><ul><li>a</li><ul><li>b</li></ul><li>a</li></ul><p>foo</p></body></html>',
+    wantLineAttribs: ['*0*1*2+1+1', '*0*3*2+1+1', '*0*1*2+1+1', '+3'],
+    wantText: ['*a', '*b', '*a', 'foo'],
+  },
+  ol: {
+    description: 'Tests if ols properly get line numbers when in a normal OL',
+    html: '<html><body><ol><li>a</li><li>b</li><li>c</li></ol><p>test</p></body></html>',
+    wantLineAttribs: ['*0*1*2*3+1+1', '*0*1*2*3+1+1', '*0*1*2*3+1+1', '+4'],
+    wantText: ['*a', '*b', '*c', 'test'],
+    noteToSelf: 'Ensure empty P does not induce line attribute marker, wont this break the editor?',
+  },
+  lineDoBreakInOl: {
+    description: 'A single completely empty line break within an ol should reset count if OL is closed off..',
+    html: '<html><body><ol><li>should be 1</li></ol><p>hello</p><ol><li>should be 1</li><li>should be 2</li></ol><p></p></body></html>',
+    wantLineAttribs: ['*0*1*2*3+1+b', '+5', '*0*1*2*4+1+b', '*0*1*2*4+1+b', ''],
+    wantText: ['*should be 1', 'hello', '*should be 1', '*should be 2', ''],
+    noteToSelf: "Shouldn't include attribute marker in the <p> line",
+  },
+  testP: {
+    description: 'A single <p></p> should create a new line',
+    html: '<html><body><p></p><p></p></body></html>',
+    wantLineAttribs: ['', ''],
+    wantText: ['', ''],
+    noteToSelf: '<p></p>should create a line break but not break numbering',
+  },
+  nestedOl: {
+    description: 'Tests if ols properly get line numbers when in a normal OL',
+    html: '<html><body>a<ol><li>b<ol><li>c</li></ol></ol>notlist<p>foo</p></body></html>',
+    wantLineAttribs: ['+1', '*0*1*2*3+1+1', '*0*4*2*5+1+1', '+7', '+3'],
+    wantText: ['a', '*b', '*c', 'notlist', 'foo'],
+    noteToSelf: 'Ensure empty P does not induce line attribute marker, wont this break the editor?',
+  },
+  nestedOl2: {
+    description: 'First item being an UL then subsequent being OL will fail',
+    html: '<html><body><ul><li>a<ol><li>b</li><li>c</li></ol></li></ul></body></html>',
+    wantLineAttribs: ['+1', '*0*1*2*3+1+1', '*0*4*2*5+1+1'],
+    wantText: ['a', '*b', '*c'],
+    noteToSelf: 'Ensure empty P does not induce line attribute marker, wont this break the editor?',
+    disabled: true,
+  },
+  lineDontBreakOL: {
+    description: 'A single completely empty line break within an ol should NOT reset count',
+    html: '<html><body><ol><li>should be 1</li><p></p><li>should be 2</li><li>should be 3</li></ol><p></p></body></html>',
+    wantLineAttribs: [],
+    wantText: ['*should be 1', '*should be 2', '*should be 3'],
+    noteToSelf: "<p></p>should create a line break but not break numbering -- This is what I can't get working!",
+    disabled: true,
+  },
+  ignoreAnyTagsOutsideBody: {
+    description: 'Content outside body should be ignored',
+    html: '<html><head><title>title</title><style></style></head><body>empty<br></body></html>',
+    wantLineAttribs: ['+5'],
+    wantText: ['empty'],
+  },
+  lineWithMultipleSpaces: {
+    description: 'Multiple spaces should be preserved',
+    html: '<html><body>Text with  more   than    one space.<br></body></html>',
+    wantLineAttribs: ['+10'],
+    wantText: ['Text with  more   than    one space.'],
+  },
+  lineWithMultipleNonBreakingAndNormalSpaces: {
+    description: 'non-breaking and normal space should be preserved',
+    html: '<html><body>Text&nbsp;with&nbsp; more&nbsp;&nbsp;&nbsp;than   &nbsp;one space.<br></body></html>',
+    wantLineAttribs: ['+10'],
+    wantText: ['Text with  more   than    one space.'],
+  },
+  multiplenbsp: {
+    description: 'Multiple nbsp should be preserved',
+    html: '<html><body>&nbsp;&nbsp;<br></body></html>',
+    wantLineAttribs: ['+2'],
+    wantText: ['  '],
+  },
+  multipleNonBreakingSpaceBetweenWords: {
+    description: 'Multiple nbsp between words ',
+    html: '<html><body>&nbsp;&nbsp;word1&nbsp;&nbsp;word2&nbsp;&nbsp;&nbsp;word3<br></body></html>',
+    wantLineAttribs: ['+m'],
+    wantText: ['  word1  word2   word3'],
+  },
+  nonBreakingSpacePreceededBySpaceBetweenWords: {
+    description: 'A non-breaking space preceded by a normal space',
+    html: '<html><body> &nbsp;word1 &nbsp;word2 &nbsp;word3<br></body></html>',
+    wantLineAttribs: ['+l'],
+    wantText: ['  word1  word2  word3'],
+  },
+  nonBreakingSpaceFollowededBySpaceBetweenWords: {
+    description: 'A non-breaking space followed by a normal space',
+    html: '<html><body>&nbsp; word1&nbsp; word2&nbsp; word3<br></body></html>',
+    wantLineAttribs: ['+l'],
+    wantText: ['  word1  word2  word3'],
+  },
+  spacesAfterNewline: {
+    description: 'Don\'t collapse spaces that follow a newline',
+    html: '<!doctype html><html><body>something<br>             something<br></body></html>',
+    wantLineAttribs: ['+9', '+m'],
+    wantText: ['something', '             something'],
+  },
+  spacesAfterNewlineP: {
+    description: 'Don\'t collapse spaces that follow a empty paragraph',
+    html: '<!doctype html><html><body>something<p></p>             something<br></body></html>',
+    wantLineAttribs: ['+9', '', '+m'],
+    wantText: ['something', '', '             something'],
+  },
+  spacesAtEndOfLine: {
+    description: 'Don\'t collapse spaces that preceed/follow a newline',
+    html: '<html><body>something            <br>             something<br></body></html>',
+    wantLineAttribs: ['+l', '+m'],
+    wantText: ['something            ', '             something'],
+  },
+  spacesAtEndOfLineP: {
+    description: 'Don\'t collapse spaces that preceed/follow a empty paragraph',
+    html: '<html><body>something            <p></p>             something<br></body></html>',
+    wantLineAttribs: ['+l', '', '+m'],
+    wantText: ['something            ', '', '             something'],
+  },
+  nonBreakingSpacesAfterNewlines: {
+    description: 'Don\'t collapse non-breaking spaces that follow a newline',
+    html: '<html><body>something<br>&nbsp;&nbsp;&nbsp;something<br></body></html>',
+    wantLineAttribs: ['+9', '+c'],
+    wantText: ['something', '   something'],
+  },
+  nonBreakingSpacesAfterNewlinesP: {
+    description: 'Don\'t collapse non-breaking spaces that follow a paragraph',
+    html: '<html><body>something<p></p>&nbsp;&nbsp;&nbsp;something<br></body></html>',
+    wantLineAttribs: ['+9', '', '+c'],
+    wantText: ['something', '', '   something'],
+  },
+  preserveSpacesInsideElements: {
+    description: 'Preserve all spaces when multiple are present',
+    html: '<html><body>Need <span> more </span> space<i>  s </i> !<br></body></html>',
+    wantLineAttribs: ['+h*0+4+2'],
+    wantText: ['Need  more  space  s  !'],
+  },
+  preserveSpacesAcrossNewlines: {
+    description: 'Newlines and multiple spaces across newlines should be preserved',
+    html: `
+      <html><body>Need
+          <span> more </span>
+          space
+          <i>  s </i>
+          !<br></body></html>`,
+    wantLineAttribs: ['+19*0+4+b'],
+    wantText: ['Need           more           space            s           !'],
+  },
+  multipleNewLinesAtBeginning: {
+    description: 'Multiple new lines at the beginning should be preserved',
+    html: '<html><body><br><br><p></p><p></p>first line<br><br>second line<br></body></html>',
+    wantLineAttribs: ['', '', '', '', '+a', '', '+b'],
+    wantText: ['', '', '', '', 'first line', '', 'second line'],
+  },
+  multiLineParagraph: {
+    description: 'A paragraph with multiple lines should not loose spaces when lines are combined',
+    html: `<html><body><p>
+а б в г ґ д е є ж з и і ї й к л м н о
+п р с т у ф х ц ч ш щ ю я ь</p>
+</body></html>`,
+    wantLineAttribs: ['+1t'],
+    wantText: ['а б в г ґ д е є ж з и і ї й к л м н о п р с т у ф х ц ч ш щ ю я ь'],
+  },
+  multiLineParagraphWithPre: {
+    description: 'lines in preformatted text should be kept intact',
+    html: `<html><body><p>
+а б в г ґ д е є ж з и і ї й к л м н о</p><pre>multiple
+lines
+in
+pre
+</pre><p>п р с т у ф х ц ч ш щ ю я
+ь</p>
+</body></html>`,
+    wantLineAttribs: ['+11', '+8', '+5', '+2', '+3', '+r'],
+    wantText: [
+      'а б в г ґ д е є ж з и і ї й к л м н о',
+      'multiple',
+      'lines',
+      'in',
+      'pre',
+      'п р с т у ф х ц ч ш щ ю я ь',
+    ],
+  },
+  preIntroducesASpace: {
+    description: 'pre should be on a new line not preceded by a space',
+    html: `<html><body><p>
+    1
+</p><pre>preline
+</pre></body></html>`,
+    wantLineAttribs: ['+6', '+7'],
+    wantText: ['    1 ', 'preline'],
+  },
+  dontDeleteSpaceInsideElements: {
+    description: 'Preserve spaces on the beginning and end of a element',
+    html: '<html><body>Need<span> more </span>space<i> s </i>!<br></body></html>',
+    wantLineAttribs: ['+f*0+3+1'],
+    wantText: ['Need more space s !'],
+  },
+  dontDeleteSpaceOutsideElements: {
+    description: 'Preserve spaces outside elements',
+    html: '<html><body>Need <span>more</span> space <i>s</i> !<br></body></html>',
+    wantLineAttribs: ['+g*0+1+2'],
+    wantText: ['Need more space s !'],
+  },
+  dontDeleteSpaceAtEndOfElement: {
+    description: 'Preserve spaces at the end of an element',
+    html: '<html><body>Need <span>more </span>space <i>s </i>!<br></body></html>',
+    wantLineAttribs: ['+g*0+2+1'],
+    wantText: ['Need more space s !'],
+  },
+  dontDeleteSpaceAtBeginOfElements: {
+    description: 'Preserve spaces at the start of an element',
+    html: '<html><body>Need<span> more</span> space<i> s</i> !<br></body></html>',
+    wantLineAttribs: ['+f*0+2+2'],
+    wantText: ['Need more space s !'],
+  },
+};
+
+describe(__filename, function () {
+  for (const test of Object.keys(tests)) {
+    const testObj = tests[test];
+    describe(test, function () {
+      if (testObj.disabled) {
+        return xit('DISABLED:', test, function (done) {
+          done();
+        });
+      }
+
+      it(testObj.description, async function () {
+        const $ = cheerio.load(testObj.html); // Load HTML into Cheerio
+        const doc = $('body')[0]; // Creates a dom-like representation of HTML
+        // Create an empty attribute pool
+        const apool = new AttributePool();
+        // Convert a dom tree into a list of lines and attribute liens
+        // using the content collector object
+        const cc = contentcollector.makeContentCollector(true, null, apool);
+        cc.collectContent(doc);
+        const result = cc.finish();
+        const gotAttributes = result.lineAttribs;
+        const wantAttributes = testObj.wantLineAttribs;
+        const gotText = new Array(result.lines);
+        const wantText = testObj.wantText;
+
+        assert.deepEqual(gotText[0], wantText);
+        assert.deepEqual(gotAttributes, wantAttributes);
+      });
+    });
+  }
+});
+
+
+function arraysEqual(a, b) {
+  if (a === b) return true;
+  if (a == null || b == null) return false;
+  if (a.length !== b.length) return false;
+
+  // If you don't care about the order of the elements inside
+  // the array, you should sort both arrays here.
+  // Please note that calling sort on an array will modify that array.
+  // you might want to clone your array first.
+
+  for (let i = 0; i < a.length; ++i) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}

src/tests/backend/specs/promises.js

@@ -0,0 +1,85 @@
+const assert = require('assert').strict;
+const promises = require('../../../node/utils/promises');
+
+describe(__filename, function () {
+  describe('promises.timesLimit', function () {
+    let wantIndex = 0;
+    const testPromises = [];
+    const makePromise = (index) => {
+      // Make sure index increases by one each time.
+      assert.equal(index, wantIndex++);
+      // Save the resolve callback (so the test can trigger resolution)
+      // and the promise itself (to wait for resolve to take effect).
+      const p = {};
+      const promise = new Promise((resolve) => {
+        p.resolve = resolve;
+      });
+      p.promise = promise;
+      testPromises.push(p);
+      return p.promise;
+    };
+
+    const total = 11;
+    const concurrency = 7;
+    const timesLimitPromise = promises.timesLimit(total, concurrency, makePromise);
+
+    it('honors concurrency', async function () {
+      assert.equal(wantIndex, concurrency);
+    });
+
+    it('creates another when one completes', async function () {
+      const {promise, resolve} = testPromises.shift();
+      resolve();
+      await promise;
+      assert.equal(wantIndex, concurrency + 1);
+    });
+
+    it('creates the expected total number of promises', async function () {
+      while (testPromises.length > 0) {
+        // Resolve them in random order to ensure that the resolution order doesn't matter.
+        const i = Math.floor(Math.random() * Math.floor(testPromises.length));
+        const {promise, resolve} = testPromises.splice(i, 1)[0];
+        resolve();
+        await promise;
+      }
+      assert.equal(wantIndex, total);
+    });
+
+    it('resolves', async function () {
+      await timesLimitPromise;
+    });
+
+    it('does not create too many promises if total < concurrency', async function () {
+      wantIndex = 0;
+      assert.equal(testPromises.length, 0);
+      const total = 7;
+      const concurrency = 11;
+      const timesLimitPromise = promises.timesLimit(total, concurrency, makePromise);
+      while (testPromises.length > 0) {
+        const {promise, resolve} = testPromises.pop();
+        resolve();
+        await promise;
+      }
+      await timesLimitPromise;
+      assert.equal(wantIndex, total);
+    });
+
+    it('accepts total === 0, concurrency > 0', async function () {
+      wantIndex = 0;
+      assert.equal(testPromises.length, 0);
+      await promises.timesLimit(0, concurrency, makePromise);
+      assert.equal(wantIndex, 0);
+    });
+
+    it('accepts total === 0, concurrency === 0', async function () {
+      wantIndex = 0;
+      assert.equal(testPromises.length, 0);
+      await promises.timesLimit(0, 0, makePromise);
+      assert.equal(wantIndex, 0);
+    });
+
+    it('rejects total > 0, concurrency === 0', async function () {
+      await assert.rejects(promises.timesLimit(total, 0, makePromise), RangeError);
+    });
+  });
+});

src/tests/backend/specs/socketio.js

@@ -0,0 +1,357 @@
+'use strict';
+
+const assert = require('assert').strict;
+const common = require('../common');
+const io = require('socket.io-client');
+const padManager = require('../../../node/db/PadManager');
+const plugins = require('../../../static/js/pluginfw/plugin_defs');
+const setCookieParser = require('set-cookie-parser');
+const settings = require('../../../node/utils/Settings');
+
+const logger = common.logger;
+
+// Waits for and returns the next named socket.io event. Rejects if there is any error while waiting
+// (unless waiting for that error event).
+const getSocketEvent = async (socket, event) => {
+  const errorEvents = [
+    'error',
+    'connect_error',
+    'connect_timeout',
+    'reconnect_error',
+    'reconnect_failed',
+  ];
+  const handlers = {};
+  let timeoutId;
+  return new Promise((resolve, reject) => {
+    timeoutId = setTimeout(() => reject(new Error(`timed out waiting for ${event} event`)), 1000);
+    for (const event of errorEvents) {
+      handlers[event] = (errorString) => {
+        logger.debug(`socket.io ${event} event: ${errorString}`);
+        reject(new Error(errorString));
+      };
+    }
+    // This will overwrite one of the above handlers if the user is waiting for an error event.
+    handlers[event] = (...args) => {
+      logger.debug(`socket.io ${event} event`);
+      if (args.length > 1) return resolve(args);
+      resolve(args[0]);
+    };
+    Object.entries(handlers).forEach(([event, handler]) => socket.on(event, handler));
+  }).finally(() => {
+    clearTimeout(timeoutId);
+    Object.entries(handlers).forEach(([event, handler]) => socket.off(event, handler));
+  });
+};
+
+// Establishes a new socket.io connection. Passes the cookies from the `set-cookie` header(s) in
+// `res` (which may be nullish) to the server. Returns a socket.io Socket object.
+const connect = async (res) => {
+  // Convert the `set-cookie` header(s) into a `cookie` header.
+  const resCookies = (res == null) ? {} : setCookieParser.parse(res, {map: true});
+  const reqCookieHdr = Object.entries(resCookies).map(
+      ([name, cookie]) => `${name}=${encodeURIComponent(cookie.value)}`).join('; ');
+
+  logger.debug('socket.io connecting...');
+  const socket = io(`${common.baseUrl}/`, {
+    forceNew: true, // Different tests will have different query parameters.
+    path: '/socket.io',
+    // socketio.js-client on node.js doesn't support cookies (see https://git.io/JU8u9), so the
+    // express_sid cookie must be passed as a query parameter.
+    query: {cookie: reqCookieHdr},
+  });
+  try {
+    await getSocketEvent(socket, 'connect');
+  } catch (e) {
+    socket.close();
+    throw e;
+  }
+  logger.debug('socket.io connected');
+
+  return socket;
+};
+
+// Helper function to exchange CLIENT_READY+CLIENT_VARS messages for the named pad.
+// Returns the CLIENT_VARS message from the server.
+const handshake = async (socket, padID) => {
+  logger.debug('sending CLIENT_READY...');
+  socket.send({
+    component: 'pad',
+    type: 'CLIENT_READY',
+    padId: padID,
+    sessionID: null,
+    token: 't.12345',
+    protocolVersion: 2,
+  });
+  logger.debug('waiting for CLIENT_VARS response...');
+  const msg = await getSocketEvent(socket, 'message');
+  logger.debug('received CLIENT_VARS message');
+  return msg;
+};
+
+describe(__filename, function () {
+  let agent;
+  let authorize;
+  const backups = {};
+  const cleanUpPads = async () => {
+    const padIds = ['pad', 'other-pad', 'päd'];
+    await Promise.all(padIds.map(async (padId) => {
+      if (await padManager.doesPadExist(padId)) {
+        const pad = await padManager.getPad(padId);
+        await pad.remove();
+      }
+    }));
+  };
+  let socket;
+
+  before(async function () { agent = await common.init(); });
+  beforeEach(async function () {
+    backups.hooks = {};
+    for (const hookName of ['preAuthorize', 'authenticate', 'authorize']) {
+      backups.hooks[hookName] = plugins.hooks[hookName];
+      plugins.hooks[hookName] = [];
+    }
+    backups.settings = {};
+    for (const setting of ['editOnly', 'requireAuthentication', 'requireAuthorization', 'users']) {
+      backups.settings[setting] = settings[setting];
+    }
+    settings.editOnly = false;
+    settings.requireAuthentication = false;
+    settings.requireAuthorization = false;
+    settings.users = {
+      admin: {password: 'admin-password', is_admin: true},
+      user: {password: 'user-password'},
+    };
+    assert(socket == null);
+    authorize = () => true;
+    plugins.hooks.authorize = [{hook_fn: (hookName, {req}, cb) => cb([authorize(req)])}];
+    await cleanUpPads();
+  });
+  afterEach(async function () {
+    if (socket) socket.close();
+    socket = null;
+    await cleanUpPads();
+    Object.assign(plugins.hooks, backups.hooks);
+    Object.assign(settings, backups.settings);
+  });
+
+  describe('Normal accesses', function () {
+    it('!authn anonymous cookie /p/pad -> 200, ok', async function () {
+      const res = await agent.get('/p/pad').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+    });
+    it('!authn !cookie -> ok', async function () {
+      socket = await connect(null);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+    });
+    it('!authn user /p/pad -> 200, ok', async function () {
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+    });
+    it('authn user /p/pad -> 200, ok', async function () {
+      settings.requireAuthentication = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+    });
+    it('authz user /p/pad -> 200, ok', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+    });
+    it('supports pad names with characters that must be percent-encoded', async function () {
+      settings.requireAuthentication = true;
+      // requireAuthorization is set to true here to guarantee that the user's padAuthorizations
+      // object is populated. Technically this isn't necessary because the user's padAuthorizations
+      // is currently populated even if requireAuthorization is false, but setting this to true
+      // ensures the test remains useful if the implementation ever changes.
+      settings.requireAuthorization = true;
+      const encodedPadId = encodeURIComponent('päd');
+      const res = await agent.get(`/p/${encodedPadId}`).auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'päd');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+    });
+  });
+
+  describe('Abnormal access attempts', function () {
+    it('authn anonymous /p/pad -> 401, error', async function () {
+      settings.requireAuthentication = true;
+      const res = await agent.get('/p/pad').expect(401);
+      // Despite the 401, try to create the pad via a socket.io connection anyway.
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it('authn !cookie -> error', async function () {
+      settings.requireAuthentication = true;
+      socket = await connect(null);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it('authorization bypass attempt -> error', async function () {
+      // Only allowed to access /p/pad.
+      authorize = (req) => req.path === '/p/pad';
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+      // First authenticate and establish a session.
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      // Accessing /p/other-pad should fail, despite the successful fetch of /p/pad.
+      const message = await handshake(socket, 'other-pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+  });
+
+  describe('Authorization levels via authorize hook', function () {
+    beforeEach(async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+    });
+
+    it("level='create' -> can create", async function () {
+      authorize = () => 'create';
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, false);
+    });
+    it('level=true -> can create', async function () {
+      authorize = () => true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, false);
+    });
+    it("level='modify' -> can modify", async function () {
+      await padManager.getPad('pad'); // Create the pad.
+      authorize = () => 'modify';
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, false);
+    });
+    it("level='create' settings.editOnly=true -> unable to create", async function () {
+      authorize = () => 'create';
+      settings.editOnly = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it("level='modify' settings.editOnly=false -> unable to create", async function () {
+      authorize = () => 'modify';
+      settings.editOnly = false;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it("level='readOnly' -> unable to create", async function () {
+      authorize = () => 'readOnly';
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it("level='readOnly' -> unable to modify", async function () {
+      await padManager.getPad('pad'); // Create the pad.
+      authorize = () => 'readOnly';
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, true);
+    });
+  });
+
+  describe('Authorization levels via user settings', function () {
+    beforeEach(async function () {
+      settings.requireAuthentication = true;
+    });
+
+    it('user.canCreate = true -> can create and modify', async function () {
+      settings.users.user.canCreate = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, false);
+    });
+    it('user.canCreate = false -> unable to create', async function () {
+      settings.users.user.canCreate = false;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it('user.readOnly = true -> unable to create', async function () {
+      settings.users.user.readOnly = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it('user.readOnly = true -> unable to modify', async function () {
+      await padManager.getPad('pad'); // Create the pad.
+      settings.users.user.readOnly = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, true);
+    });
+    it('user.readOnly = false -> can create and modify', async function () {
+      settings.users.user.readOnly = false;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const clientVars = await handshake(socket, 'pad');
+      assert.equal(clientVars.type, 'CLIENT_VARS');
+      assert.equal(clientVars.data.readonly, false);
+    });
+    it('user.readOnly = true, user.canCreate = true -> unable to create', async function () {
+      settings.users.user.canCreate = true;
+      settings.users.user.readOnly = true;
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+  });
+
+  describe('Authorization level interaction between authorize hook and user settings', function () {
+    beforeEach(async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+    });
+
+    it('authorize hook does not elevate level from user settings', async function () {
+      settings.users.user.readOnly = true;
+      authorize = () => 'create';
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+    it('user settings does not elevate level from authorize hook', async function () {
+      settings.users.user.readOnly = false;
+      settings.users.user.canCreate = true;
+      authorize = () => 'readOnly';
+      const res = await agent.get('/p/pad').auth('user', 'user-password').expect(200);
+      socket = await connect(res);
+      const message = await handshake(socket, 'pad');
+      assert.equal(message.accessStatus, 'deny');
+    });
+  });
+});

src/tests/backend/specs/specialpages.js

@@ -0,0 +1,25 @@
+const common = require('../common');
+const settings = require('../../../node/utils/Settings');
+
+describe(__filename, function () {
+  let agent;
+  const backups = {};
+  before(async function () { agent = await common.init(); });
+  beforeEach(async function () {
+    backups.settings = {};
+    for (const setting of ['requireAuthentication', 'requireAuthorization']) {
+      backups.settings[setting] = settings[setting];
+    }
+    settings.requireAuthentication = false;
+    settings.requireAuthorization = false;
+  });
+  afterEach(async function () {
+    Object.assign(settings, backups.settings);
+  });
+
+  describe('/javascript', function () {
+    it('/javascript -> 200', async function () {
+      await agent.get('/javascript').expect(200);
+    });
+  });
+});

src/tests/backend/specs/webaccess.js

@@ -0,0 +1,481 @@
+'use strict';
+
+const assert = require('assert').strict;
+const common = require('../common');
+const plugins = require('../../../static/js/pluginfw/plugin_defs');
+const settings = require('../../../node/utils/Settings');
+
+describe(__filename, function () {
+  let agent;
+  const backups = {};
+  const authHookNames = ['preAuthorize', 'authenticate', 'authorize'];
+  const failHookNames = ['preAuthzFailure', 'authnFailure', 'authzFailure', 'authFailure'];
+  const makeHook = (hookName, hookFn) => ({
+    hook_fn: hookFn,
+    hook_fn_name: `fake_plugin/${hookName}`,
+    hook_name: hookName,
+    part: {plugin: 'fake_plugin'},
+  });
+
+  before(async function () { agent = await common.init(); });
+  beforeEach(async function () {
+    backups.hooks = {};
+    for (const hookName of authHookNames.concat(failHookNames)) {
+      backups.hooks[hookName] = plugins.hooks[hookName];
+      plugins.hooks[hookName] = [];
+    }
+    backups.settings = {};
+    for (const setting of ['requireAuthentication', 'requireAuthorization', 'users']) {
+      backups.settings[setting] = settings[setting];
+    }
+    settings.requireAuthentication = false;
+    settings.requireAuthorization = false;
+    settings.users = {
+      admin: {password: 'admin-password', is_admin: true},
+      user: {password: 'user-password'},
+    };
+  });
+  afterEach(async function () {
+    Object.assign(plugins.hooks, backups.hooks);
+    Object.assign(settings, backups.settings);
+  });
+
+  describe('webaccess: without plugins', function () {
+    it('!authn !authz anonymous / -> 200', async function () {
+      settings.requireAuthentication = false;
+      settings.requireAuthorization = false;
+      await agent.get('/').expect(200);
+    });
+    it('!authn !authz anonymous /admin/ -> 401', async function () {
+      settings.requireAuthentication = false;
+      settings.requireAuthorization = false;
+      await agent.get('/admin/').expect(401);
+    });
+    it('authn !authz anonymous / -> 401', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = false;
+      await agent.get('/').expect(401);
+    });
+    it('authn !authz user / -> 200', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = false;
+      await agent.get('/').auth('user', 'user-password').expect(200);
+    });
+    it('authn !authz user /admin/ -> 403', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = false;
+      await agent.get('/admin/').auth('user', 'user-password').expect(403);
+    });
+    it('authn !authz admin / -> 200', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = false;
+      await agent.get('/').auth('admin', 'admin-password').expect(200);
+    });
+    it('authn !authz admin /admin/ -> 200', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = false;
+      await agent.get('/admin/').auth('admin', 'admin-password').expect(200);
+    });
+    it('authn authz user / -> 403', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+      await agent.get('/').auth('user', 'user-password').expect(403);
+    });
+    it('authn authz user /admin/ -> 403', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+      await agent.get('/admin/').auth('user', 'user-password').expect(403);
+    });
+    it('authn authz admin / -> 200', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+      await agent.get('/').auth('admin', 'admin-password').expect(200);
+    });
+    it('authn authz admin /admin/ -> 200', async function () {
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+      await agent.get('/admin/').auth('admin', 'admin-password').expect(200);
+    });
+
+    describe('login fails if password is nullish', function () {
+      for (const adminPassword of [undefined, null]) {
+        // https://tools.ietf.org/html/rfc7617 says that the username and password are sent as
+        // base64(username + ':' + password), but there's nothing stopping a malicious user from
+        // sending just base64(username) (no colon). The lack of colon could throw off credential
+        // parsing, resulting in successful comparisons against a null or undefined password.
+        for (const creds of ['admin', 'admin:']) {
+          it(`admin password: ${adminPassword} credentials: ${creds}`, async function () {
+            settings.users.admin.password = adminPassword;
+            const encCreds = Buffer.from(creds).toString('base64');
+            await agent.get('/admin/').set('Authorization', `Basic ${encCreds}`).expect(401);
+          });
+        }
+      }
+    });
+  });
+
+  describe('webaccess: preAuthorize, authenticate, and authorize hooks', function () {
+    let callOrder;
+    const Handler = class {
+      constructor(hookName, suffix) {
+        this.called = false;
+        this.hookName = hookName;
+        this.innerHandle = () => [];
+        this.id = hookName + suffix;
+        this.checkContext = () => {};
+      }
+      handle(hookName, context, cb) {
+        assert.equal(hookName, this.hookName);
+        assert(context != null);
+        assert(context.req != null);
+        assert(context.res != null);
+        assert(context.next != null);
+        this.checkContext(context);
+        assert(!this.called);
+        this.called = true;
+        callOrder.push(this.id);
+        return cb(this.innerHandle(context.req));
+      }
+    };
+    const handlers = {};
+
+    beforeEach(async function () {
+      callOrder = [];
+      for (const hookName of authHookNames) {
+        // Create two handlers for each hook to test deferral to the next function.
+        const h0 = new Handler(hookName, '_0');
+        const h1 = new Handler(hookName, '_1');
+        handlers[hookName] = [h0, h1];
+        plugins.hooks[hookName] = [
+          makeHook(hookName, h0.handle.bind(h0)),
+          makeHook(hookName, h1.handle.bind(h1)),
+        ];
+      }
+    });
+
+    describe('preAuthorize', function () {
+      beforeEach(async function () {
+        settings.requireAuthentication = false;
+        settings.requireAuthorization = false;
+      });
+
+      it('defers if it returns []', async function () {
+        await agent.get('/').expect(200);
+        // Note: The preAuthorize hook always runs even if requireAuthorization is false.
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1']);
+      });
+      it('bypasses authenticate and authorize hooks when true is returned', async function () {
+        settings.requireAuthentication = true;
+        settings.requireAuthorization = true;
+        handlers.preAuthorize[0].innerHandle = () => [true];
+        await agent.get('/').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0']);
+      });
+      it('bypasses authenticate and authorize hooks when false is returned', async function () {
+        settings.requireAuthentication = true;
+        settings.requireAuthorization = true;
+        handlers.preAuthorize[0].innerHandle = () => [false];
+        await agent.get('/').expect(403);
+        assert.deepEqual(callOrder, ['preAuthorize_0']);
+      });
+      it('bypasses authenticate and authorize hooks for static content, defers', async function () {
+        settings.requireAuthentication = true;
+        settings.requireAuthorization = true;
+        await agent.get('/static/robots.txt').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1']);
+      });
+      it('cannot grant access to /admin', async function () {
+        handlers.preAuthorize[0].innerHandle = () => [true];
+        await agent.get('/admin/').expect(401);
+        // Notes:
+        //   * preAuthorize[1] is called despite preAuthorize[0] returning a non-empty list because
+        //     'true' entries are ignored for /admin/* requests.
+        //   * The authenticate hook always runs for /admin/* requests even if
+        //     settings.requireAuthentication is false.
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('can deny access to /admin', async function () {
+        handlers.preAuthorize[0].innerHandle = () => [false];
+        await agent.get('/admin/').auth('admin', 'admin-password').expect(403);
+        assert.deepEqual(callOrder, ['preAuthorize_0']);
+      });
+      it('runs preAuthzFailure hook when access is denied', async function () {
+        handlers.preAuthorize[0].innerHandle = () => [false];
+        let called = false;
+        plugins.hooks.preAuthzFailure = [makeHook('preAuthzFailure', (hookName, {req, res}, cb) => {
+          assert.equal(hookName, 'preAuthzFailure');
+          assert(req != null);
+          assert(res != null);
+          assert(!called);
+          called = true;
+          res.status(200).send('injected');
+          return cb([true]);
+        })];
+        await agent.get('/admin/').auth('admin', 'admin-password').expect(200, 'injected');
+        assert(called);
+      });
+      it('returns 500 if an exception is thrown', async function () {
+        handlers.preAuthorize[0].innerHandle = () => { throw new Error('exception test'); };
+        await agent.get('/').expect(500);
+      });
+    });
+
+    describe('authenticate', function () {
+      beforeEach(async function () {
+        settings.requireAuthentication = true;
+        settings.requireAuthorization = false;
+      });
+
+      it('is not called if !requireAuthentication and not /admin/*', async function () {
+        settings.requireAuthentication = false;
+        await agent.get('/').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1']);
+      });
+      it('is called if !requireAuthentication and /admin/*', async function () {
+        settings.requireAuthentication = false;
+        await agent.get('/admin/').expect(401);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('defers if empty list returned', async function () {
+        await agent.get('/').expect(401);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('does not defer if return [true], 200', async function () {
+        handlers.authenticate[0].innerHandle = (req) => { req.session.user = {}; return [true]; };
+        await agent.get('/').expect(200);
+        // Note: authenticate_1 was not called because authenticate_0 handled it.
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1', 'authenticate_0']);
+      });
+      it('does not defer if return [false], 401', async function () {
+        handlers.authenticate[0].innerHandle = (req) => [false];
+        await agent.get('/').expect(401);
+        // Note: authenticate_1 was not called because authenticate_0 handled it.
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1', 'authenticate_0']);
+      });
+      it('falls back to HTTP basic auth', async function () {
+        await agent.get('/').auth('user', 'user-password').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('passes settings.users in context', async function () {
+        handlers.authenticate[0].checkContext = ({users}) => {
+          assert.equal(users, settings.users);
+        };
+        await agent.get('/').expect(401);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('passes user, password in context if provided', async function () {
+        handlers.authenticate[0].checkContext = ({username, password}) => {
+          assert.equal(username, 'user');
+          assert.equal(password, 'user-password');
+        };
+        await agent.get('/').auth('user', 'user-password').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('does not pass user, password in context if not provided', async function () {
+        handlers.authenticate[0].checkContext = ({username, password}) => {
+          assert(username == null);
+          assert(password == null);
+        };
+        await agent.get('/').expect(401);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('errors if req.session.user is not created', async function () {
+        handlers.authenticate[0].innerHandle = () => [true];
+        await agent.get('/').expect(500);
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1', 'authenticate_0']);
+      });
+      it('returns 500 if an exception is thrown', async function () {
+        handlers.authenticate[0].innerHandle = () => { throw new Error('exception test'); };
+        await agent.get('/').expect(500);
+        assert.deepEqual(callOrder, ['preAuthorize_0', 'preAuthorize_1', 'authenticate_0']);
+      });
+    });
+
+    describe('authorize', function () {
+      beforeEach(async function () {
+        settings.requireAuthentication = true;
+        settings.requireAuthorization = true;
+      });
+
+      it('is not called if !requireAuthorization (non-/admin)', async function () {
+        settings.requireAuthorization = false;
+        await agent.get('/').auth('user', 'user-password').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('is not called if !requireAuthorization (/admin)', async function () {
+        settings.requireAuthorization = false;
+        await agent.get('/admin/').auth('admin', 'admin-password').expect(200);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1']);
+      });
+      it('defers if empty list returned', async function () {
+        await agent.get('/').auth('user', 'user-password').expect(403);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1',
+          'authorize_0',
+          'authorize_1']);
+      });
+      it('does not defer if return [true], 200', async function () {
+        handlers.authorize[0].innerHandle = () => [true];
+        await agent.get('/').auth('user', 'user-password').expect(200);
+        // Note: authorize_1 was not called because authorize_0 handled it.
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1',
+          'authorize_0']);
+      });
+      it('does not defer if return [false], 403', async function () {
+        handlers.authorize[0].innerHandle = (req) => [false];
+        await agent.get('/').auth('user', 'user-password').expect(403);
+        // Note: authorize_1 was not called because authorize_0 handled it.
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1',
+          'authorize_0']);
+      });
+      it('passes req.path in context', async function () {
+        handlers.authorize[0].checkContext = ({resource}) => {
+          assert.equal(resource, '/');
+        };
+        await agent.get('/').auth('user', 'user-password').expect(403);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1',
+          'authorize_0',
+          'authorize_1']);
+      });
+      it('returns 500 if an exception is thrown', async function () {
+        handlers.authorize[0].innerHandle = () => { throw new Error('exception test'); };
+        await agent.get('/').auth('user', 'user-password').expect(500);
+        assert.deepEqual(callOrder, ['preAuthorize_0',
+          'preAuthorize_1',
+          'authenticate_0',
+          'authenticate_1',
+          'authorize_0']);
+      });
+    });
+  });
+
+  describe('webaccess: authnFailure, authzFailure, authFailure hooks', function () {
+    const Handler = class {
+      constructor(hookName) {
+        this.hookName = hookName;
+        this.shouldHandle = false;
+        this.called = false;
+      }
+      handle(hookName, context, cb) {
+        assert.equal(hookName, this.hookName);
+        assert(context != null);
+        assert(context.req != null);
+        assert(context.res != null);
+        assert(!this.called);
+        this.called = true;
+        if (this.shouldHandle) {
+          context.res.status(200).send(this.hookName);
+          return cb([true]);
+        }
+        return cb([]);
+      }
+    };
+    const handlers = {};
+
+    beforeEach(async function () {
+      failHookNames.forEach((hookName) => {
+        const handler = new Handler(hookName);
+        handlers[hookName] = handler;
+        plugins.hooks[hookName] = [makeHook(hookName, handler.handle.bind(handler))];
+      });
+      settings.requireAuthentication = true;
+      settings.requireAuthorization = true;
+    });
+
+    // authn failure tests
+    it('authn fail, no hooks handle -> 401', async function () {
+      await agent.get('/').expect(401);
+      assert(handlers.authnFailure.called);
+      assert(!handlers.authzFailure.called);
+      assert(handlers.authFailure.called);
+    });
+    it('authn fail, authnFailure handles', async function () {
+      handlers.authnFailure.shouldHandle = true;
+      await agent.get('/').expect(200, 'authnFailure');
+      assert(handlers.authnFailure.called);
+      assert(!handlers.authzFailure.called);
+      assert(!handlers.authFailure.called);
+    });
+    it('authn fail, authFailure handles', async function () {
+      handlers.authFailure.shouldHandle = true;
+      await agent.get('/').expect(200, 'authFailure');
+      assert(handlers.authnFailure.called);
+      assert(!handlers.authzFailure.called);
+      assert(handlers.authFailure.called);
+    });
+    it('authnFailure trumps authFailure', async function () {
+      handlers.authnFailure.shouldHandle = true;
+      handlers.authFailure.shouldHandle = true;
+      await agent.get('/').expect(200, 'authnFailure');
+      assert(handlers.authnFailure.called);
+      assert(!handlers.authFailure.called);
+    });
+
+    // authz failure tests
+    it('authz fail, no hooks handle -> 403', async function () {
+      await agent.get('/').auth('user', 'user-password').expect(403);
+      assert(!handlers.authnFailure.called);
+      assert(handlers.authzFailure.called);
+      assert(handlers.authFailure.called);
+    });
+    it('authz fail, authzFailure handles', async function () {
+      handlers.authzFailure.shouldHandle = true;
+      await agent.get('/').auth('user', 'user-password').expect(200, 'authzFailure');
+      assert(!handlers.authnFailure.called);
+      assert(handlers.authzFailure.called);
+      assert(!handlers.authFailure.called);
+    });
+    it('authz fail, authFailure handles', async function () {
+      handlers.authFailure.shouldHandle = true;
+      await agent.get('/').auth('user', 'user-password').expect(200, 'authFailure');
+      assert(!handlers.authnFailure.called);
+      assert(handlers.authzFailure.called);
+      assert(handlers.authFailure.called);
+    });
+    it('authzFailure trumps authFailure', async function () {
+      handlers.authzFailure.shouldHandle = true;
+      handlers.authFailure.shouldHandle = true;
+      await agent.get('/').auth('user', 'user-password').expect(200, 'authzFailure');
+      assert(handlers.authzFailure.called);
+      assert(!handlers.authFailure.called);
+    });
+  });
+});

src/tests/container/loadSettings.js

@@ -0,0 +1,37 @@
+/*
+ * ACHTUNG: this file is a hack used to load "settings.json.docker" instead of
+ *          "settings.json", since in its present form the Settings module does
+ *          not allow it.
+ *          This is a remnant of an analogous file that was placed in
+ *          <basedir>/tests/backend/loadSettings.js
+ *
+ * TODO: modify the Settings module:
+ *       1) no side effects on module load
+ *       2) write a factory method that loads a configuration file (taking the
+ *          file name from the command line, a function argument, or falling
+ *          back to a default)
+ */
+
+const fs = require('fs');
+const jsonminify = require('jsonminify');
+
+function loadSettings() {
+  let settingsStr = fs.readFileSync(`${__dirname}/../../../settings.json.docker`).toString();
+  // try to parse the settings
+  try {
+    if (settingsStr) {
+      settingsStr = jsonminify(settingsStr).replace(',]', ']').replace(',}', '}');
+      const settings = JSON.parse(settingsStr);
+
+      // custom settings for running in a container
+      settings.ip = 'localhost';
+      settings.port = '9001';
+
+      return settings;
+    }
+  } catch (e) {
+    console.error('whoops something is bad with settings');
+  }
+}
+
+exports.loadSettings = loadSettings;

src/tests/container/specs/api/pad.js

@@ -0,0 +1,38 @@
+/*
+ * ACHTUNG: this file was copied & modified from the analogous
+ * <basedir>/tests/backend/specs/api/pad.js
+ *
+ * TODO: unify those two files, and merge in a single one.
+ */
+
+const settings = require('../../loadSettings').loadSettings();
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+const apiVersion = 1;
+
+describe('Connectivity', function () {
+  it('can connect', function (done) {
+    api.get('/api/')
+        .expect('Content-Type', /json/)
+        .expect(200, done);
+  });
+});
+
+describe('API Versioning', function () {
+  it('finds the version tag', function (done) {
+    api.get('/api/')
+        .expect((res) => {
+          if (!res.body.currentVersion) throw new Error('No version set in API');
+          return;
+        })
+        .expect(200, done);
+  });
+});
+
+describe('Permission', function () {
+  it('errors with invalid APIKey', function (done) {
+    api.get(`/api/${apiVersion}/createPad?apikey=wrong_password&padID=test`)
+        .expect(401, done);
+  });
+});

src/tests/frontend/helper.js

@@ -0,0 +1,297 @@
+'use strict';
+const helper = {}; // eslint-disable-line no-redeclare
+
+(function () {
+  let $iframe; const
+    jsLibraries = {};
+
+  helper.init = function (cb) {
+    $.get('/static/js/jquery.js').done((code) => {
+      // make sure we don't override existing jquery
+      jsLibraries.jquery = `if(typeof $ === 'undefined') {\n${code}\n}`;
+
+      $.get('/tests/frontend/lib/sendkeys.js').done((code) => {
+        jsLibraries.sendkeys = code;
+
+        cb();
+      });
+    });
+  };
+
+  helper.randomString = function randomString(len) {
+    const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+    let randomstring = '';
+    for (let i = 0; i < len; i++) {
+      const rnum = Math.floor(Math.random() * chars.length);
+      randomstring += chars.substring(rnum, rnum + 1);
+    }
+    return randomstring;
+  };
+
+  const getFrameJQuery = function ($iframe) {
+    /*
+      I tried over 9001 ways to inject javascript into iframes.
+      This is the only way I found that worked in IE 7+8+9, FF and Chrome
+    */
+    const win = $iframe[0].contentWindow;
+    const doc = win.document;
+
+    // IE 8+9 Hack to make eval appear
+    // http://stackoverflow.com/questions/2720444/why-does-this-window-object-not-have-the-eval-function
+    win.execScript && win.execScript('null');
+
+    win.eval(jsLibraries.jquery);
+    win.eval(jsLibraries.sendkeys);
+
+    win.$.window = win;
+    win.$.document = doc;
+
+    return win.$;
+  };
+
+  helper.clearSessionCookies = function () {
+    // Expire cookies, so author and language are changed after reloading the pad.
+    // See https://developer.mozilla.org/en-US/docs/Web/API/Document/cookie#Example_4_Reset_the_previous_cookie
+    window.document.cookie = 'token=;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/';
+    window.document.cookie = 'language=;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/';
+  };
+
+  // Can only happen when the iframe exists, so we're doing it separately from other cookies
+  helper.clearPadPrefCookie = function () {
+    helper.padChrome$.document.cookie = 'prefsHttp=;expires=Thu, 01 Jan 1970 00:00:00 GMT';
+  };
+
+  // Overwrite all prefs in pad cookie. Assumes http, not https.
+  //
+  // `helper.padChrome$.document.cookie` (the iframe) and `window.document.cookie`
+  // seem to have independent cookies, UNLESS we put path=/ here (which we don't).
+  // I don't fully understand it, but this function seems to properly simulate
+  // padCookie.setPref in the client code
+  helper.setPadPrefCookie = function (prefs) {
+    helper.padChrome$.document.cookie =
+        (`prefsHttp=${escape(JSON.stringify(prefs))};expires=Thu, 01 Jan 3000 00:00:00 GMT`);
+  };
+
+  // Functionality for knowing what key event type is required for tests
+  let evtType = 'keydown';
+  // if it's IE require keypress
+  if (window.navigator.userAgent.indexOf('MSIE') > -1) {
+    evtType = 'keypress';
+  }
+  // Edge also requires keypress.
+  if (window.navigator.userAgent.indexOf('Edge') > -1) {
+    evtType = 'keypress';
+  }
+  // Opera also requires keypress.
+  if (window.navigator.userAgent.indexOf('OPR') > -1) {
+    evtType = 'keypress';
+  }
+  helper.evtType = evtType;
+
+  // @todo needs fixing asap
+  // newPad occasionally timeouts, might be a problem with ready/onload code during page setup
+  // This ensures that tests run regardless of this problem
+  helper.retry = 0;
+
+  helper.newPad = function (cb, padName) {
+    // build opts object
+    let opts = {clearCookies: true};
+    if (typeof cb === 'function') {
+      opts.cb = cb;
+    } else {
+      opts = _.defaults(cb, opts);
+    }
+
+    // if opts.params is set we manipulate the URL to include URL parameters IE ?foo=Bah.
+    let encodedParams;
+    if (opts.params) {
+      encodedParams = `?${$.param(opts.params)}`;
+    }
+    let hash;
+    if (opts.hash) {
+      hash = `#${opts.hash}`;
+    }
+
+    // clear cookies
+    if (opts.clearCookies) {
+      helper.clearSessionCookies();
+    }
+
+    if (!padName) padName = `FRONTEND_TEST_${helper.randomString(20)}`;
+    $iframe = $(`<iframe src='/p/${padName}${hash || ''}${encodedParams || ''}'></iframe>`);
+    // needed for retry
+    const origPadName = padName;
+
+    // clean up inner iframe references
+    helper.padChrome$ = helper.padOuter$ = helper.padInner$ = null;
+
+    // remove old iframe
+    $('#iframe-container iframe').remove();
+    // set new iframe
+    $('#iframe-container').append($iframe);
+    $iframe.one('load', () => {
+      helper.padChrome$ = getFrameJQuery($('#iframe-container iframe'));
+      if (opts.clearCookies) {
+        helper.clearPadPrefCookie();
+      }
+      if (opts.padPrefs) {
+        helper.setPadPrefCookie(opts.padPrefs);
+      }
+      helper.waitFor(() => !$iframe.contents().find('#editorloadingbox')
+          .is(':visible'), 10000).done(() => {
+        helper.padOuter$ = getFrameJQuery(helper.padChrome$('iframe[name="ace_outer"]'));
+        helper.padInner$ = getFrameJQuery(helper.padOuter$('iframe[name="ace_inner"]'));
+
+        // disable all animations, this makes tests faster and easier
+        helper.padChrome$.fx.off = true;
+        helper.padOuter$.fx.off = true;
+        helper.padInner$.fx.off = true;
+
+        /*
+         * chat messages received
+         * @type {Array}
+         */
+        helper.chatMessages = [];
+
+        /*
+         * changeset commits from the server
+         * @type {Array}
+         */
+        helper.commits = [];
+
+        /*
+         * userInfo messages from the server
+         * @type {Array}
+         */
+        helper.userInfos = [];
+
+        // listen for server messages
+        helper.spyOnSocketIO();
+        opts.cb();
+      }).fail(() => {
+        if (helper.retry > 3) {
+          throw new Error('Pad never loaded');
+        }
+        helper.retry++;
+        helper.newPad(cb, origPadName);
+      });
+    });
+
+    return padName;
+  };
+
+  helper.waitFor = function (conditionFunc, timeoutTime = 1900, intervalTime = 10) {
+    const deferred = new $.Deferred();
+
+    const _fail = deferred.fail.bind(deferred);
+    let listenForFail = false;
+    deferred.fail = (...args) => {
+      listenForFail = true;
+      return _fail(...args);
+    };
+
+    const check = () => {
+      try {
+        if (!conditionFunc()) return;
+        deferred.resolve();
+      } catch (err) {
+        deferred.reject(err);
+      }
+      clearInterval(intervalCheck);
+      clearTimeout(timeout);
+    };
+
+    const intervalCheck = setInterval(check, intervalTime);
+
+    const timeout = setTimeout(() => {
+      clearInterval(intervalCheck);
+      const error = new Error(`wait for condition never became true ${conditionFunc.toString()}`);
+      deferred.reject(error);
+
+      if (!listenForFail) {
+        throw error;
+      }
+    }, timeoutTime);
+
+    // Check right away to avoid an unnecessary sleep if the condition is already true.
+    check();
+
+    return deferred;
+  };
+
+  /**
+   * Same as `waitFor` but using Promises
+   *
+   * @returns {Promise}
+   *
+   */
+  helper.waitForPromise = async function (...args) {
+    // Note: waitFor() has a strange API: On timeout it rejects, but it also throws an uncatchable
+    // exception unless .fail() has been called. That uncatchable exception is disabled here by
+    // passing a no-op function to .fail().
+    return await this.waitFor(...args).fail(() => {});
+  };
+
+  helper.selectLines = function ($startLine, $endLine, startOffset, endOffset) {
+    // if no offset is provided, use beginning of start line and end of end line
+    startOffset = startOffset || 0;
+    endOffset = endOffset === undefined ? $endLine.text().length : endOffset;
+
+    const inner$ = helper.padInner$;
+    const selection = inner$.document.getSelection();
+    const range = selection.getRangeAt(0);
+
+    const start = getTextNodeAndOffsetOf($startLine, startOffset);
+    const end = getTextNodeAndOffsetOf($endLine, endOffset);
+
+    range.setStart(start.node, start.offset);
+    range.setEnd(end.node, end.offset);
+
+    selection.removeAllRanges();
+    selection.addRange(range);
+  };
+
+  const getTextNodeAndOffsetOf = function ($targetLine, targetOffsetAtLine) {
+    const $textNodes = $targetLine.find('*').contents().filter(function () {
+      return this.nodeType === Node.TEXT_NODE;
+    });
+
+    // search node where targetOffsetAtLine is reached, and its 'inner offset'
+    let textNodeWhereOffsetIs = null;
+    let offsetBeforeTextNode = 0;
+    let offsetInsideTextNode = 0;
+    $textNodes.each((index, element) => {
+      const elementTotalOffset = element.textContent.length;
+      textNodeWhereOffsetIs = element;
+      offsetInsideTextNode = targetOffsetAtLine - offsetBeforeTextNode;
+
+      const foundTextNode = offsetBeforeTextNode + elementTotalOffset >= targetOffsetAtLine;
+      if (foundTextNode) {
+        return false; // stop .each by returning false
+      }
+
+      offsetBeforeTextNode += elementTotalOffset;
+    });
+
+    // edge cases
+    if (textNodeWhereOffsetIs == null) {
+      // there was no text node inside $targetLine, so it is an empty line (<br>).
+      // Use beginning of line
+      textNodeWhereOffsetIs = $targetLine.get(0);
+      offsetInsideTextNode = 0;
+    }
+    // avoid errors if provided targetOffsetAtLine is higher than line offset (maxOffset).
+    // Use max allowed instead
+    const maxOffset = textNodeWhereOffsetIs.textContent.length;
+    offsetInsideTextNode = Math.min(offsetInsideTextNode, maxOffset);
+
+    return {
+      node: textNodeWhereOffsetIs,
+      offset: offsetInsideTextNode,
+    };
+  };
+
+  /* Ensure console.log doesn't blow up in IE, ugly but ok for a test framework imho*/
+  window.console = window.console || {};
+  window.console.log = window.console.log || function () {};
+})();

src/tests/frontend/helper/methods.js

@@ -0,0 +1,238 @@
+'use strict';
+
+/**
+ * Spys on socket.io messages and saves them into several arrays
+ * that are visible in tests
+ */
+helper.spyOnSocketIO = function () {
+  helper.contentWindow().pad.socket.on('message', (msg) => {
+    if (msg.type === 'COLLABROOM') {
+      if (msg.data.type === 'ACCEPT_COMMIT') {
+        helper.commits.push(msg);
+      } else if (msg.data.type === 'USER_NEWINFO') {
+        helper.userInfos.push(msg);
+      } else if (msg.data.type === 'CHAT_MESSAGE') {
+        helper.chatMessages.push(msg);
+      }
+    }
+  });
+};
+
+/**
+ * Makes an edit via `sendkeys` to the position of the caret and ensures ACCEPT_COMMIT
+ * is returned by the server
+ * It does not check if the ACCEPT_COMMIT is the edit sent, though
+ * If `line` is not given, the edit goes to line no. 1
+ *
+ * @param {string} message The edit to make - can be anything supported by `sendkeys`
+ * @param {number} [line] the optional line to make the edit on starting from 1
+ * @returns {Promise}
+ * @todo needs to support writing to a specified caret position
+ *
+ */
+helper.edit = async function (message, line) {
+  const editsNum = helper.commits.length;
+  line = line ? line - 1 : 0;
+  helper.linesDiv()[line].sendkeys(message);
+  return helper.waitForPromise(() => editsNum + 1 === helper.commits.length);
+};
+
+/**
+ * The pad text as an array of divs
+ *
+ * @example
+ * helper.linesDiv()[2].sendkeys('abc') // sends abc to the third line
+ *
+ * @returns {Array.<HTMLElement>} array of divs
+ */
+helper.linesDiv = function () {
+  return helper.padInner$('.ace-line').map(function () {
+    return $(this);
+  }).get();
+};
+
+/**
+ * The pad text as an array of lines
+ * For lines in timeslider use `helper.timesliderTextLines()`
+ *
+ * @returns {Array.<string>} lines of text
+ */
+helper.textLines = function () {
+  return helper.linesDiv().map((div) => div.text());
+};
+
+/**
+ * The default pad text transmitted via `clientVars`
+ *
+ * @returns {string}
+ */
+helper.defaultText = function () {
+  return helper.padChrome$.window.clientVars.collab_client_vars.initialAttributedText.text;
+};
+
+/**
+ * Sends a chat `message` via `sendKeys`
+ * You *must* include `{enter}` at the end of the string or it will
+ * just fill the input field but not send the message.
+ *
+ * @todo Cannot send multiple messages at once
+ *
+ * @example
+ *
+ * `helper.sendChatMessage('hi{enter}')`
+ *
+ * @param {string} message the chat message to be sent
+ * @returns {Promise}
+ */
+helper.sendChatMessage = function (message) {
+  const noOfChatMessages = helper.chatMessages.length;
+  helper.padChrome$('#chatinput').sendkeys(message);
+  return helper.waitForPromise(() => noOfChatMessages + 1 === helper.chatMessages.length);
+};
+
+/**
+ * Opens the settings menu if its hidden via button
+ *
+ * @returns {Promise}
+ */
+helper.showSettings = function () {
+  if (!helper.isSettingsShown()) {
+    helper.settingsButton().click();
+    return helper.waitForPromise(() => helper.isSettingsShown(), 2000);
+  }
+};
+
+/**
+ * Hide the settings menu if its open via button
+ *
+ * @returns {Promise}
+ * @todo untested
+ */
+helper.hideSettings = function () {
+  if (helper.isSettingsShown()) {
+    helper.settingsButton().click();
+    return helper.waitForPromise(() => !helper.isSettingsShown(), 2000);
+  }
+};
+
+/**
+ * Makes the chat window sticky via settings menu if the settings menu is
+ * open and sticky button is not checked
+ *
+ * @returns {Promise}
+ */
+helper.enableStickyChatviaSettings = function () {
+  const stickyChat = helper.padChrome$('#options-stickychat');
+  if (helper.isSettingsShown() && !stickyChat.is(':checked')) {
+    stickyChat.click();
+    return helper.waitForPromise(() => helper.isChatboxSticky(), 2000);
+  }
+};
+
+/**
+ * Unsticks the chat window via settings menu if the settings menu is open
+ * and sticky button is checked
+ *
+ * @returns {Promise}
+ */
+helper.disableStickyChatviaSettings = function () {
+  const stickyChat = helper.padChrome$('#options-stickychat');
+  if (helper.isSettingsShown() && stickyChat.is(':checked')) {
+    stickyChat.click();
+    return helper.waitForPromise(() => !helper.isChatboxSticky(), 2000);
+  }
+};
+
+/**
+ * Makes the chat window sticky via an icon on the top right of the chat
+ * window
+ *
+ * @returns {Promise}
+ */
+helper.enableStickyChatviaIcon = function () {
+  const stickyChat = helper.padChrome$('#titlesticky');
+  if (helper.isChatboxShown() && !helper.isChatboxSticky()) {
+    stickyChat.click();
+    return helper.waitForPromise(() => helper.isChatboxSticky(), 2000);
+  }
+};
+
+/**
+ * Disables the stickyness of the chat window via an icon on the
+ * upper right
+ *
+ * @returns {Promise}
+ */
+helper.disableStickyChatviaIcon = function () {
+  if (helper.isChatboxShown() && helper.isChatboxSticky()) {
+    helper.titlecross().click();
+    return helper.waitForPromise(() => !helper.isChatboxSticky(), 2000);
+  }
+};
+
+/**
+ * Sets the src-attribute of the main iframe to the timeslider
+ * In case a revision is given, sets the timeslider to this specific revision.
+ * Defaults to going to the last revision.
+ * It waits until the timer is filled with date and time, because it's one of the
+ * last things that happen during timeslider load
+ *
+ * @param {number} [revision] the optional revision
+ * @returns {Promise}
+ * @todo for some reason this does only work the first time, you cannot
+ * goto rev 0 and then via the same method to rev 5. Use buttons instead
+ */
+helper.gotoTimeslider = function (revision) {
+  revision = Number.isInteger(revision) ? `#${revision}` : '';
+  const iframe = $('#iframe-container iframe');
+  iframe.attr('src', `${iframe.attr('src')}/timeslider${revision}`);
+
+  return helper.waitForPromise(() => helper.timesliderTimerTime() &&
+      !Number.isNaN(new Date(helper.timesliderTimerTime()).getTime()), 10000);
+};
+
+/**
+ * Clicks in the timeslider at a specific offset
+ * It's used to navigate the timeslider
+ *
+ * @todo no mousemove test
+ * @param {number} X coordinate
+ */
+helper.sliderClick = function (X) {
+  const sliderBar = helper.sliderBar();
+  const edown = new jQuery.Event('mousedown');
+  const eup = new jQuery.Event('mouseup');
+  edown.clientX = eup.clientX = X;
+  edown.clientY = eup.clientY = sliderBar.offset().top;
+
+  sliderBar.trigger(edown);
+  sliderBar.trigger(eup);
+};
+
+/**
+ * The timeslider text as an array of lines
+ *
+ * @returns {Array.<string>} lines of text
+ */
+helper.timesliderTextLines = function () {
+  return helper.contentWindow().$('.ace-line').map(function () {
+    return $(this).text();
+  }).get();
+};
+
+helper.padIsEmpty = () => (
+  !helper.padInner$.document.getSelection().isCollapsed ||
+  (helper.padInner$('div').length === 1 && helper.padInner$('div').first().html() === '<br>'));
+
+helper.clearPad = async () => {
+  if (helper.padIsEmpty()) return;
+  const commitsBefore = helper.commits.length;
+  const lines = helper.linesDiv();
+  helper.selectLines(lines[0], lines[lines.length - 1]);
+  await helper.waitForPromise(() => !helper.padInner$.document.getSelection().isCollapsed);
+  const e = new helper.padInner$.Event(helper.evtType);
+  e.keyCode = 8; // delete key
+  helper.padInner$('#innerdocbody').trigger(e);
+  await helper.waitForPromise(helper.padIsEmpty);
+  await helper.waitForPromise(() => helper.commits.length > commitsBefore);
+};

src/tests/frontend/helper/ui.js

@@ -0,0 +1,191 @@
+'use strict';
+
+/**
+ * the contentWindow is either the normal pad or timeslider
+ *
+ * @returns {HTMLElement} contentWindow
+ */
+helper.contentWindow = function () {
+  return $('#iframe-container iframe')[0].contentWindow;
+};
+
+/**
+ * Opens the chat unless it is already open via an
+ * icon on the bottom right of the page
+ *
+ * @returns {Promise}
+ */
+helper.showChat = function () {
+  const chaticon = helper.chatIcon();
+  if (chaticon.hasClass('visible')) {
+    chaticon.click();
+    return helper.waitForPromise(() => !chaticon.hasClass('visible'), 2000);
+  }
+};
+
+/**
+ * Closes the chat window if it is shown and not sticky
+ *
+ * @returns {Promise}
+ */
+helper.hideChat = function () {
+  if (helper.isChatboxShown() && !helper.isChatboxSticky()) {
+    helper.titlecross().click();
+    return helper.waitForPromise(() => !helper.isChatboxShown(), 2000);
+  }
+};
+
+/**
+ * Gets the chat icon from the bottom right of the page
+ *
+ * @returns {HTMLElement} the chat icon
+ */
+helper.chatIcon = function () { return helper.padChrome$('#chaticon'); };
+
+/**
+ * The chat messages from the UI
+ *
+ * @returns {Array.<HTMLElement>}
+ */
+helper.chatTextParagraphs = function () { return helper.padChrome$('#chattext').children('p'); };
+
+/**
+ * Returns true if the chat box is sticky
+ *
+ * @returns {boolean} stickyness of the chat box
+ */
+helper.isChatboxSticky = function () {
+  return helper.padChrome$('#chatbox').hasClass('stickyChat');
+};
+
+/**
+ * Returns true if the chat box is shown
+ *
+ * @returns {boolean} visibility of the chat box
+ */
+helper.isChatboxShown = function () {
+  return helper.padChrome$('#chatbox').hasClass('visible');
+};
+
+/**
+ * Gets the settings menu
+ *
+ * @returns {HTMLElement} the settings menu
+ */
+helper.settingsMenu = function () { return helper.padChrome$('#settings'); };
+
+/**
+ * Gets the settings button
+ *
+ * @returns {HTMLElement} the settings button
+ */
+helper.settingsButton = function () {
+  return helper.padChrome$("button[data-l10n-id='pad.toolbar.settings.title']");
+};
+
+/**
+ * Toggles user list
+ */
+helper.toggleUserList = async function () {
+  const isVisible = helper.userListShown();
+  const button = helper.padChrome$("button[data-l10n-id='pad.toolbar.showusers.title']");
+  button.click();
+  await helper.waitForPromise(() => !isVisible);
+};
+
+/**
+ * Gets the user name input field
+ *
+ * @returns {HTMLElement} user name input field
+ */
+helper.usernameField = function () {
+  return helper.padChrome$("input[data-l10n-id='pad.userlist.entername']");
+};
+
+/**
+ * Is the user list popup shown?
+ *
+ * @returns {boolean}
+ */
+helper.userListShown = function () {
+  return helper.padChrome$('div#users').hasClass('popup-show');
+};
+
+/**
+ * Sets the user name
+ *
+ */
+helper.setUserName = async (name) => {
+  const userElement = helper.usernameField();
+  userElement.click();
+  userElement.val(name);
+  userElement.blur();
+  await helper.waitForPromise(() => !helper.usernameField().hasClass('editactive'));
+};
+
+/**
+ * Gets the titlecross icon
+ *
+ * @returns {HTMLElement} the titlecross icon
+ */
+helper.titlecross = function () { return helper.padChrome$('#titlecross'); };
+
+/**
+ * Returns true if the settings menu is visible
+ *
+ * @returns {boolean} is the settings menu shown?
+ */
+helper.isSettingsShown = function () {
+  return helper.padChrome$('#settings').hasClass('popup-show');
+};
+
+/**
+ * Gets the timer div of a timeslider that has the datetime of the revision
+ *
+ * @returns {HTMLElement} timer
+ */
+helper.timesliderTimer = function () {
+  if (typeof helper.contentWindow().$ === 'function') {
+    return helper.contentWindow().$('#timer');
+  }
+};
+
+/**
+ * Gets the time of the revision on a timeslider
+ *
+ * @returns {HTMLElement} timer
+ */
+helper.timesliderTimerTime = function () {
+  if (helper.timesliderTimer()) {
+    return helper.timesliderTimer().text();
+  }
+};
+
+/**
+ * The ui-slidar-bar element in the timeslider
+ *
+ * @returns {HTMLElement}
+ */
+helper.sliderBar = function () {
+  return helper.contentWindow().$('#ui-slider-bar');
+};
+
+/**
+ * revision_date element
+ * like "Saved October 10, 2020"
+ *
+ * @returns {HTMLElement}
+ */
+helper.revisionDateElem = function () {
+  return helper.contentWindow().$('#revision_date').text();
+};
+
+/**
+ * revision_label element
+ * like "Version 1"
+ *
+ * @returns {HTMLElement}
+ */
+helper.revisionLabelElem = function () {
+  return helper.contentWindow().$('#revision_label');
+};

src/tests/frontend/index.html

@@ -0,0 +1,26 @@
+<!doctype html>
+<html>
+    <title>Frontend tests</title>
+    <meta charset="utf-8">
+
+    <link rel="stylesheet" href="runner.css" />
+
+    <div id="console"></div>
+    <div id="mocha"></div>
+    <div id="iframe-container"></div>
+
+    <script src="/static/js/jquery.js"></script>
+    <script src="/static/js/browser.js"></script>
+    <script src="lib/underscore.js"></script>
+
+    <script src="lib/mocha.js"></script>
+    <script> mocha.setup({ui: 'bdd', checkLeaks: true, timeout: 60000}) </script>
+    <script src="lib/expect.js"></script>
+
+    <script src="helper.js"></script>
+    <script src="helper/methods.js"></script>
+    <script src="helper/ui.js"></script>
+    
+    <script src="specs_list.js"></script>
+    <script src="runner.js"></script>
+</html>

src/tests/frontend/lib/sendkeys.js

@@ -0,0 +1,467 @@
+// Cross-broswer implementation of text ranges and selections
+// documentation: http://bililite.com/blog/2011/01/11/cross-browser-.and-selections/
+// Version: 1.1
+// Copyright (c) 2010 Daniel Wachsstock
+// MIT license:
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+(function($){
+
+bililiteRange = function(el, debug){
+  var ret;
+  if (debug){
+    ret = new NothingRange(); // Easier to force it to use the no-selection type than to try to find an old browser
+  }else if (document.selection && !document.addEventListener){
+    // Internet Explorer 8 and lower
+    ret = new IERange();
+  }else if (window.getSelection && el.setSelectionRange){
+    // Standards. Element is an input or textarea
+    ret = new InputRange();
+  }else if (window.getSelection){
+    // Standards, with any other kind of element
+    ret = new W3CRange()
+  }else{
+    // doesn't support selection
+    ret = new NothingRange();
+  }
+  ret._el = el;
+  ret._doc = el.ownerDocument;
+  ret._win = 'defaultView' in ret._doc ? ret._doc.defaultView : ret._doc.parentWindow;
+  ret._textProp = textProp(el);
+  ret._bounds = [0, ret.length()];
+  return ret;
+}
+
+function textProp(el){
+  // returns the property that contains the text of the element
+  if (typeof el.value != 'undefined') return 'value';
+  if (typeof el.text != 'undefined') return 'text';
+  if (typeof el.textContent != 'undefined') return 'textContent';
+  return 'innerText';
+}
+
+// base class
+function Range(){}
+Range.prototype = {
+  length: function() {
+    return this._el[this._textProp].replace(/\r/g, '').length; // need to correct for IE's CrLf weirdness
+  },
+  bounds: function(s){
+    if (s === 'all'){
+      this._bounds = [0, this.length()];
+    }else if (s === 'start'){
+      this._bounds = [0, 0];
+    }else if (s === 'end'){
+      this._bounds = [this.length(), this.length()];
+    }else if (s === 'selection'){
+      this.bounds ('all'); // first select the whole thing for constraining
+      this._bounds = this._nativeSelection();
+    }else if (s){
+      this._bounds = s; // don't error check now; the element may change at any moment, so constrain it when we need it.
+    }else{
+      var b = [
+        Math.max(0, Math.min (this.length(), this._bounds[0])),
+        Math.max(0, Math.min (this.length(), this._bounds[1]))
+      ];
+      return b; // need to constrain it to fit
+    }
+    return this; // allow for chaining
+  },
+  select: function(){
+    this._nativeSelect(this._nativeRange(this.bounds()));
+    return this; // allow for chaining
+  },
+  text: function(text, select){
+    if (arguments.length){
+      this._nativeSetText(text, this._nativeRange(this.bounds()));
+      if (select == 'start'){
+        this.bounds ([this._bounds[0], this._bounds[0]]);
+        this.select();
+      }else if (select == 'end'){
+        this.bounds ([this._bounds[0]+text.length, this._bounds[0]+text.length]);
+        this.select();
+      }else if (select == 'all'){
+        this.bounds ([this._bounds[0], this._bounds[0]+text.length]);
+        this.select();
+      }
+      return this; // allow for chaining
+    }else{
+      return this._nativeGetText(this._nativeRange(this.bounds()));
+    }
+  },
+  insertEOL: function (){
+    this._nativeEOL();
+    this._bounds = [this._bounds[0]+1, this._bounds[0]+1]; // move past the EOL marker
+    return this;
+  }
+};
+
+
+function IERange(){}
+IERange.prototype = new Range();
+IERange.prototype._nativeRange = function (bounds){
+  var rng;
+  if (this._el.tagName == 'INPUT'){
+    // IE 8 is very inconsistent; textareas have createTextRange but it doesn't work
+    rng = this._el.createTextRange();
+  }else{
+    rng = this._doc.body.createTextRange ();
+    rng.moveToElementText(this._el);
+  }
+  if (bounds){
+    if (bounds[1] < 0) bounds[1] = 0; // IE tends to run elements out of bounds
+    if (bounds[0] > this.length()) bounds[0] = this.length();
+    if (bounds[1] < rng.text.replace(/\r/g, '').length){ // correct for IE's CrLf wierdness
+      // block-display elements have an invisible, uncounted end of element marker, so we move an extra one and use the current length of the range
+      rng.moveEnd ('character', -1);
+      rng.moveEnd ('character', bounds[1]-rng.text.replace(/\r/g, '').length);
+    }
+    if (bounds[0] > 0) rng.moveStart('character', bounds[0]);
+  }
+  return rng;
+};
+IERange.prototype._nativeSelect = function (rng){
+  rng.select();
+};
+IERange.prototype._nativeSelection = function (){
+  // returns [start, end] for the selection constrained to be in element
+  var rng = this._nativeRange(); // range of the element to constrain to
+  var len = this.length();
+  if (this._doc.selection.type != 'Text') return [0,0]; // append to the end
+  var sel = this._doc.selection.createRange();
+  try{
+    return [
+      iestart(sel, rng),
+      ieend (sel, rng)
+    ];
+  }catch (e){
+    // IE gets upset sometimes about comparing text to input elements, but the selections cannot overlap, so make a best guess
+    return (sel.parentElement().sourceIndex < this._el.sourceIndex) ? [0,0] : [len, len];
+  }
+};
+IERange.prototype._nativeGetText = function (rng){
+  return rng.text.replace(/\r/g, ''); // correct for IE's CrLf weirdness
+};
+IERange.prototype._nativeSetText = function (text, rng){
+  rng.text = text;
+};
+IERange.prototype._nativeEOL = function(){
+  if (typeof this._el.value != 'undefined'){
+    this.text('\n'); // for input and textarea, insert it straight
+  }else{
+    this._nativeRange(this.bounds()).pasteHTML('<br/>');
+  }
+};
+// IE internals
+function iestart(rng, constraint){
+  // returns the position (in character) of the start of rng within constraint. If it's not in constraint, returns 0 if it's before, length if it's after
+  var len = constraint.text.replace(/\r/g, '').length; // correct for IE's CrLf wierdness
+  if (rng.compareEndPoints ('StartToStart', constraint) <= 0) return 0; // at or before the beginning
+  if (rng.compareEndPoints ('StartToEnd', constraint) >= 0) return len;
+  for (var i = 0; rng.compareEndPoints ('StartToStart', constraint) > 0; ++i, rng.moveStart('character', -1));
+  return i;
+}
+function ieend (rng, constraint){
+  // returns the position (in character) of the end of rng within constraint. If it's not in constraint, returns 0 if it's before, length if it's after
+  var len = constraint.text.replace(/\r/g, '').length; // correct for IE's CrLf wierdness
+  if (rng.compareEndPoints ('EndToEnd', constraint) >= 0) return len; // at or after the end
+  if (rng.compareEndPoints ('EndToStart', constraint) <= 0) return 0;
+  for (var i = 0; rng.compareEndPoints ('EndToStart', constraint) > 0; ++i, rng.moveEnd('character', -1));
+  return i;
+}
+
+// an input element in a standards document. "Native Range" is just the bounds array
+function InputRange(){}
+InputRange.prototype = new Range();
+InputRange.prototype._nativeRange = function(bounds) {
+  return bounds || [0, this.length()];
+};
+InputRange.prototype._nativeSelect = function (rng){
+  this._el.setSelectionRange(rng[0], rng[1]);
+};
+InputRange.prototype._nativeSelection = function(){
+  return [this._el.selectionStart, this._el.selectionEnd];
+};
+InputRange.prototype._nativeGetText = function(rng){
+  return this._el.value.substring(rng[0], rng[1]);
+};
+InputRange.prototype._nativeSetText = function(text, rng){
+  var val = this._el.value;
+  this._el.value = val.substring(0, rng[0]) + text + val.substring(rng[1]);
+};
+InputRange.prototype._nativeEOL = function(){
+  this.text('\n');
+};
+
+function W3CRange(){}
+W3CRange.prototype = new Range();
+W3CRange.prototype._nativeRange = function (bounds){
+  var rng = this._doc.createRange();
+  rng.selectNodeContents(this._el);
+  if (bounds){
+    w3cmoveBoundary (rng, bounds[0], true, this._el);
+    rng.collapse (true);
+    w3cmoveBoundary (rng, bounds[1]-bounds[0], false, this._el);
+  }
+  return rng;
+};
+W3CRange.prototype._nativeSelect = function (rng){
+  this._win.getSelection().removeAllRanges();
+  this._win.getSelection().addRange (rng);
+};
+W3CRange.prototype._nativeSelection = function (){
+    // returns [start, end] for the selection constrained to be in element
+    var rng = this._nativeRange(); // range of the element to constrain to
+    if (this._win.getSelection().rangeCount == 0) return [this.length(), this.length()]; // append to the end
+    var sel = this._win.getSelection().getRangeAt(0);
+    return [
+      w3cstart(sel, rng),
+      w3cend (sel, rng)
+    ];
+  }
+W3CRange.prototype._nativeGetText = function (rng){
+  return rng.toString();
+};
+W3CRange.prototype._nativeSetText = function (text, rng){
+  rng.deleteContents();
+  rng.insertNode (this._doc.createTextNode(text));
+  this._el.normalize(); // merge the text with the surrounding text
+};
+W3CRange.prototype._nativeEOL = function(){
+  var rng = this._nativeRange(this.bounds());
+  rng.deleteContents();
+  var br = this._doc.createElement('br');
+  br.setAttribute ('_moz_dirty', ''); // for Firefox
+  rng.insertNode (br);
+  rng.insertNode (this._doc.createTextNode('\n'));
+  rng.collapse (false);
+};
+// W3C internals
+function nextnode (node, root){
+  //  in-order traversal
+  // we've already visited node, so get kids then siblings
+  if (node.firstChild) return node.firstChild;
+  if (node.nextSibling) return node.nextSibling;
+  if (node===root) return null;
+  while (node.parentNode){
+    // get uncles
+    node = node.parentNode;
+    if (node == root) return null;
+    if (node.nextSibling) return node.nextSibling;
+  }
+  return null;
+}
+function w3cmoveBoundary (rng, n, bStart, el){
+  // move the boundary (bStart == true ? start : end) n characters forward, up to the end of element el. Forward only!
+  // if the start is moved after the end, then an exception is raised
+  if (n <= 0) return;
+  var node = rng[bStart ? 'startContainer' : 'endContainer'];
+  if (node.nodeType == 3){
+    // we may be starting somewhere into the text
+    n += rng[bStart ? 'startOffset' : 'endOffset'];
+  }
+  while (node){
+    if (node.nodeType == 3){
+      if (n <= node.nodeValue.length){
+        rng[bStart ? 'setStart' : 'setEnd'](node, n);
+        // special case: if we end next to a <br>, include that node.
+        if (n == node.nodeValue.length){
+          // skip past zero-length text nodes
+          for (var next = nextnode (node, el); next && next.nodeType==3 && next.nodeValue.length == 0; next = nextnode(next, el)){
+            rng[bStart ? 'setStartAfter' : 'setEndAfter'](next);
+          }
+          if (next && next.nodeType == 1 && next.nodeName == "BR") rng[bStart ? 'setStartAfter' : 'setEndAfter'](next);
+        }
+        return;
+      }else{
+        rng[bStart ? 'setStartAfter' : 'setEndAfter'](node); // skip past this one
+        n -= node.nodeValue.length; // and eat these characters
+      }
+    }
+    node = nextnode (node, el);
+  }
+}
+var     START_TO_START                 = 0; // from the w3c definitions
+var     START_TO_END                   = 1;
+var     END_TO_END                     = 2;
+var     END_TO_START                   = 3;
+// from the Mozilla documentation, for range.compareBoundaryPoints(how, sourceRange)
+// -1, 0, or 1, indicating whether the corresponding boundary-point of range is respectively before, equal to, or after the corresponding boundary-point of sourceRange.
+    // * Range.END_TO_END compares the end boundary-point of sourceRange to the end boundary-point of range.
+    // * Range.END_TO_START compares the end boundary-point of sourceRange to the start boundary-point of range.
+    // * Range.START_TO_END compares the start boundary-point of sourceRange to the end boundary-point of range.
+    // * Range.START_TO_START compares the start boundary-point of sourceRange to the start boundary-point of range.
+function w3cstart(rng, constraint){
+  if (rng.compareBoundaryPoints (START_TO_START, constraint) <= 0) return 0; // at or before the beginning
+  if (rng.compareBoundaryPoints (END_TO_START, constraint) >= 0) return constraint.toString().length;
+  rng = rng.cloneRange(); // don't change the original
+  rng.setEnd (constraint.endContainer, constraint.endOffset); // they now end at the same place
+  return constraint.toString().length - rng.toString().length;
+}
+function w3cend (rng, constraint){
+  if (rng.compareBoundaryPoints (END_TO_END, constraint) >= 0) return constraint.toString().length; // at or after the end
+  if (rng.compareBoundaryPoints (START_TO_END, constraint) <= 0) return 0;
+  rng = rng.cloneRange(); // don't change the original
+  rng.setStart (constraint.startContainer, constraint.startOffset); // they now start at the same place
+  return rng.toString().length;
+}
+
+function NothingRange(){}
+NothingRange.prototype = new Range();
+NothingRange.prototype._nativeRange = function(bounds) {
+  return bounds || [0,this.length()];
+};
+NothingRange.prototype._nativeSelect = function (rng){ // do nothing
+};
+NothingRange.prototype._nativeSelection = function(){
+  return [0,0];
+};
+NothingRange.prototype._nativeGetText = function (rng){
+  return this._el[this._textProp].substring(rng[0], rng[1]);
+};
+NothingRange.prototype._nativeSetText = function (text, rng){
+  var val = this._el[this._textProp];
+  this._el[this._textProp] = val.substring(0, rng[0]) + text + val.substring(rng[1]);
+};
+NothingRange.prototype._nativeEOL = function(){
+  this.text('\n');
+};
+
+})(jQuery);
+
+// insert characters in a textarea or text input field
+// special characters are enclosed in {}; use {{} for the { character itself
+// documentation: http://bililite.com/blog/2008/08/20/the-fnsendkeys-plugin/
+// Version: 2.0
+// Copyright (c) 2010 Daniel Wachsstock
+// MIT license:
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+(function($){
+
+$.fn.sendkeys = function (x, opts){
+  return this.each( function(){
+    var localkeys = $.extend({}, opts, $(this).data('sendkeys')); // allow for element-specific key functions
+    // most elements to not keep track of their selection when they lose focus, so we have to do it for them
+    var rng = $.data (this, 'sendkeys.selection');
+    if (!rng){
+      rng = bililiteRange(this).bounds('selection');
+      $.data(this, 'sendkeys.selection', rng);
+      $(this).bind('mouseup.sendkeys', function(){
+        // we have to update the saved range. The routines here update the bounds with each press, but actual keypresses and mouseclicks do not
+        $.data(this, 'sendkeys.selection').bounds('selection');
+      }).bind('keyup.sendkeys', function(evt){
+        // restore the selection if we got here with a tab (a click should select what was clicked on)
+        if (evt.which == 9){
+          // there's a flash of selection when we restore the focus, but I don't know how to avoid that
+          $.data(this, 'sendkeys.selection').select();
+        }else{
+          $.data(this, 'sendkeys.selection').bounds('selection');
+        }
+      });
+    }
+    this.focus();
+    if (typeof x === 'undefined') return; // no string, so we just set up the event handlers
+    $.data(this, 'sendkeys.originalText', rng.text());
+    x.replace(/\n/g, '{enter}'). // turn line feeds into explicit break insertions
+      replace(/{[^}]*}|[^{]+/g, function(s){
+        (localkeys[s] || $.fn.sendkeys.defaults[s] || $.fn.sendkeys.defaults.simplechar)(rng, s);
+      });
+    $(this).trigger({type: 'sendkeys', which: x});
+  });
+}; // sendkeys
+
+
+// add the functions publicly so they can be overridden
+$.fn.sendkeys.defaults = {
+  simplechar: function (rng, s){
+    rng.text(s, 'end');
+    for (var i =0; i < s.length; ++i){
+      var x = s.charCodeAt(i);
+      // a bit of cheating: rng._el is the element associated with rng.
+      $(rng._el).trigger({type: 'keypress', keyCode: x, which: x, charCode: x});
+    }
+  },
+  '{{}': function (rng){
+    $.fn.sendkeys.defaults.simplechar (rng, '{')
+  },
+  '{enter}': function (rng){
+    rng.insertEOL();
+    rng.select();
+    var x = '\n'.charCodeAt(0);
+    $(rng._el).trigger({type: 'keypress', keyCode: x, which: x, charCode: x});
+  },
+  '{backspace}': function (rng){
+    var b = rng.bounds();
+    if (b[0] == b[1]) rng.bounds([b[0]-1, b[0]]); // no characters selected; it's just an insertion point. Remove the previous character
+    rng.text('', 'end'); // delete the characters and update the selection
+  },
+  '{del}': function (rng){
+    var b = rng.bounds();
+    if (b[0] == b[1]) rng.bounds([b[0], b[0]+1]); // no characters selected; it's just an insertion point. Remove the next character
+    rng.text('', 'end'); // delete the characters and update the selection
+  },
+  '{rightarrow}':  function (rng){
+    var b = rng.bounds();
+    if (b[0] == b[1]) ++b[1]; // no characters selected; it's just an insertion point. Move to the right
+    rng.bounds([b[1], b[1]]).select();
+  },
+  '{leftarrow}': function (rng){
+    var b = rng.bounds();
+    if (b[0] == b[1]) --b[0]; // no characters selected; it's just an insertion point. Move to the left
+    rng.bounds([b[0], b[0]]).select();
+  },
+  '{selectall}' : function (rng){
+    rng.bounds('all').select();
+  },
+  '{selection}': function (rng){
+    $.fn.sendkeys.defaults.simplechar(rng, $.data(rng._el, 'sendkeys.originalText'));
+  },
+  '{mark}' : function (rng){
+    var bounds = rng.bounds();
+    $(rng._el).one('sendkeys', function(){
+      // set up the event listener to change the selection after the sendkeys is done
+      rng.bounds(bounds).select();
+    });
+  }
+};
+
+})(jQuery)

src/tests/frontend/runner.css

@@ -0,0 +1,246 @@
+html {
+  height: 100%;
+}
+
+body {
+  padding: 0px;
+  margin: 0px;
+  height: 100%;
+  display: flex;
+  flex-direction: row;
+  overflow: hidden;
+}
+
+#console {
+  display: none;
+}
+
+#iframe-container {
+  width: 80%;
+  min-width: 820px;
+  height: 100%;
+}
+
+#iframe-container iframe {
+  height: 100%;
+  width:100%;
+}
+
+#mocha {
+  font: 20px/1.5 "Helvetica Neue", Helvetica, Arial, sans-serif;
+  border-right: 2px solid #999;
+  flex: 1 auto;
+  height: 100%;
+  overflow: auto;
+  width:20%;
+  font-size:80%;
+  
+}
+
+#mocha #report {
+  margin: 0;
+  padding: 0;
+  margin-top: 10px;
+}
+
+#mocha li {
+  margin: 0;
+  padding: 0;
+}
+
+#mocha ul {
+  list-style: none;
+}
+
+#mocha h1, #mocha h2 {
+  margin: 0;
+}
+
+#mocha h1 {
+  margin-top: 15px;
+  font-size: 1em;
+  font-weight: 200;
+}
+
+#mocha h1 a:visited
+{
+  color: #00E;
+}
+
+#mocha .suite .suite h1 {
+  margin-top: 0;
+  font-size: .8em;
+}
+
+#mocha h2 {
+  font-size: 12px;
+  font-weight: normal;
+  cursor: pointer;
+}
+
+#mocha .suite {
+  margin-left: 0px;
+}
+
+#mocha .test {
+  margin-left: 5px;
+}
+
+#mocha .test:hover h2::after {
+  position: relative;
+  top: 0;
+  right: -10px;
+  content: '(view source)';
+  font-size: 12px;
+  font-family: arial;
+  color: #888;
+}
+
+#mocha .test.pending:hover h2::after {
+  content: '(pending)';
+  font-family: arial;
+}
+
+#mocha .test.pass.medium .duration {
+  background: #C09853;
+}
+
+#mocha .test.pass.slow .duration {
+  background: #B94A48;
+}
+
+#mocha .test.pass::before {
+  content: '✓';
+  font-size: 12px;
+  display: block;
+  float: left;
+  margin-right: 5px;
+  color: #00d6b2;
+}
+
+#mocha .test.pass .duration {
+  font-size: 9px;
+  margin-left: 5px;
+  padding: 2px 5px;
+  color: white;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.2);
+  -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.2);
+  box-shadow: inset 0 1px 1px rgba(0,0,0,.2);
+  -webkit-border-radius: 5px;
+  -moz-border-radius: 5px;
+  -ms-border-radius: 5px;
+  -o-border-radius: 5px;
+  border-radius: 5px;
+}
+
+#mocha .test.pass.fast .duration {
+  display: none;
+}
+
+#mocha .test.pending {
+  color: #0b97c4;
+}
+
+#mocha .test.pending::before {
+  content: '◦';
+  color: #0b97c4;
+}
+
+#mocha .test.fail {
+  color: #c00;
+}
+
+#mocha .test.fail pre {
+  color: black;
+}
+
+#mocha .test.fail::before {
+  content: '✖';
+  font-size: 12px;
+  display: block;
+  float: left;
+  margin-right: 5px;
+  color: #c00;
+}
+
+#mocha .test pre.error {
+  color: #c00;
+}
+
+#mocha .test pre {
+  display: inline-block;
+  font: 12px/1.5 monaco, monospace;
+  margin: 5px;
+  padding: 15px;
+  border: 1px solid #eee;
+  border-bottom-color: #ddd;
+  -webkit-border-radius: 3px;
+  -webkit-box-shadow: 0 1px 3px #eee;
+}
+
+#report ul {
+  padding: 0;
+}
+
+#report.pass .test.fail {
+  display: none;
+}
+
+#report.fail .test.pass {
+  display: none;
+}
+
+#error {
+  color: #c00;
+  font-size: 1.5  em;
+  font-weight: 100;
+  letter-spacing: 1px;
+}
+
+#stats {
+  padding: 10px;
+  font-size: 12px;
+  margin: 0;
+  color: #888;
+  text-align: right;
+}
+
+#mocha-stats {
+  height: 80px;
+}
+
+#mocha-stats .progress {
+  float: right;
+  padding-top: 0;
+  margin-right:5px;
+}
+
+#stats em {
+  color: black;
+}
+
+#stats a {
+  text-decoration: none;
+  color: inherit;
+}
+
+#stats a:hover {
+  border-bottom: 1px solid #eee;
+}
+
+#stats li {
+  display: inline-block;
+  margin: 0 5px;
+  list-style: none;
+  padding-top: 11px;
+}
+
+code .comment { color: #ddd }
+code .init { color: #2F6FAD }
+code .string { color: #5890AD }
+code .keyword { color: #8A6343 }
+code .number { color: #2F6FAD }
+
+ul{
+  padding-left:5px;
+}

src/tests/frontend/runner.js

@@ -0,0 +1,184 @@
+'use strict';
+
+/* global specs_list */
+
+$(() => {
+  const stringifyException = (exception) => {
+    let err = exception.stack || exception.toString();
+
+    // FF / Opera do not add the message
+    if (!~err.indexOf(exception.message)) {
+      err = `${exception.message}\n${err}`;
+    }
+
+    // <=IE7 stringifies to [Object Error]. Since it can be overloaded, we
+    // check for the result of the stringifying.
+    if (err === '[object Error]') err = exception.message;
+
+    // Safari doesn't give you a stack. Let's at least provide a source line.
+    if (!exception.stack && exception.sourceURL && exception.line !== undefined) {
+      err += `\n(${exception.sourceURL}:${exception.line})`;
+    }
+
+    return err;
+  };
+
+  const customRunner = (runner) => {
+    const stats = {suites: 0, tests: 0, passes: 0, pending: 0, failures: 0};
+    let level = 0;
+
+    if (!runner) return;
+
+    runner.on('start', () => {
+      stats.start = new Date();
+    });
+
+    runner.on('suite', (suite) => {
+      suite.root || stats.suites++;
+      if (suite.root) return;
+      append(suite.title);
+      level++;
+    });
+
+    runner.on('suite end', (suite) => {
+      if (suite.root) return;
+      level--;
+
+      if (level === 0) {
+        append('');
+      }
+    });
+
+    // Scroll down test display after each test
+    const mochaEl = $('#mocha')[0];
+    runner.on('test', () => {
+      mochaEl.scrollTop = mochaEl.scrollHeight;
+    });
+
+    // max time a test is allowed to run
+    // TODO this should be lowered once timeslider_revision.js is faster
+    let killTimeout;
+    runner.on('test end', () => {
+      stats.tests++;
+    });
+
+    runner.on('pass', (test) => {
+      if (killTimeout) clearTimeout(killTimeout);
+      killTimeout = setTimeout(() => {
+        append('FINISHED - [red]no test started since 3 minutes, tests stopped[clear]');
+      }, 60000 * 3);
+
+      const medium = test.slow() / 2;
+      test.speed = test.duration > test.slow()
+        ? 'slow'
+        : test.duration > medium
+          ? 'medium'
+          : 'fast';
+
+      stats.passes++;
+      append(`-> [green]PASSED[clear] : ${test.title}   ${test.duration} ms`);
+    });
+
+    runner.on('fail', (test, err) => {
+      if (killTimeout) clearTimeout(killTimeout);
+      killTimeout = setTimeout(() => {
+        append('FINISHED - [red]no test started since 3 minutes, tests stopped[clear]');
+      }, 60000 * 3);
+
+      stats.failures++;
+      test.err = err;
+      append(`-> [red]FAILED[clear] : ${test.title} ${stringifyException(test.err)}`);
+    });
+
+    runner.on('pending', (test) => {
+      if (killTimeout) clearTimeout(killTimeout);
+      killTimeout = setTimeout(() => {
+        append('FINISHED - [red]no test started since 3 minutes, tests stopped[clear]');
+      }, 60000 * 3);
+
+      stats.pending++;
+      append(`-> [yellow]PENDING[clear]: ${test.title}`);
+    });
+
+    const $console = $('#console');
+    const append = (text) => {
+      const oldText = $console.text();
+
+      let space = '';
+      for (let i = 0; i < level * 2; i++) {
+        space += ' ';
+      }
+
+      let splitedText = '';
+      _(text.split('\n')).each((line) => {
+        while (line.length > 0) {
+          const split = line.substr(0, 100);
+          line = line.substr(100);
+          if (splitedText.length > 0) splitedText += '\n';
+          splitedText += split;
+        }
+      });
+
+      // indent all lines with the given amount of space
+      const newText = _(splitedText.split('\n')).map((line) => space + line).join('\\n');
+
+      $console.text(`${oldText + newText}\\n`);
+    };
+
+    const total = runner.total;
+    runner.on('end', () => {
+      stats.end = new Date();
+      stats.duration = stats.end - stats.start;
+      const minutes = Math.floor(stats.duration / 1000 / 60);
+      // chrome < 57 does not like this .toString().padStart('2', '0');
+      const seconds = Math.round((stats.duration / 1000) % 60);
+      if (stats.tests === total) {
+        append(`FINISHED - ${stats.passes} tests passed, ${stats.failures} tests failed, ` +
+               `${stats.pending} pending, duration: ${minutes}:${seconds}`);
+      } else if (stats.tests > total) {
+        append(`FINISHED - but more tests than planned returned ${stats.passes} tests passed, ` +
+               `${stats.failures} tests failed, ${stats.pending} pending, ` +
+               `duration: ${minutes}:${seconds}`);
+        append(`${total} tests, but ${stats.tests} returned. ` +
+               'There is probably a problem with your async code or error handling, ' +
+               'see https://github.com/mochajs/mocha/issues/1327');
+      } else {
+        append(`FINISHED - but not all tests returned ${stats.passes} tests passed, ` +
+               `${stats.failures} tests failed, ${stats.pending} tests pending, ` +
+               `duration: ${minutes}:${seconds}`);
+        append(`${total} tests, but only ${stats.tests} returned. ` +
+               'Check for failed before/beforeEach-hooks (no `test end` is called for them ' +
+               'and subsequent tests of the same suite are skipped), ' +
+               'see https://github.com/mochajs/mocha/pull/1043');
+      }
+    });
+  };
+
+  const getURLParameter = (name) => (new URLSearchParams(location.search)).get(name);
+
+  // get the list of specs and filter it if requested
+  const specs = specs_list.slice();
+
+  // inject spec scripts into the dom
+  const $body = $('body');
+  $.each(specs, (i, spec) => {
+    // if the spec isn't a plugin spec which means the spec file might be in a different subfolder
+    if (!spec.startsWith('/')) {
+      $body.append(`<script src="specs/${spec}"></script>`);
+    } else {
+      $body.append(`<script src="${spec}"></script>`);
+    }
+  });
+
+  // initialize the test helper
+  helper.init(() => {
+    // configure and start the test framework
+    const grep = getURLParameter('grep');
+    if (grep != null) {
+      mocha.grep(grep);
+    }
+
+    const runner = mocha.run();
+    customRunner(runner);
+  });
+});

src/tests/frontend/specs/alphabet.js

@@ -0,0 +1,25 @@
+'use strict';
+
+describe('All the alphabet works n stuff', function () {
+  const expectedString = 'abcdefghijklmnopqrstuvwxyz';
+
+  // create a new pad before each test run
+  beforeEach(function (cb) {
+    helper.newPad(cb);
+    this.timeout(60000);
+  });
+
+  it('when you enter any char it appears right', function (done) {
+    const inner$ = helper.padInner$;
+
+    // get the first text element out of the inner iframe
+    const firstTextElement = inner$('div').first();
+
+    // simulate key presses to delete content
+    firstTextElement.sendkeys('{selectall}'); // select all
+    firstTextElement.sendkeys('{del}'); // clear the first line
+    firstTextElement.sendkeys(expectedString); // insert the string
+
+    helper.waitFor(() => inner$('div').first().text() === expectedString, 2000).done(done);
+  });
+});

src/tests/frontend/specs/authorship_of_editions.js

@@ -0,0 +1,107 @@
+'use strict';
+
+describe('author of pad edition', function () {
+  const REGULAR_LINE = 0;
+  const LINE_WITH_ORDERED_LIST = 1;
+  const LINE_WITH_UNORDERED_LIST = 2;
+
+  // author 1 creates a new pad with some content (regular lines and lists)
+  before(function (done) {
+    const padId = helper.newPad(() => {
+      // make sure pad has at least 3 lines
+      const $firstLine = helper.padInner$('div').first();
+      const threeLines = ['regular line', 'line with ordered list', 'line with unordered list']
+          .join('<br>');
+      $firstLine.html(threeLines);
+
+      // wait for lines to be processed by Etherpad
+      helper.waitFor(() => {
+        const $lineWithUnorderedList = getLine(LINE_WITH_UNORDERED_LIST);
+        return $lineWithUnorderedList.text() === 'line with unordered list';
+      }).done(() => {
+        // create the unordered list
+        const $lineWithUnorderedList = getLine(LINE_WITH_UNORDERED_LIST);
+        $lineWithUnorderedList.sendkeys('{selectall}');
+
+        const $insertUnorderedListButton = helper.padChrome$('.buttonicon-insertunorderedlist');
+        $insertUnorderedListButton.click();
+
+        helper.waitFor(() => {
+          const $lineWithUnorderedList = getLine(LINE_WITH_UNORDERED_LIST);
+          return $lineWithUnorderedList.find('ul li').length === 1;
+        }).done(() => {
+          // create the ordered list
+          const $lineWithOrderedList = getLine(LINE_WITH_ORDERED_LIST);
+          $lineWithOrderedList.sendkeys('{selectall}');
+
+          const $insertOrderedListButton = helper.padChrome$('.buttonicon-insertorderedlist');
+          $insertOrderedListButton.click();
+
+          helper.waitFor(() => {
+            const $lineWithOrderedList = getLine(LINE_WITH_ORDERED_LIST);
+            return $lineWithOrderedList.find('ol li').length === 1;
+          }).done(() => {
+            // Reload pad, to make changes as a second user. Need a timeout here to make sure
+            // all changes were saved before reloading
+            setTimeout(() => {
+              // Expire cookie, so author is changed after reloading the pad.
+              // See https://developer.mozilla.org/en-US/docs/Web/API/Document/cookie#Example_4_Reset_the_previous_cookie
+              helper.padChrome$.document.cookie =
+                  'token=foo;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/';
+
+              helper.newPad(done, padId);
+            }, 1000);
+          });
+        });
+      });
+    });
+    this.timeout(60000);
+  });
+
+  // author 2 makes some changes on the pad
+  it('marks only the new content as changes of the second user on a regular line', function (done) {
+    changeLineAndCheckOnlyThatChangeIsFromThisAuthor(REGULAR_LINE, 'x', done);
+  });
+
+  it('marks only the new content as changes of the second user on a ' +
+      'line with ordered list', function (done) {
+    changeLineAndCheckOnlyThatChangeIsFromThisAuthor(LINE_WITH_ORDERED_LIST, 'y', done);
+  });
+
+  it('marks only the new content as changes of the second user on ' +
+      'a line with unordered list', function (done) {
+    changeLineAndCheckOnlyThatChangeIsFromThisAuthor(LINE_WITH_UNORDERED_LIST, 'z', done);
+  });
+
+  /* ********************** Helper functions ************************ */
+  const getLine = (lineNumber) => helper.padInner$('div').eq(lineNumber);
+
+  const getAuthorFromClassList = (classes) => classes.find((cls) => cls.startsWith('author'));
+
+  const changeLineAndCheckOnlyThatChangeIsFromThisAuthor = (lineNumber, textChange, done) => {
+    // get original author class
+    const classes = getLine(lineNumber).find('span').first().attr('class').split(' ');
+    const originalAuthor = getAuthorFromClassList(classes);
+
+    // make change on target line
+    const $regularLine = getLine(lineNumber);
+    helper.selectLines($regularLine, $regularLine, 2, 2); // place caret after 2nd char of line
+    $regularLine.sendkeys(textChange);
+
+    // wait for change to be processed by Etherpad
+    let otherAuthorsOfLine;
+    helper.waitFor(() => {
+      const authorsOfLine = getLine(lineNumber).find('span').map(function () {
+        return getAuthorFromClassList($(this).attr('class').split(' '));
+      }).get();
+      otherAuthorsOfLine = authorsOfLine.filter((author) => author !== originalAuthor);
+      const lineHasChangeOfThisAuthor = otherAuthorsOfLine.length > 0;
+      return lineHasChangeOfThisAuthor;
+    }).done(() => {
+      const thisAuthor = otherAuthorsOfLine[0];
+      const $changeOfThisAuthor = getLine(lineNumber).find(`span.${thisAuthor}`);
+      expect($changeOfThisAuthor.text()).to.be(textChange);
+      done();
+    });
+  };
+});

src/tests/frontend/specs/bold.js

@@ -0,0 +1,65 @@
+'use strict';
+
+describe('bold button', function () {
+  // create a new pad before each test run
+  beforeEach(function (cb) {
+    helper.newPad(cb);
+    this.timeout(60000);
+  });
+
+  it('makes text bold on click', function (done) {
+    const inner$ = helper.padInner$;
+    const chrome$ = helper.padChrome$;
+
+    // get the first text element out of the inner iframe
+    const $firstTextElement = inner$('div').first();
+
+    // select this text element
+    $firstTextElement.sendkeys('{selectall}');
+
+    // get the bold button and click it
+    const $boldButton = chrome$('.buttonicon-bold');
+    $boldButton.click();
+
+    const $newFirstTextElement = inner$('div').first();
+
+    // is there a <b> element now?
+    const isBold = $newFirstTextElement.find('b').length === 1;
+
+    // expect it to be bold
+    expect(isBold).to.be(true);
+
+    // make sure the text hasn't changed
+    expect($newFirstTextElement.text()).to.eql($firstTextElement.text());
+
+    done();
+  });
+
+  it('makes text bold on keypress', function (done) {
+    const inner$ = helper.padInner$;
+
+    // get the first text element out of the inner iframe
+    const $firstTextElement = inner$('div').first();
+
+    // select this text element
+    $firstTextElement.sendkeys('{selectall}');
+
+    const e = new inner$.Event(helper.evtType);
+    e.ctrlKey = true; // Control key
+    e.which = 66; // b
+    inner$('#innerdocbody').trigger(e);
+
+    const $newFirstTextElement = inner$('div').first();
+
+    // is there a <b> element now?
+    const isBold = $newFirstTextElement.find('b').length === 1;
+
+    // expect it to be bold
+    expect(isBold).to.be(true);
+
+    // make sure the text hasn't changed
+    expect($newFirstTextElement.text()).to.eql($firstTextElement.text());
+
+    done();
+  });
+});

src/tests/frontend/specs/caret.js

@@ -0,0 +1,352 @@
+'use strict';
+
+describe('As the caret is moved is the UI properly updated?', function () {
+  /*
+  let padName;
+  const numberOfRows = 50;
+
+  //create a new pad before each test run
+  beforeEach(function(cb){
+    helper.newPad(cb);
+    this.timeout(60000);
+  });
+
+  xit("creates a pad", function(done) {
+    padName = helper.newPad(done);
+    this.timeout(60000);
+  });
+*/
+
+  /* Tests to do
+  * Keystroke up (38), down (40), left (37), right (39)
+  * with and without special keys IE control / shift
+  * Page up (33) / down (34) with and without special keys
+  * Page up on the first line shouldn't move the viewport
+  * Down down on the last line shouldn't move the viewport
+  * Down arrow on any other line except the last lines shouldn't move the viewport
+  * Do all of the above tests after a copy/paste event
+  */
+
+  /* Challenges
+  * How do we keep the authors focus on a line if the lines above the author are modified?
+  * We should only redraw the user to a location if they are typing and make sure shift
+  * and arrow keys aren't redrawing the UI else highlight - copy/paste would get broken
+  * How can we simulate an edit event in the test framework?
+  */
+  /*
+  // THIS DOESNT WORK IN CHROME AS IT DOESNT MOVE THE CURSOR!
+  it("down arrow", function(done){
+    var inner$ = helper.padInner$;
+    var chrome$ = helper.padChrome$;
+
+    var $newFirstTextElement = inner$("div").first();
+    $newFirstTextElement.focus();
+    keyEvent(inner$, 37, false, false); // arrow down
+    keyEvent(inner$, 37, false, false); // arrow down
+
+    done();
+  });
+
+  it("Creates N lines", function(done){
+    var inner$ = helper.padInner$;
+console.log(inner$);
+    var chrome$ = helper.padChrome$;
+    var $newFirstTextElement = inner$("div").first();
+
+    prepareDocument(numberOfRows, $newFirstTextElement); // N lines into the first div as a target
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      return inner$("div").first().text().length == 6;
+    }).done(function(){ // Once the DOM has registered the items
+      done();
+    });
+  });
+
+  it("Moves caret up a line", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.y;
+    var newCaretPos;
+    keyEvent(inner$, 38, false, false); // arrow up
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.y;
+      return (newCaretPos < originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.lessThan(originalPos);
+      done();
+    });
+  });
+
+  it("Moves caret down a line", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.y;
+    var newCaretPos;
+    keyEvent(inner$, 40, false, false); // arrow down
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.y;
+      return (newCaretPos > originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.moreThan(originalPos);
+      done();
+    });
+  });
+
+  it("Moves caret to top of doc", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.y;
+    var newCaretPos;
+
+    var i = 0;
+    while(i < numberOfRows){ // press pageup key N times
+      keyEvent(inner$, 33, false, false);
+      i++;
+    }
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.y;
+      return (newCaretPos < originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.lessThan(originalPos);
+      done();
+    });
+  });
+
+  it("Moves caret right a position", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.x;
+    var newCaretPos;
+    keyEvent(inner$, 39, false, false); // arrow right
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.x;
+      return (newCaretPos > originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.moreThan(originalPos);
+      done();
+    });
+  });
+
+  it("Moves caret left a position", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.x;
+    var newCaretPos;
+    keyEvent(inner$, 33, false, false); // arrow left
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.x;
+      return (newCaretPos < originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.lessThan(originalPos);
+      done();
+    });
+  });
+
+  it("Moves caret to the next line using right arrow", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.y;
+    var newCaretPos;
+    keyEvent(inner$, 39, false, false); // arrow right
+    keyEvent(inner$, 39, false, false); // arrow right
+    keyEvent(inner$, 39, false, false); // arrow right
+    keyEvent(inner$, 39, false, false); // arrow right
+    keyEvent(inner$, 39, false, false); // arrow right
+    keyEvent(inner$, 39, false, false); // arrow right
+    keyEvent(inner$, 39, false, false); // arrow right
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.y;
+      return (newCaretPos > originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.moreThan(originalPos);
+      done();
+    });
+  });
+
+  it("Moves caret to the previous line using left arrow", function(done){
+    var inner$ = helper.padInner$;
+    var $newFirstTextElement = inner$("div").first();
+    var originalCaretPosition = caretPosition(inner$);
+    var originalPos = originalCaretPosition.y;
+    var newCaretPos;
+    keyEvent(inner$, 33, false, false); // arrow left
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      var newCaretPosition = caretPosition(inner$);
+      newCaretPos = newCaretPosition.y;
+      return (newCaretPos < originalPos);
+    }).done(function(){
+      expect(newCaretPos).to.be.lessThan(originalPos);
+      done();
+    });
+  });
+
+
+/*
+  it("Creates N rows, changes height of rows, updates UI by caret key events", function(done){
+    var inner$ = helper.padInner$;
+    var chrome$ = helper.padChrome$;
+    var numberOfRows = 50;
+
+    // ace creates a new dom element when you press a keystroke,
+    // so just get the first text element again
+    var $newFirstTextElement = inner$("div").first();
+    var originalDivHeight = inner$("div").first().css("height");
+    prepareDocument(numberOfRows, $newFirstTextElement); // N lines into the first div as a target
+
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      return inner$("div").first().text().length == 6;
+    }).done(function(){ // Once the DOM has registered the items
+      // Randomize the item heights (replicates images / headings etc)
+      inner$("div").each(function(index){
+        var random = Math.floor(Math.random() * (50)) + 20;
+        $(this).css("height", random+"px");
+      });
+
+      console.log(caretPosition(inner$));
+      var newDivHeight = inner$("div").first().css("height");
+      // has the new div height changed from the original div height
+      var heightHasChanged = originalDivHeight != newDivHeight;
+      expect(heightHasChanged).to.be(true); // expect the first line to be blank
+    });
+
+    // Is this Element now visible to the pad user?
+    helper.waitFor(function(){ // Wait for the DOM to register the new items
+      // Wait for the DOM to scroll into place
+      return isScrolledIntoView(inner$("div:nth-child("+numberOfRows+")"), inner$);
+    }).done(function(){ // Once the DOM has registered the items
+      // Randomize the item heights (replicates images / headings etc)
+      inner$("div").each(function(index){
+        var random = Math.floor(Math.random() * (80 - 20 + 1)) + 20;
+        $(this).css("height", random+"px");
+      });
+
+      var newDivHeight = inner$("div").first().css("height");
+      // has the new div height changed from the original div height
+      var heightHasChanged = originalDivHeight != newDivHeight;
+      expect(heightHasChanged).to.be(true); // expect the first line to be blank
+    });
+    var i = 0;
+    while(i < numberOfRows){ // press down arrow
+      keyEvent(inner$, 40, false, false);
+      i++;
+    }
+
+    // Does scrolling back up the pad with the up arrow show the correct contents?
+    helper.waitFor(function(){ // Wait for the new position to be in place
+      try{
+        // Wait for the DOM to scroll into place
+        return isScrolledIntoView(inner$("div:nth-child("+numberOfRows+")"), inner$);
+      }catch(e){
+        return false;
+      }
+    }).done(function(){ // Once the DOM has registered the items
+
+      var i = 0;
+      while(i < numberOfRows){ // press down arrow
+        keyEvent(inner$, 33, false, false); // doesn't work
+        i++;
+      }
+
+      // Does scrolling back up the pad with the up arrow show the correct contents?
+      helper.waitFor(function(){ // Wait for the new position to be in place
+        try{
+          // Wait for the DOM to scroll into place
+          return isScrolledIntoView(inner$("div:nth-child(0)"), inner$);
+        }catch(e){
+          return false;
+        }
+      }).done(function(){ // Once the DOM has registered the items
+
+
+      });
+   });
+
+
+     var i = 0;
+      while(i < numberOfRows){ // press down arrow
+        keyEvent(inner$, 33, false, false); // doesn't work
+        i++;
+      }
+
+
+    // Does scrolling back up the pad with the up arrow show the correct contents?
+    helper.waitFor(function(){ // Wait for the new position to be in place
+      // Wait for the DOM to scroll into place
+      return isScrolledIntoView(inner$("div:nth-child(1)"), inner$);
+    }).done(function(){ // Once the DOM has registered the items
+      expect(true).to.be(true);
+      done();
+    });
+*/
+});
+
+// generates a random document with random content on n lines
+const prepareDocument = (n, target) => {
+  let i = 0;
+  while (i < n) { // for each line
+    target.sendkeys(makeStr()); // generate a random string and send that to the editor
+    target.sendkeys('{enter}'); // generator an enter keypress
+    i++; // rinse n times
+  }
+};
+
+// sends a charCode to the window
+const keyEvent = (target, charCode, ctrl, shift) => {
+  const e = new target.Event(helper.evtType);
+  if (ctrl) {
+    e.ctrlKey = true; // Control key
+  }
+  if (shift) {
+    e.shiftKey = true; // Shift Key
+  }
+  e.which = charCode;
+  e.keyCode = charCode;
+  target('#innerdocbody').trigger(e);
+};
+
+
+// from http://stackoverflow.com/questions/1349404/generate-a-string-of-5-random-characters-in-javascript
+const makeStr = () => {
+  let text = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 5; i++) text += possible.charAt(Math.floor(Math.random() * possible.length));
+  return text;
+};
+
+// from http://stackoverflow.com/questions/487073/check-if-element-is-visible-after-scrolling
+const isScrolledIntoView = (elem, $) => {
+  const docViewTop = $(window).scrollTop();
+  const docViewBottom = docViewTop + $(window).height();
+  const elemTop = $(elem).offset().top; // how far the element is from the top of it's container
+  // how far plus the height of the elem..  IE is it all in?
+  let elemBottom = elemTop + $(elem).height();
+  elemBottom -= 16; // don't ask, sorry but this is needed..
+  return ((elemBottom <= docViewBottom) && (elemTop >= docViewTop));
+};
+
+const caretPosition = ($) => {
+  const doc = $.window.document;
+  const pos = doc.getSelection();
+  pos.y = pos.anchorNode.parentElement.offsetTop;
+  pos.x = pos.anchorNode.parentElement.offsetLeft;
+  return pos;
+};

src/tests/frontend/specs/change_user_color.js

@@ -0,0 +1,107 @@
+'use strict';
+
+describe('change user color', function () {
+  // create a new pad before each test run
+  beforeEach(function (cb) {
+    helper.newPad(cb);
+    this.timeout(60000);
+  });
+
+  it('Color picker matches original color and remembers the user color' +
+      ' after a refresh', function (done) {
+    this.timeout(60000);
+    const chrome$ = helper.padChrome$;
+
+    // click on the settings button to make settings visible
+    const $userButton = chrome$('.buttonicon-showusers');
+    $userButton.click();
+
+    const $userSwatch = chrome$('#myswatch');
+    $userSwatch.click();
+
+    const fb = chrome$.farbtastic('#colorpicker');
+    const $colorPickerSave = chrome$('#mycolorpickersave');
+    const $colorPickerPreview = chrome$('#mycolorpickerpreview');
+
+    // Same color represented in two different ways
+    const testColorHash = '#abcdef';
+    const testColorRGB = 'rgb(171, 205, 239)';
+
+    // Check that the color picker matches the automatically assigned random color on the swatch.
+    // NOTE: This has a tiny chance of creating a false positive for passing in the
+    // off-chance the randomly assigned color is the same as the test color.
+    expect($colorPickerPreview.css('background-color')).to.be($userSwatch.css('background-color'));
+
+    // The swatch updates as the test color is picked.
+    fb.setColor(testColorHash);
+    expect($colorPickerPreview.css('background-color')).to.be(testColorRGB);
+    $colorPickerSave.click();
+    expect($userSwatch.css('background-color')).to.be(testColorRGB);
+
+    setTimeout(() => { // give it a second to save the color on the server side
+      helper.newPad({ // get a new pad, but don't clear the cookies
+        clearCookies: false,
+        cb() {
+          const chrome$ = helper.padChrome$;
+
+          // click on the settings button to make settings visible
+          const $userButton = chrome$('.buttonicon-showusers');
+          $userButton.click();
+
+          const $userSwatch = chrome$('#myswatch');
+          $userSwatch.click();
+
+          const $colorPickerPreview = chrome$('#mycolorpickerpreview');
+
+          expect($colorPickerPreview.css('background-color')).to.be(testColorRGB);
+          expect($userSwatch.css('background-color')).to.be(testColorRGB);
+
+          done();
+        },
+      });
+    }, 1000);
+  });
+
+  it('Own user color is shown when you enter a chat', function (done) {
+    const chrome$ = helper.padChrome$;
+
+    const $colorOption = helper.padChrome$('#options-colorscheck');
+    if (!$colorOption.is(':checked')) {
+      $colorOption.click();
+    }
+
+    // click on the settings button to make settings visible
+    const $userButton = chrome$('.buttonicon-showusers');
+    $userButton.click();
+
+    const $userSwatch = chrome$('#myswatch');
+    $userSwatch.click();
+
+    const fb = chrome$.farbtastic('#colorpicker');
+    const $colorPickerSave = chrome$('#mycolorpickersave');
+
+    // Same color represented in two different ways
+    const testColorHash = '#abcdef';
+    const testColorRGB = 'rgb(171, 205, 239)';
+
+    fb.setColor(testColorHash);
+    $colorPickerSave.click();
+
+    // click on the chat button to make chat visible
+    const $chatButton = chrome$('#chaticon');
+    $chatButton.click();
+    const $chatInput = chrome$('#chatinput');
+    $chatInput.sendkeys('O hi'); // simulate a keypress of typing user
+    // simulate a keypress of enter actually does evt.which = 10 not 13
+    $chatInput.sendkeys('{enter}');
+
+    // wait until the chat message shows up
+    helper.waitFor(() => chrome$('#chattext').children('p').length !== 0
+    ).done(() => {
+      const $firstChatMessage = chrome$('#chattext').children('p');
+      // expect the first chat message to be of the user's color
+      expect($firstChatMessage.css('background-color')).to.be(testColorRGB);
+      done();
+    });
+  });
+});

src/tests/frontend/specs/change_user_name.js

@@ -0,0 +1,36 @@
+'use strict';
+
+describe('change username value', function () {
+  // create a new pad before each test run
+  beforeEach(function (cb) {
+    helper.newPad(cb);
+  });
+
+  it('Remembers the user name after a refresh', async function () {
+    helper.toggleUserList();
+    helper.setUserName('😃');
+
+    helper.newPad({ // get a new pad, but don't clear the cookies
+      clearCookies: false,
+      cb() {
+        helper.toggleUserList();
+
+        expect(helper.usernameField().val()).to.be('😃');
+      },
+    });
+  });
+
+  it('Own user name is shown when you enter a chat', async function () {
+    helper.toggleUserList();
+    helper.setUserName('😃');
+
+    helper.showChat();
+    helper.sendChatMessage('O hi{enter}');
+
+    await helper.waitForPromise(() => {
+      // username:hours:minutes text
+      const chatText = helper.chatTextParagraphs().text();
+      return chatText.indexOf('😃') === 0;
+    });
+  });
+});

src/tests/frontend/specs/chat.js

@@ -0,0 +1,116 @@
+'use strict';
+
+describe('Chat messages and UI', function () {
+  // create a new pad before each test run
+  beforeEach(function (cb) {
+    helper.newPad(cb);
+  });
+
+  it('opens chat, sends a message, makes sure it exists ' +
+      'on the page and hides chat', async function () {
+    const chatValue = 'JohnMcLear';
+
+    await helper.showChat();
+    await helper.sendChatMessage(`${chatValue}{enter}`);
+
+    expect(helper.chatTextParagraphs().length).to.be(1);
+
+    // <p data-authorid="a.qjkwNs4z0pPROphS"
+    //   class="author-a-qjkwz78zs4z122z0pz80zz82zz79zphz83z">
+    //   <b>unnamed:</b>
+    //   <span class="time author-a-qjkwz78zs4z122z0pz80zz82zz79zphz83z">12:38
+    //   </span> JohnMcLear
+    // </p>
+    const username = helper.chatTextParagraphs().children('b').text();
+    const time = helper.chatTextParagraphs().children('.time').text();
+
+    expect(helper.chatTextParagraphs().text()).to.be(`${username}${time} ${chatValue}`);
+
+    await helper.hideChat();
+  });
+
+  it("makes sure that an empty message can't be sent", async function () {
+    const chatValue = 'mluto';
+
+    await helper.showChat();
+
+    // simulate a keypress of typing enter, mluto and enter (to send 'mluto')
+    await helper.sendChatMessage(`{enter}${chatValue}{enter}`);
+
+    const chat = helper.chatTextParagraphs();
+
+    expect(chat.length).to.be(1);
+
+    // check that the received message is not the empty one
+    const username = chat.children('b').text();
+    const time = chat.children('.time').text();
+
+    expect(chat.text()).to.be(`${username}${time} ${chatValue}`);
+  });
+
+  it('makes chat stick to right side of the screen via settings, ' +
+      'remove sticky via settings, close it', async function () {
+    await helper.showSettings();
+
+    await helper.enableStickyChatviaSettings();
+    expect(helper.isChatboxShown()).to.be(true);
+    expect(helper.isChatboxSticky()).to.be(true);
+
+    await helper.disableStickyChatviaSettings();
+    expect(helper.isChatboxSticky()).to.be(false);
+    expect(helper.isChatboxShown()).to.be(true);
+
+    await helper.hideChat();
+    expect(helper.isChatboxSticky()).to.be(false);
+    expect(helper.isChatboxShown()).to.be(false);
+  });
+
+  it('makes chat stick to right side of the screen via icon on the top' +
+      ' right, remove sticky via icon, close it', async function () {
+    await helper.showChat();
+
+    await helper.enableStickyChatviaIcon();
+    expect(helper.isChatboxShown()).to.be(true);
+    expect(helper.isChatboxSticky()).to.be(true);
+
+    await helper.disableStickyChatviaIcon();
+    expect(helper.isChatboxShown()).to.be(true);
+    expect(helper.isChatboxSticky()).to.be(false);
+
+    await helper.hideChat();
+    expect(helper.isChatboxSticky()).to.be(false);
+    expect(helper.isChatboxShown()).to.be(false);
+  });
+
+  xit('Checks showChat=false URL Parameter hides chat then' +
+      ' when removed it shows chat', function (done) {
+    this.timeout(60000);
+
+    setTimeout(() => { // give it a second to save the username on the server side
+      helper.newPad({ // get a new pad, but don't clear the cookies
+        clearCookies: false,
+        params: {
+          showChat: 'false',
+        }, cb() {
+          const chrome$ = helper.padChrome$;
+          const chaticon = chrome$('#chaticon');
+          // chat should be hidden.
+          expect(chaticon.is(':visible')).to.be(false);
+
+          setTimeout(() => { // give it a second to save the username on the server side
+            helper.newPad({ // get a new pad, but don't clear the cookies
+              clearCookies: false,
+              cb() {
+                const chrome$ = helper.padChrome$;
+                const chaticon = chrome$('#chaticon');
+                // chat should be visible.
+                expect(chaticon.is(':visible')).to.be(true);
+                done();
+              },
+            });
+          }, 1000);
+        },
+      });
+    }, 1000);
+  });
+});