restructure: move bin/ and tests/ to src/

Also add symlinks from the old `bin/` and `tests/` locations to avoid
breaking scripts and other tools.

Motivations:

  * Scripts and tests no longer have to do dubious things like:

        require('ep_etherpad-lite/node_modules/foo')

    to access packages installed as dependencies in
    `src/package.json`.

  * Plugins can access the backend test helper library in a non-hacky
    way:

        require('ep_etherpad-lite/tests/backend/common')

  * We can delete the top-level `package.json` without breaking our
    ability to lint the files in `bin/` and `tests/`.

    Deleting the top-level `package.json` has downsides: It will cause
    `npm` to print warnings whenever plugins are installed, npm will
    no longer be able to enforce a plugin's peer dependency on
    ep_etherpad-lite, and npm will keep deleting the
    `node_modules/ep_etherpad-lite` symlink that points to `../src`.

    But there are significant upsides to deleting the top-level
    `package.json`: It will drastically speed up plugin installation
    because `npm` doesn't have to recursively walk the dependencies in
    `src/package.json`. Also, deleting the top-level `package.json`
    avoids npm's horrible dependency hoisting behavior (where it moves
    stuff from `src/node_modules/` to the top-level `node_modules/`
    directory). Dependency hoisting causes numerous mysterious
    problems such as silent failures in `npm outdated` and `npm
    update`. Dependency hoisting also breaks plugins that do:

        require('ep_etherpad-lite/node_modules/foo')

src/bin/buildDebian.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# IMPORTANT
+# Protect against misspelling a var and rm -rf /
+set -u
+set -e
+
+SRC=/tmp/etherpad-deb-src
+DIST=/tmp/etherpad-deb-dist
+SYSROOT=${SRC}/sysroot
+DEBIAN=${SRC}/DEBIAN
+
+rm -rf ${DIST}
+mkdir -p ${DIST}/
+
+rm -rf ${SRC}
+rsync -a bin/deb-src/ ${SRC}/
+mkdir -p ${SYSROOT}/opt/
+
+rsync --exclude '.git' -a . ${SYSROOT}/opt/etherpad/ --delete
+mkdir -p ${SYSROOT}/usr/share/doc
+cp README.md ${SYSROOT}/usr/share/doc/etherpad
+find ${SRC}/ -type d -exec chmod 0755 {} \;
+find ${SRC}/ -type f -exec chmod go-w {} \;
+chown -R root:root ${SRC}/
+
+let SIZE=$(du -s ${SYSROOT} | sed s'/\s\+.*//')+8
+pushd ${SYSROOT}/
+tar czf ${DIST}/data.tar.gz [a-z]*
+popd
+sed s"/SIZE/${SIZE}/" -i ${DEBIAN}/control
+pushd ${DEBIAN}
+tar czf ${DIST}/control.tar.gz *
+popd
+
+pushd ${DIST}/
+echo 2.0 > ./debian-binary
+
+find ${DIST}/ -type d -exec chmod 0755 {} \;
+find ${DIST}/ -type f -exec chmod go-w {} \;
+chown -R root:root ${DIST}/
+ar r ${DIST}/etherpad-1.deb debian-binary control.tar.gz data.tar.gz
+popd
+rsync -a ${DIST}/etherpad-1.deb ./

src/bin/buildForWindows.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Is wget installed?
+is_cmd wget || fatal "Please install wget"
+
+# Is zip installed?
+is_cmd zip || fatal "Please install zip"
+
+# Is zip installed?
+is_cmd unzip || fatal "Please install unzip"
+
+START_FOLDER=$(pwd);
+TMP_FOLDER=$(mktemp -d)
+
+log "create a clean environment in $TMP_FOLDER..."
+cp -ar . "$TMP_FOLDER"
+cd "$TMP_FOLDER"
+rm -rf node_modules
+rm -f etherpad-lite-win.zip
+
+# setting NODE_ENV=production ensures that dev dependencies are not installed,
+# making the windows package smaller
+export NODE_ENV=production
+
+log "do a normal unix install first..."
+bin/installDeps.sh || exit 1
+
+log "copy the windows settings template..."
+cp settings.json.template settings.json
+
+log "resolve symbolic links..."
+cp -rL node_modules node_modules_resolved
+rm -rf node_modules
+mv node_modules_resolved node_modules
+
+log "download windows node..."
+cd bin
+wget "https://nodejs.org/dist/latest-erbium/win-x86/node.exe" -O ../node.exe
+
+log "remove git history to reduce folder size"
+rm -rf .git/objects
+
+log "remove windows jsdom-nocontextify/test folder"
+rm -rf "$TMP_FOLDER"/src/node_modules/wd/node_modules/request/node_modules/form-data/node_modules/combined-stream/test
+rm -rf "$TMP_FOLDER"/src/node_modules/nodemailer/node_modules/mailcomposer/node_modules/mimelib/node_modules/encoding/node_modules/iconv-lite/encodings/tables
+
+log "create the zip..."
+cd "$TMP_FOLDER"
+zip -9 -r "$START_FOLDER"/etherpad-lite-win.zip ./*
+
+log "clean up..."
+rm -rf "$TMP_FOLDER"
+
+log "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"

src/bin/checkAllPads.js

@@ -0,0 +1,88 @@
+'use strict';
+/*
+ * This is a debug tool. It checks all revisions for data corruption
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js');
+
+(async () => {
+  // initialize the database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load modules
+  const Changeset = require('../static/js/Changeset');
+  const padManager = require('../node/db/PadManager');
+
+  let revTestedCount = 0;
+
+  // get all pads
+  const res = await padManager.listAllPads();
+  for (const padId of res.padIDs) {
+    const pad = await padManager.getPad(padId);
+
+    // check if the pad has a pool
+    if (pad.pool == null) {
+      console.error(`[${pad.id}] Missing attribute pool`);
+      continue;
+    }
+    // create an array with key kevisions
+    // key revisions always save the full pad atext
+    const head = pad.getHeadRevisionNumber();
+    const keyRevisions = [];
+    for (let rev = 0; rev < head; rev += 100) {
+      keyRevisions.push(rev);
+    }
+
+    // run through all key revisions
+    for (const keyRev of keyRevisions) {
+      // create an array of revisions we need till the next keyRevision or the End
+      const revisionsNeeded = [];
+      for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
+        revisionsNeeded.push(rev);
+      }
+
+      // this array will hold all revision changesets
+      const revisions = [];
+
+      // run through all needed revisions and get them from the database
+      for (const revNum of revisionsNeeded) {
+        const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
+        revisions[revNum] = revision;
+      }
+
+      // check if the revision exists
+      if (revisions[keyRev] == null) {
+        console.error(`[${pad.id}] Missing revision ${keyRev}`);
+        continue;
+      }
+
+      // check if there is a atext in the keyRevisions
+      let {meta: {atext} = {}} = revisions[keyRev];
+      if (atext == null) {
+        console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
+        continue;
+      }
+
+      const apool = pad.pool;
+      for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
+        try {
+          const cs = revisions[rev].changeset;
+          atext = Changeset.applyToAText(cs, atext, apool);
+          revTestedCount++;
+        } catch (e) {
+          console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
+        }
+      }
+    }
+  }
+  if (revTestedCount === 0) {
+    throw new Error('No revisions tested');
+  }
+  console.log(`Finished: Tested ${revTestedCount} revisions`);
+})();

src/bin/checkPad.js

@@ -0,0 +1,82 @@
+'use strict';
+/*
+ * This is a debug tool. It checks all revisions for data corruption
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+let checkRevisionCount = 0;
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load modules
+  const Changeset = require('../static/js/Changeset');
+  const padManager = require('../node/db/PadManager');
+
+  const exists = await padManager.doesPadExists(padId);
+  if (!exists) throw new Error('Pad does not exist');
+
+  // get the pad
+  const pad = await padManager.getPad(padId);
+
+  // create an array with key revisions
+  // key revisions always save the full pad atext
+  const head = pad.getHeadRevisionNumber();
+  const keyRevisions = [];
+  for (let rev = 0; rev < head; rev += 100) {
+    keyRevisions.push(rev);
+  }
+
+  // run through all key revisions
+  for (let keyRev of keyRevisions) {
+    keyRev = parseInt(keyRev);
+    // create an array of revisions we need till the next keyRevision or the End
+    const revisionsNeeded = [];
+    for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
+      revisionsNeeded.push(rev);
+    }
+
+    // this array will hold all revision changesets
+    const revisions = [];
+
+    // run through all needed revisions and get them from the database
+    for (const revNum of revisionsNeeded) {
+      const revision = await db.get(`pad:${padId}:revs:${revNum}`);
+      revisions[revNum] = revision;
+    }
+
+    // check if the pad has a pool
+    if (pad.pool == null) throw new Error('Attribute pool is missing');
+
+    // check if there is an atext in the keyRevisions
+    let {meta: {atext} = {}} = revisions[keyRev] || {};
+    if (atext == null) {
+      console.error(`No atext in key revision ${keyRev}`);
+      continue;
+    }
+
+    const apool = pad.pool;
+
+    for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
+      checkRevisionCount++;
+      try {
+        const cs = revisions[rev].changeset;
+        atext = Changeset.applyToAText(cs, atext, apool);
+      } catch (e) {
+        console.error(`Bad changeset at revision ${rev} - ${e.message}`);
+        continue;
+      }
+    }
+    console.log(`Finished: Checked ${checkRevisionCount} revisions`);
+  }
+})();

src/bin/checkPadDeltas.js

@@ -0,0 +1,103 @@
+'use strict';
+/*
+ * This is a debug tool. It checks all revisions for data corruption
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 3) throw new Error('Use: node bin/checkPadDeltas.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+const expect = require('../tests/frontend/lib/expect');
+const diff = require('diff');
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load modules
+  const Changeset = require('../static/js/Changeset');
+  const padManager = require('../node/db/PadManager');
+
+  const exists = await padManager.doesPadExists(padId);
+  if (!exists) throw new Error('Pad does not exist');
+
+  // get the pad
+  const pad = await padManager.getPad(padId);
+
+  // create an array with key revisions
+  // key revisions always save the full pad atext
+  const head = pad.getHeadRevisionNumber();
+  const keyRevisions = [];
+  for (let i = 0; i < head; i += 100) {
+    keyRevisions.push(i);
+  }
+
+  // create an array with all revisions
+  const revisions = [];
+  for (let i = 0; i <= head; i++) {
+    revisions.push(i);
+  }
+
+  let atext = Changeset.makeAText('\n');
+
+  // run through all revisions
+  for (const revNum of revisions) {
+    // console.log('Fetching', revNum)
+    const revision = await db.get(`pad:${padId}:revs:${revNum}`);
+    // check if there is a atext in the keyRevisions
+    const {meta: {atext: revAtext} = {}} = revision || {};
+    if (~keyRevisions.indexOf(revNum) && revAtext == null) {
+      console.error(`No atext in key revision ${revNum}`);
+      continue;
+    }
+
+    // try glue everything together
+    try {
+      // console.log("check revision ", revNum);
+      const cs = revision.changeset;
+      atext = Changeset.applyToAText(cs, atext, pad.pool);
+    } catch (e) {
+      console.error(`Bad changeset at revision ${revNum} - ${e.message}`);
+      continue;
+    }
+
+    // check things are working properly
+    if (~keyRevisions.indexOf(revNum)) {
+      try {
+        expect(revision.meta.atext.text).to.eql(atext.text);
+        expect(revision.meta.atext.attribs).to.eql(atext.attribs);
+      } catch (e) {
+        console.error(`Atext in key revision ${revNum} doesn't match computed one.`);
+        console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => {
+          if (!op.added && !op.removed) op.value = op.value.length;
+          return op;
+        }));
+        // console.error(e)
+        // console.log('KeyRev. :', revision.meta.atext)
+        // console.log('Computed:', atext)
+        continue;
+      }
+    }
+  }
+
+  // check final text is right...
+  if (pad.atext.text === atext.text) {
+    console.log('ok');
+  } else {
+    console.error('Pad AText doesn\'t match computed one! (Computed ',
+        atext.text.length, ', db', pad.atext.text.length, ')');
+    console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => {
+      if (!op.added && !op.removed) {
+        op.value = op.value.length;
+        return op;
+      }
+    }));
+  }
+})();

src/bin/cleanRun.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+#Was this script started in the bin folder? if yes move out
+if [ -d "../bin" ]; then
+  cd "../"
+fi
+
+ignoreRoot=0
+for ARG in "$@"
+do
+  if [ "$ARG" = "--root" ]; then
+    ignoreRoot=1
+  fi
+done
+
+#Stop the script if it's started as root
+if [ "$(id -u)" -eq 0 ] && [ $ignoreRoot -eq 0 ]; then
+   echo "You shouldn't start Etherpad as root!"
+   echo "Please type 'Etherpad rocks my socks' or supply the '--root' argument if you still want to start it as root"
+   read rocks
+   if [ ! $rocks = "Etherpad rocks my socks" ]
+   then
+     echo "Your input was incorrect"
+     exit 1
+   fi
+fi
+
+#Clean the current environment
+rm -rf src/node_modules
+
+#Prepare the environment
+bin/installDeps.sh "$@" || exit 1
+
+#Move to the node folder and start
+echo "Started Etherpad..."
+
+SCRIPTPATH=$(pwd -P)
+node $(compute_node_args) "${SCRIPTPATH}/node_modules/ep_etherpad-lite/node/server.js" "$@"

src/bin/convertSettings.json.template

@@ -0,0 +1,10 @@
+{
+  "etherpadDB":
+  {
+    "host": "localhost",
+    "port": 3306,
+    "database": "etherpad",
+    "user": "etherpaduser",
+    "password": "yourpassword"
+  }
+}

src/bin/createRelease.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+#
+# WARNING: since Etherpad 1.7.0 (2018-08-17), this script is DEPRECATED, and
+#          will be removed/modified in a future version.
+#          It's left here just for documentation.
+#          The branching policies for releases have been changed.
+#
+# This script is used to publish a new release/version of etherpad on github
+#
+# Work that is done by this script:
+# ETHER_REPO:
+# - Add text to CHANGELOG.md
+# - Replace version of etherpad in src/package.json
+# - Create a release branch and push it to github
+# - Merges this release branch into master branch
+# - Creating the windows build and the docs
+# ETHER_WEB_REPO:
+# - Creating a new branch with the docs and the windows build
+# - Replacing the version numbers in the index.html
+# - Push this branch and merge it to master
+# ETHER_REPO:
+# - Create a new release on github
+
+printf "WARNING: since Etherpad 1.7.0 this script is DEPRECATED, and will be removed/modified in a future version.\n\n"
+while true; do
+    read -p "Do you want to continue? This is discouraged. [y/N]" yn
+    case $yn in
+        [Yy]* ) break;;
+        [Nn]* ) exit;;
+        * ) printf "Please answer yes or no.\n\n";;
+    esac
+done
+
+ETHER_REPO="https://github.com/ether/etherpad-lite.git"
+ETHER_WEB_REPO="https://github.com/ether/ether.github.com.git"
+TMP_DIR="/tmp/"
+
+echo "WARNING: You can only run this script if your github api token is allowed to create and merge branches on $ETHER_REPO and $ETHER_WEB_REPO."
+echo "This script automatically changes the version number in package.json and adds a text to CHANGELOG.md."
+echo "When you use this script you should be in the branch that you want to release (develop probably) on latest version. Any changes that are currently not committed will be committed."
+echo "-----"
+
+# Get the latest version
+LATEST_GIT_TAG=$(git tag | tail -n 1)
+
+# Current environment
+echo "Current environment: "
+echo "- branch: $(git branch | grep '* ')"
+echo "- last commit date: $(git show --quiet --pretty=format:%ad)"
+echo "- current version: $LATEST_GIT_TAG"
+echo "- temp dir: $TMP_DIR"
+
+# Get new version number
+# format: x.x.x
+echo -n "Enter new version (x.x.x): "
+read VERSION
+
+# Get the message for the changelogs
+read -p "Enter new changelog entries (press enter): "
+tmp=$(mktemp)
+"${EDITOR:-vi}" $tmp
+changelogText=$(<$tmp)
+echo "$changelogText"
+rm $tmp
+
+if [ "$changelogText" != "" ]; then
+  changelogText="# $VERSION\n$changelogText"
+fi
+
+# get the token for the github api
+echo -n "Enter your github api token: "
+read API_TOKEN
+
+function check_api_token {
+  echo "Checking if github api token is valid..."
+  CURL_RESPONSE=$(curl --silent -i https://api.github.com/user?access_token=$API_TOKEN | iconv -f utf8)
+  HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+  [[ $HTTP_STATUS != "200" ]] && echo "Aborting: Invalid github api token" && exit 1
+}
+
+function modify_files {
+  # Add changelog text to first line of CHANGELOG.md
+
+  msg=""
+  # source: https://unix.stackexchange.com/questions/9784/how-can-i-read-line-by-line-from-a-variable-in-bash#9789
+  while IFS= read -r line
+  do
+    # replace newlines with literal "\n" for using with sed
+    msg+="$line\n"
+  done < <(printf '%s\n' "${changelogText}")
+
+  sed -i "1s/^/${msg}\n/" CHANGELOG.md
+  [[ $? != 0 ]] && echo "Aborting: Error modifying CHANGELOG.md" && exit 1
+
+  # Replace version number of etherpad in package.json
+  sed -i -r "s/(\"version\"[ ]*: \").*(\")/\1$VERSION\2/" src/package.json
+  [[ $? != 0 ]] && echo "Aborting: Error modifying package.json" && exit 1
+}
+
+function create_release_branch {
+  echo "Creating new release branch..."
+  git rev-parse --verify release/$VERSION 2>/dev/null
+  if [ $? == 0 ]; then
+    echo "Aborting: Release branch already present"
+    exit 1
+  fi
+  git checkout -b release/$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error creating release branch" && exit 1
+
+  echo "Committing CHANGELOG.md and package.json"
+  git add CHANGELOG.md
+  git add src/package.json
+  git commit -m "Release version $VERSION"
+
+  echo "Pushing release branch to github..."
+  git push -u $ETHER_REPO release/$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error pushing release branch to github" && exit 1
+}
+
+function merge_release_branch {
+  echo "Merging release to master branch on github..."
+  API_JSON=$(printf '{"base": "master","head": "release/%s","commit_message": "Merge new release into master branch!"}' $VERSION)
+  CURL_RESPONSE=$(curl --silent -i -N --data "$API_JSON" https://api.github.com/repos/ether/etherpad-lite/merges?access_token=$API_TOKEN  | iconv -f utf8)
+  echo $CURL_RESPONSE
+  HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+  [[ $HTTP_STATUS != "200" ]] && echo "Aborting: Error merging release branch on github" && exit 1
+}
+
+function create_builds {
+  echo "Cloning etherpad-lite repo and ether.github.com repo..."
+  cd $TMP_DIR
+  rm -rf etherpad-lite ether.github.com
+  git clone $ETHER_REPO --branch master
+  git clone $ETHER_WEB_REPO
+  echo "Creating windows build..."
+  cd etherpad-lite
+  bin/buildForWindows.sh
+  [[ $? != 0 ]] && echo "Aborting: Error creating build for windows" && exit 1
+  echo "Creating docs..."
+  make docs
+  [[ $? != 0 ]] && echo "Aborting: Error generating docs" && exit 1
+}
+
+function push_builds {
+  cd $TMP_DIR/etherpad-lite/
+  echo "Copying windows build and docs to website repo..."
+  GIT_SHA=$(git rev-parse HEAD | cut -c1-10)
+  mv etherpad-lite-win.zip $TMP_DIR/ether.github.com/downloads/etherpad-lite-win-$VERSION-$GIT_SHA.zip
+
+  mv out/doc $TMP_DIR/ether.github.com/doc/v$VERSION
+
+  cd $TMP_DIR/ether.github.com/
+  sed -i "s/etherpad-lite-win.*\.zip/etherpad-lite-win-$VERSION-$GIT_SHA.zip/" index.html
+  sed -i "s/$LATEST_GIT_TAG/$VERSION/g" index.html
+  git checkout -b release_$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error creating new release branch" && exit 1
+  git add doc/
+  git add downloads/
+  git commit -a -m "Release version $VERSION"
+  git push -u $ETHER_WEB_REPO release_$VERSION
+  [[ $? != 0 ]] && echo "Aborting: Error pushing release branch to github" && exit 1
+}
+
+function merge_web_branch {
+        echo "Merging release to master branch on github..."
+        API_JSON=$(printf '{"base": "master","head": "release_%s","commit_message": "Release version %s"}' $VERSION $VERSION)
+         CURL_RESPONSE=$(curl --silent -i -N --data "$API_JSON" https://api.github.com/repos/ether/ether.github.com/merges?access_token=$API_TOKEN | iconv -f utf8)
+  echo $CURL_RESPONSE
+  HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+  [[ $HTTP_STATUS != "200" ]] && echo "Aborting: Error merging release branch" && exit 1
+}
+
+function publish_release {
+  echo -n "Do you want to publish a new release on github (y/n)? "
+  read PUBLISH_RELEASE
+  if [ $PUBLISH_RELEASE = "y" ]; then
+    # create a new release on github
+    API_JSON=$(printf '{"tag_name": "%s","target_commitish": "master","name": "Release %s","body": "%s","draft": false,"prerelease": false}' $VERSION $VERSION $changelogText)
+    CURL_RESPONSE=$(curl --silent -i -N --data "$API_JSON" https://api.github.com/repos/ether/etherpad-lite/releases?access_token=$API_TOKEN | iconv -f utf8)
+    HTTP_STATUS=$(echo $CURL_RESPONSE | head -1 | sed -r 's/.* ([0-9]{3}) .*/\1/')
+    [[ $HTTP_STATUS != "201" ]] && echo "Aborting: Error publishing release on github" && exit 1
+  else
+    echo "No release published on github!"
+  fi
+}
+
+function todo_notification {
+  echo "Release procedure was successful, but you have to do some steps manually:"
+  echo "- Update the wiki at https://github.com/ether/etherpad-lite/wiki"
+  echo "- Create a pull request on github to merge the master branch back to develop"
+  echo "- Announce the new release on the mailing list, blog.etherpad.org and Twitter"
+}
+
+# Call functions
+check_api_token
+modify_files
+create_release_branch
+merge_release_branch
+create_builds
+push_builds
+merge_web_branch
+publish_release
+todo_notification

src/bin/createUserSession.js

@@ -0,0 +1,51 @@
+'use strict';
+
+/*
+ * A tool for generating a test user session which can be used for debugging configs
+ * that require sessions.
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const fs = require('fs');
+const path = require('path');
+const querystring = require('querystring');
+const settings = require('../node/utils/Settings');
+const supertest = require('supertest');
+
+(async () => {
+  const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+  const filePath = path.join(__dirname, '../../APIKEY.txt');
+  const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+
+  let res;
+
+  res = await api.get('/api/');
+  const apiVersion = res.body.currentVersion;
+  if (!apiVersion) throw new Error('No version set in API');
+  const uri = (cmd, args) => `/api/${apiVersion}/${cmd}?${querystring.stringify(args)}`;
+
+  res = await api.post(uri('createGroup', {apikey}));
+  if (res.body.code === 1) throw new Error(`Error creating group: ${res.body}`);
+  const groupID = res.body.data.groupID;
+  console.log('groupID', groupID);
+
+  res = await api.post(uri('createGroupPad', {apikey, groupID}));
+  if (res.body.code === 1) throw new Error(`Error creating group pad: ${res.body}`);
+  console.log('Test Pad ID ====> ', res.body.data.padID);
+
+  res = await api.post(uri('createAuthor', {apikey}));
+  if (res.body.code === 1) throw new Error(`Error creating author: ${res.body}`);
+  const authorID = res.body.data.authorID;
+  console.log('authorID', authorID);
+
+  const validUntil = Math.floor(new Date() / 1000) + 60000;
+  console.log('validUntil', validUntil);
+  res = await api.post(uri('createSession', {apikey, groupID, authorID, validUntil}));
+  if (res.body.code === 1) throw new Error(`Error creating session: ${res.body}`);
+  console.log('Session made: ====> create a cookie named sessionID and set the value to',
+      res.body.data.sessionID);
+})();

src/bin/deb-src/DEBIAN/control

@@ -0,0 +1,9 @@
+Package: etherpad
+Version: 1.3
+Section: base
+Priority: optional
+Architecture: i386
+Installed-Size: SIZE
+Depends:
+Maintainer: John McLear <john@mclear.co.uk>
+Description: Etherpad is a collaborative editor.  

src/bin/deb-src/DEBIAN/postinst

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Start the services!
+
+service etherpad start
+echo "Give Etherpad about 3 minutes to install dependencies then visit http://localhost:9001 in your web browser"
+echo "To stop etherpad type 'service etherpad stop', To restart type 'service etherpad restart'".
+rm -f /tmp/etherpad.log /tmp/etherpad.err

src/bin/deb-src/DEBIAN/preinst

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Installs node if it isn't already installed
+#
+# Don't steamroll over a previously installed node version
+# TODO provide a local version of node?
+
+VER="0.10.4"
+ARCH="x86"
+if [ `arch | grep 64` ]
+then
+  ARCH="x64"
+fi
+
+# TODO test version
+if [ ! -f /usr/local/bin/node ]
+then
+  pushd /tmp
+  wget -c "http://nodejs.org/dist/v${VER}/node-v${VER}-linux-${ARCH}.tar.gz"
+  rm -rf /tmp/node-v${VER}-linux-${ARCH}
+  tar xf node-v${VER}-linux-${ARCH}.tar.gz -C /tmp/
+  cp -a /tmp/node-v${VER}-linux-${ARCH}/* /usr/local/
+fi
+
+# Create Etherpad user
+adduser --system etherpad

src/bin/deb-src/DEBIAN/prerm

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Stop the appserver:
+service etherpad stop || true

src/bin/deb-src/sysroot/etc/init/etherpad.conf

@@ -0,0 +1,28 @@
+description "etherpad"
+
+start on started networking
+stop on runlevel [!2345]
+
+env EPHOME=/opt/etherpad
+env EPLOGS=/var/log/etherpad
+env EPUSER=etherpad
+
+respawn
+
+pre-start script
+    cd $EPHOME
+    mkdir $EPLOGS                        ||true
+    chown $EPUSER $EPLOGS                ||true
+    chmod 0755 $EPLOGS                   ||true
+    chown -R $EPUSER $EPHOME/var         ||true
+    $EPHOME/bin/installDeps.sh >> $EPLOGS/error.log || { stop; exit 1; }
+end script
+
+script
+  cd $EPHOME/
+  exec su -s /bin/sh -c 'exec "$0" "$@"' $EPUSER -- node node_modules/ep_etherpad-lite/node/server.js \
+                        >> $EPLOGS/access.log \
+                        2>> $EPLOGS/error.log
+  echo "Etherpad is running on http://localhost:9001 - To change settings edit /opt/etherpad/settings.json"
+
+end script

src/bin/debugRun.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+# Prepare the environment
+bin/installDeps.sh || exit 1
+
+echo "If you are new to debugging Node.js with Chrome DevTools, take a look at this page:"
+echo "https://medium.com/@paul_irish/debugging-node-js-nightlies-with-chrome-devtools-7c4a1b95ae27"
+echo "Open 'chrome://inspect' on Chrome to start debugging."
+
+# Use 0.0.0.0 to allow external connections to the debugger
+# (ex: running Etherpad on a docker container). Use default port # (9229)
+node $(compute_node_args) --inspect=0.0.0.0:9229 node_modules/ep_etherpad-lite/node/server.js "$@"

src/bin/deleteAllGroupSessions.js

@@ -0,0 +1,47 @@
+'use strict';
+
+/*
+* A tool for deleting ALL GROUP sessions Etherpad user sessions from the CLI,
+* because sometimes a brick is required to fix a face.
+*/
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const path = require('path');
+const fs = require('fs');
+const supertest = require('supertest');
+
+// Set a delete counter which will increment on each delete attempt
+// TODO: Check delete is successful before incrementing
+let deleteCount = 0;
+
+// get the API Key
+const filePath = path.join(__dirname, '../../APIKEY.txt');
+console.log('Deleting all group sessions, please be patient.');
+
+(async () => {
+  const settings = require('../tests/container/loadSettings').loadSettings();
+  const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+  const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+  const apiVersionResponse = await api.get('/api/');
+  const apiVersion = apiVersionResponse.body.currentVersion; // 1.12.5
+
+  const groupsResponse = await api.get(`/api/${apiVersion}/listAllGroups?apikey=${apikey}`);
+  const groups = groupsResponse.body.data.groupIDs; // ['whateverGroupID']
+
+  for (const groupID of groups) {
+    const sessionURI = `/api/${apiVersion}/listSessionsOfGroup?apikey=${apikey}&groupID=${groupID}`;
+    const sessionsResponse = await api.get(sessionURI);
+    const sessions = sessionsResponse.body.data;
+
+    for (const sessionID of Object.keys(sessions)) {
+      const deleteURI = `/api/${apiVersion}/deleteSession?apikey=${apikey}&sessionID=${sessionID}`;
+      await api.post(deleteURI); // delete
+      deleteCount++;
+    }
+  }
+  console.log(`Deleted ${deleteCount} sessions`);
+})();

src/bin/deletePad.js

@@ -0,0 +1,38 @@
+'use strict';
+
+/*
+ * A tool for deleting pads from the CLI, because sometimes a brick is required
+ * to fix a window.
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const settings = require('../tests/container/loadSettings').loadSettings();
+const path = require('path');
+const fs = require('fs');
+const supertest = require('supertest');
+
+const api = supertest(`http://${settings.ip}:${settings.port}`);
+
+if (process.argv.length !== 3) throw new Error('Use: node deletePad.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+// get the API Key
+const filePath = path.join(__dirname, '../../APIKEY.txt');
+const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+
+(async () => {
+  let apiVersion = await api.get('/api/');
+  apiVersion = apiVersion.body.currentVersion;
+  if (!apiVersion) throw new Error('No version set in API');
+
+  // Now we know the latest API version, let's delete pad
+  const uri = `/api/${apiVersion}/deletePad?apikey=${apikey}&padID=${padId}`;
+  const deleteAttempt = await api.post(uri);
+  if (deleteAttempt.body.code === 1) throw new Error(`Error deleting pad ${deleteAttempt.body}`);
+  console.log('Deleted pad', deleteAttempt.body);
+})();

src/bin/dirty-db-cleaner.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env PYTHONUNBUFFERED=1 python
+#
+# Created by Bjarni R. Einarsson, placed in the public domain. Go wild!
+#
+import json
+import os
+import sys
+
+try:
+    dirtydb_input = sys.argv[1]
+    dirtydb_output = '%s.new' % dirtydb_input
+    assert(os.path.exists(dirtydb_input))
+    assert(not os.path.exists(dirtydb_output))
+except:
+    print()
+    print('Usage: %s /path/to/dirty.db' % sys.argv[0])
+    print()
+    print('Note: Will create a file named dirty.db.new in the same folder,')
+    print('      please make sure permissions are OK and a file by that')
+    print('      name does not exist already. This script works by omitting')
+    print('      duplicate lines from the dirty.db file, keeping only the')
+    print('      last (latest) instance. No revision data should be lost,')
+    print('      but be careful, make backups. If it breaks you get to keep')
+    print('      both pieces!')
+    print()
+    sys.exit(1)
+
+dirtydb = {}
+lines = 0
+with open(dirtydb_input, 'r') as fd:
+    print('Reading %s' % dirtydb_input)
+    for line in fd:
+        lines += 1
+        try:
+            data = json.loads(line)
+            dirtydb[data['key']] = line
+        except:
+            print("Skipping invalid JSON!")
+        if lines % 10000 == 0:
+            sys.stderr.write('.')
+print()
+print('OK, found %d unique keys in %d lines' % (len(dirtydb), lines))
+
+with open(dirtydb_output, 'w') as fd:
+    for data in list(dirtydb.values()):
+        fd.write(data)
+
+print('Wrote data to %s. All done!' % dirtydb_output)

src/bin/doc/LICENSE

@@ -0,0 +1,18 @@
+Copyright Joyent, Inc. and other Node contributors. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

src/bin/doc/README.md

@@ -0,0 +1,76 @@
+Here's how the node docs work.
+
+Each type of heading has a description block.
+
+
+    ## module
+
+        Stability: 3 - Stable
+
+    description and examples.
+
+    ### module.property
+
+    * Type
+
+    description of the property.
+
+    ### module.someFunction(x, y, [z=100])
+
+    * `x` {String} the description of the string
+    * `y` {Boolean} Should I stay or should I go?
+    * `z` {Number} How many zebras to bring.
+
+    A description of the function.
+
+    ### Event: 'blerg'
+
+    * Argument: SomeClass object.
+
+    Modules don't usually raise events on themselves.  `cluster` is the
+    only exception.
+
+    ## Class: SomeClass
+
+    description of the class.
+
+    ### Class Method: SomeClass.classMethod(anArg)
+
+    * `anArg` {Object}  Just an argument
+      * `field` {String} anArg can have this field.
+      * `field2` {Boolean}  Another field.  Default: `false`.
+    * Return: {Boolean} `true` if it worked.
+
+    Description of the method for humans.
+
+    ### someClass.nextSibling()
+
+    * Return: {SomeClass object | null}  The next someClass in line.
+
+    ### someClass.someProperty
+
+    * String
+
+    The indication of what someProperty is.
+
+    ### Event: 'grelb'
+
+    * `isBlerg` {Boolean}
+
+    This event is emitted on instances of SomeClass, not on the module itself.
+
+
+* Modules have (description, Properties, Functions, Classes, Examples)
+* Properties have (type, description)
+* Functions have (list of arguments, description)
+* Classes have (description, Properties, Methods, Events)
+* Events have (list of arguments, description)
+* Methods have (list of arguments, description)
+* Properties have (type, description)
+
+# CLI usage
+
+Run the following from the etherpad-lite root directory:
+```sh
+$ node bin/doc/generate doc/index.md --format=html --template=doc/template.html > out.html
+```

src/bin/doc/generate.js

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+
+'use strict';
+
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+const fs = require('fs');
+const path = require('path');
+
+// parse the args.
+// Don't use nopt or whatever for this.  It's simple enough.
+
+const args = process.argv.slice(2);
+let format = 'json';
+let template = null;
+let inputFile = null;
+
+args.forEach((arg) => {
+  if (!arg.match(/^--/)) {
+    inputFile = arg;
+  } else if (arg.match(/^--format=/)) {
+    format = arg.replace(/^--format=/, '');
+  } else if (arg.match(/^--template=/)) {
+    template = arg.replace(/^--template=/, '');
+  }
+});
+
+
+if (!inputFile) {
+  throw new Error('No input file specified');
+}
+
+
+console.error('Input file = %s', inputFile);
+fs.readFile(inputFile, 'utf8', (er, input) => {
+  if (er) throw er;
+  // process the input for @include lines
+  processIncludes(inputFile, input, next);
+});
+
+
+const includeExpr = /^@include\s+([A-Za-z0-9-_/]+)(?:\.)?([a-zA-Z]*)$/gmi;
+const includeData = {};
+const processIncludes = (inputFile, input, cb) => {
+  const includes = input.match(includeExpr);
+  if (includes == null) return cb(null, input);
+  let errState = null;
+  console.error(includes);
+  let incCount = includes.length;
+  if (incCount === 0) cb(null, input);
+
+  includes.forEach((include) => {
+    let fname = include.replace(/^@include\s+/, '');
+    if (!fname.match(/\.md$/)) fname += '.md';
+
+    if (Object.prototype.hasOwnProperty.call(includeData, fname)) {
+      input = input.split(include).join(includeData[fname]);
+      incCount--;
+      if (incCount === 0) {
+        return cb(null, input);
+      }
+    }
+
+    const fullFname = path.resolve(path.dirname(inputFile), fname);
+    fs.readFile(fullFname, 'utf8', (er, inc) => {
+      if (errState) return;
+      if (er) return cb(errState = er);
+      processIncludes(fullFname, inc, (er, inc) => {
+        if (errState) return;
+        if (er) return cb(errState = er);
+        incCount--;
+        includeData[fname] = inc;
+        input = input.split(include).join(includeData[fname]);
+        if (incCount === 0) {
+          return cb(null, input);
+        }
+      });
+    });
+  });
+};
+
+
+const next = (er, input) => {
+  if (er) throw er;
+  switch (format) {
+    case 'json':
+      require('./json.js')(input, inputFile, (er, obj) => {
+        console.log(JSON.stringify(obj, null, 2));
+        if (er) throw er;
+      });
+      break;
+
+    case 'html':
+      require('./html.js')(input, inputFile, template, (er, html) => {
+        if (er) throw er;
+        console.log(html);
+      });
+      break;
+
+    default:
+      throw new Error(`Invalid format: ${format}`);
+  }
+};

src/bin/doc/html.js

@@ -0,0 +1,172 @@
+'use strict';
+
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+const fs = require('fs');
+const marked = require('marked');
+const path = require('path');
+
+
+const toHTML = (input, filename, template, cb) => {
+  const lexed = marked.lexer(input);
+  fs.readFile(template, 'utf8', (er, template) => {
+    if (er) return cb(er);
+    render(lexed, filename, template, cb);
+  });
+};
+module.exports = toHTML;
+
+const render = (lexed, filename, template, cb) => {
+  // get the section
+  const section = getSection(lexed);
+
+  filename = path.basename(filename, '.md');
+
+  lexed = parseLists(lexed);
+
+  // generate the table of contents.
+  // this mutates the lexed contents in-place.
+  buildToc(lexed, filename, (er, toc) => {
+    if (er) return cb(er);
+
+    template = template.replace(/__FILENAME__/g, filename);
+    template = template.replace(/__SECTION__/g, section);
+    template = template.replace(/__TOC__/g, toc);
+
+    // content has to be the last thing we do with
+    // the lexed tokens, because it's destructive.
+    const content = marked.parser(lexed);
+    template = template.replace(/__CONTENT__/g, content);
+
+    cb(null, template);
+  });
+};
+
+
+// just update the list item text in-place.
+// lists that come right after a heading are what we're after.
+const parseLists = (input) => {
+  let state = null;
+  let depth = 0;
+  const output = [];
+  output.links = input.links;
+  input.forEach((tok) => {
+    if (state == null) {
+      if (tok.type === 'heading') {
+        state = 'AFTERHEADING';
+      }
+      output.push(tok);
+      return;
+    }
+    if (state === 'AFTERHEADING') {
+      if (tok.type === 'list_start') {
+        state = 'LIST';
+        if (depth === 0) {
+          output.push({type: 'html', text: '<div class="signature">'});
+        }
+        depth++;
+        output.push(tok);
+        return;
+      }
+      state = null;
+      output.push(tok);
+      return;
+    }
+    if (state === 'LIST') {
+      if (tok.type === 'list_start') {
+        depth++;
+        output.push(tok);
+        return;
+      }
+      if (tok.type === 'list_end') {
+        depth--;
+        if (depth === 0) {
+          state = null;
+          output.push({type: 'html', text: '</div>'});
+        }
+        output.push(tok);
+        return;
+      }
+      if (tok.text) {
+        tok.text = parseListItem(tok.text);
+      }
+    }
+    output.push(tok);
+  });
+
+  return output;
+};
+
+
+const parseListItem = (text) => {
+  text = text.replace(/\{([^}]+)\}/, '<span class="type">$1</span>');
+  // XXX maybe put more stuff here?
+  return text;
+};
+
+
+// section is just the first heading
+const getSection = (lexed) => {
+  for (let i = 0, l = lexed.length; i < l; i++) {
+    const tok = lexed[i];
+    if (tok.type === 'heading') return tok.text;
+  }
+  return '';
+};
+
+
+const buildToc = (lexed, filename, cb) => {
+  let toc = [];
+  let depth = 0;
+  lexed.forEach((tok) => {
+    if (tok.type !== 'heading') return;
+    if (tok.depth - depth > 1) {
+      return cb(new Error(`Inappropriate heading level\n${
+        JSON.stringify(tok)}`));
+    }
+
+    depth = tok.depth;
+    const id = getId(`${filename}_${tok.text.trim()}`);
+    toc.push(`${new Array((depth - 1) * 2 + 1).join(' ')
+    }* <a href="#${id}">${
+      tok.text}</a>`);
+    tok.text += `<span><a class="mark" href="#${id}" ` +
+                `id="${id}">#</a></span>`;
+  });
+
+  toc = marked.parse(toc.join('\n'));
+  cb(null, toc);
+};
+
+const idCounters = {};
+const getId = (text) => {
+  text = text.toLowerCase();
+  text = text.replace(/[^a-z0-9]+/g, '_');
+  text = text.replace(/^_+|_+$/, '');
+  text = text.replace(/^([^a-z])/, '_$1');
+  if (Object.prototype.hasOwnProperty.call(idCounters, text)) {
+    text += `_${++idCounters[text]}`;
+  } else {
+    idCounters[text] = 0;
+  }
+  return text;
+};

src/bin/doc/json.js

@@ -0,0 +1,556 @@
+'use strict';
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module.exports = doJSON;
+
+// Take the lexed input, and return a JSON-encoded object
+// A module looks like this: https://gist.github.com/1777387
+
+const marked = require('marked');
+
+const doJSON = (input, filename, cb) => {
+  const root = {source: filename};
+  const stack = [root];
+  let depth = 0;
+  let current = root;
+  let state = null;
+  const lexed = marked.lexer(input);
+  lexed.forEach((tok) => {
+    const type = tok.type;
+    let text = tok.text;
+
+    // <!-- type = module -->
+    // This is for cases where the markdown semantic structure is lacking.
+    if (type === 'paragraph' || type === 'html') {
+      const metaExpr = /<!--([^=]+)=([^-]+)-->\n*/g;
+      text = text.replace(metaExpr, (_0, k, v) => {
+        current[k.trim()] = v.trim();
+        return '';
+      });
+      text = text.trim();
+      if (!text) return;
+    }
+
+    if (type === 'heading' &&
+        !text.trim().match(/^example/i)) {
+      if (tok.depth - depth > 1) {
+        return cb(new Error(`Inappropriate heading level\n${
+          JSON.stringify(tok)}`));
+      }
+
+      // Sometimes we have two headings with a single
+      // blob of description.  Treat as a clone.
+      if (current &&
+          state === 'AFTERHEADING' &&
+          depth === tok.depth) {
+        const clone = current;
+        current = newSection(tok);
+        current.clone = clone;
+        // don't keep it around on the stack.
+        stack.pop();
+      } else {
+        // if the level is greater than the current depth,
+        // then it's a child, so we should just leave the stack
+        // as it is.
+        // However, if it's a sibling or higher, then it implies
+        // the closure of the other sections that came before.
+        // root is always considered the level=0 section,
+        // and the lowest heading is 1, so this should always
+        // result in having a valid parent node.
+        let d = tok.depth;
+        while (d <= depth) {
+          finishSection(stack.pop(), stack[stack.length - 1]);
+          d++;
+        }
+        current = newSection(tok);
+      }
+
+      depth = tok.depth;
+      stack.push(current);
+      state = 'AFTERHEADING';
+      return;
+    } // heading
+
+    // Immediately after a heading, we can expect the following
+    //
+    // { type: 'code', text: 'Stability: ...' },
+    //
+    // a list: starting with list_start, ending with list_end,
+    // maybe containing other nested lists in each item.
+    //
+    // If one of these isn't found, then anything that comes between
+    // here and the next heading should be parsed as the desc.
+    let stability;
+    if (state === 'AFTERHEADING') {
+      if (type === 'code' &&
+          (stability = text.match(/^Stability: ([0-5])(?:\s*-\s*)?(.*)$/))) {
+        current.stability = parseInt(stability[1], 10);
+        current.stabilityText = stability[2].trim();
+        return;
+      } else if (type === 'list_start' && !tok.ordered) {
+        state = 'AFTERHEADING_LIST';
+        current.list = current.list || [];
+        current.list.push(tok);
+        current.list.level = 1;
+      } else {
+        current.desc = current.desc || [];
+        if (!Array.isArray(current.desc)) {
+          current.shortDesc = current.desc;
+          current.desc = [];
+        }
+        current.desc.push(tok);
+        state = 'DESC';
+      }
+      return;
+    }
+
+    if (state === 'AFTERHEADING_LIST') {
+      current.list.push(tok);
+      if (type === 'list_start') {
+        current.list.level++;
+      } else if (type === 'list_end') {
+        current.list.level--;
+      }
+      if (current.list.level === 0) {
+        state = 'AFTERHEADING';
+        processList(current);
+      }
+      return;
+    }
+
+    current.desc = current.desc || [];
+    current.desc.push(tok);
+  });
+
+  // finish any sections left open
+  while (root !== (current = stack.pop())) {
+    finishSection(current, stack[stack.length - 1]);
+  }
+
+  return cb(null, root);
+};
+
+
+// go from something like this:
+// [ { type: 'list_item_start' },
+//   { type: 'text',
+//     text: '`settings` Object, Optional' },
+//   { type: 'list_start', ordered: false },
+//   { type: 'list_item_start' },
+//   { type: 'text',
+//     text: 'exec: String, file path to worker file.  Default: `__filename`' },
+//   { type: 'list_item_end' },
+//   { type: 'list_item_start' },
+//   { type: 'text',
+//     text: 'args: Array, string arguments passed to worker.' },
+//   { type: 'text',
+//     text: 'Default: `process.argv.slice(2)`' },
+//   { type: 'list_item_end' },
+//   { type: 'list_item_start' },
+//   { type: 'text',
+//     text: 'silent: Boolean, whether or not to send output to parent\'s stdio.' },
+//   { type: 'text', text: 'Default: `false`' },
+//   { type: 'space' },
+//   { type: 'list_item_end' },
+//   { type: 'list_end' },
+//   { type: 'list_item_end' },
+//   { type: 'list_end' } ]
+// to something like:
+// [ { name: 'settings',
+//     type: 'object',
+//     optional: true,
+//     settings:
+//      [ { name: 'exec',
+//          type: 'string',
+//          desc: 'file path to worker file',
+//          default: '__filename' },
+//        { name: 'args',
+//          type: 'array',
+//          default: 'process.argv.slice(2)',
+//          desc: 'string arguments passed to worker.' },
+//        { name: 'silent',
+//          type: 'boolean',
+//          desc: 'whether or not to send output to parent\'s stdio.',
+//          default: 'false' } ] } ]
+
+const processList = (section) => {
+  const list = section.list;
+  const values = [];
+  let current;
+  const stack = [];
+
+  // for now, *just* build the hierarchical list
+  list.forEach((tok) => {
+    const type = tok.type;
+    if (type === 'space') return;
+    if (type === 'list_item_start') {
+      if (!current) {
+        const n = {};
+        values.push(n);
+        current = n;
+      } else {
+        current.options = current.options || [];
+        stack.push(current);
+        const n = {};
+        current.options.push(n);
+        current = n;
+      }
+      return;
+    } else if (type === 'list_item_end') {
+      if (!current) {
+        throw new Error(`invalid list - end without current item\n${
+          JSON.stringify(tok)}\n${
+          JSON.stringify(list)}`);
+      }
+      current = stack.pop();
+    } else if (type === 'text') {
+      if (!current) {
+        throw new Error(`invalid list - text without current item\n${
+          JSON.stringify(tok)}\n${
+          JSON.stringify(list)}`);
+      }
+      current.textRaw = current.textRaw || '';
+      current.textRaw += `${tok.text} `;
+    }
+  });
+
+  // shove the name in there for properties, since they are always
+  // just going to be the value etc.
+  if (section.type === 'property' && values[0]) {
+    values[0].textRaw = `\`${section.name}\` ${values[0].textRaw}`;
+  }
+
+  // now pull the actual values out of the text bits.
+  values.forEach(parseListItem);
+
+  // Now figure out what this list actually means.
+  // depending on the section type, the list could be different things.
+
+  switch (section.type) {
+    case 'ctor':
+    case 'classMethod':
+    case 'method': {
+      // each item is an argument, unless the name is 'return',
+      // in which case it's the return value.
+      section.signatures = section.signatures || [];
+      const sig = {};
+      section.signatures.push(sig);
+      sig.params = values.filter((v) => {
+        if (v.name === 'return') {
+          sig.return = v;
+          return false;
+        }
+        return true;
+      });
+      parseSignature(section.textRaw, sig);
+      break;
+    }
+    case 'property': {
+      // there should be only one item, which is the value.
+      // copy the data up to the section.
+      const value = values[0] || {};
+      delete value.name;
+      section.typeof = value.type;
+      delete value.type;
+      Object.keys(value).forEach((k) => {
+        section[k] = value[k];
+      });
+      break;
+    }
+
+    case 'event': {
+      // event: each item is an argument.
+      section.params = values;
+      break;
+    }
+  }
+
+  delete section.list;
+};
+
+
+// textRaw = "someobject.someMethod(a, [b=100], [c])"
+const parseSignature = (text, sig) => {
+  let params = text.match(paramExpr);
+  if (!params) return;
+  params = params[1];
+  // the ] is irrelevant. [ indicates optionalness.
+  params = params.replace(/\]/g, '');
+  params = params.split(/,/);
+  params.forEach((p, i, _) => {
+    p = p.trim();
+    if (!p) return;
+    let param = sig.params[i];
+    let optional = false;
+    let def;
+    // [foo] -> optional
+    if (p.charAt(0) === '[') {
+      optional = true;
+      p = p.substr(1);
+    }
+    const eq = p.indexOf('=');
+    if (eq !== -1) {
+      def = p.substr(eq + 1);
+      p = p.substr(0, eq);
+    }
+    if (!param) {
+      param = sig.params[i] = {name: p};
+    }
+    // at this point, the name should match.
+    if (p !== param.name) {
+      console.error('Warning: invalid param "%s"', p);
+      console.error(` > ${JSON.stringify(param)}`);
+      console.error(` > ${text}`);
+    }
+    if (optional) param.optional = true;
+    if (def !== undefined) param.default = def;
+  });
+};
+
+
+const parseListItem = (item) => {
+  if (item.options) item.options.forEach(parseListItem);
+  if (!item.textRaw) return;
+
+  // the goal here is to find the name, type, default, and optional.
+  // anything left over is 'desc'
+  let text = item.textRaw.trim();
+  // text = text.replace(/^(Argument|Param)s?\s*:?\s*/i, '');
+
+  text = text.replace(/^, /, '').trim();
+  const retExpr = /^returns?\s*:?\s*/i;
+  const ret = text.match(retExpr);
+  if (ret) {
+    item.name = 'return';
+    text = text.replace(retExpr, '');
+  } else {
+    const nameExpr = /^['`"]?([^'`": {]+)['`"]?\s*:?\s*/;
+    const name = text.match(nameExpr);
+    if (name) {
+      item.name = name[1];
+      text = text.replace(nameExpr, '');
+    }
+  }
+
+  text = text.trim();
+  const defaultExpr = /\(default\s*[:=]?\s*['"`]?([^, '"`]*)['"`]?\)/i;
+  const def = text.match(defaultExpr);
+  if (def) {
+    item.default = def[1];
+    text = text.replace(defaultExpr, '');
+  }
+
+  text = text.trim();
+  const typeExpr = /^\{([^}]+)\}/;
+  const type = text.match(typeExpr);
+  if (type) {
+    item.type = type[1];
+    text = text.replace(typeExpr, '');
+  }
+
+  text = text.trim();
+  const optExpr = /^Optional\.|(?:, )?Optional$/;
+  const optional = text.match(optExpr);
+  if (optional) {
+    item.optional = true;
+    text = text.replace(optExpr, '');
+  }
+
+  text = text.replace(/^\s*-\s*/, '');
+  text = text.trim();
+  if (text) item.desc = text;
+};
+
+
+const finishSection = (section, parent) => {
+  if (!section || !parent) {
+    throw new Error(`Invalid finishSection call\n${
+      JSON.stringify(section)}\n${
+      JSON.stringify(parent)}`);
+  }
+
+  if (!section.type) {
+    section.type = 'module';
+    if (parent && (parent.type === 'misc')) {
+      section.type = 'misc';
+    }
+    section.displayName = section.name;
+    section.name = section.name.toLowerCase()
+        .trim().replace(/\s+/g, '_');
+  }
+
+  if (section.desc && Array.isArray(section.desc)) {
+    section.desc.links = section.desc.links || [];
+    section.desc = marked.parser(section.desc);
+  }
+
+  if (!section.list) section.list = [];
+  processList(section);
+
+  // classes sometimes have various 'ctor' children
+  // which are actually just descriptions of a constructor
+  // class signature.
+  // Merge them into the parent.
+  if (section.type === 'class' && section.ctors) {
+    section.signatures = section.signatures || [];
+    const sigs = section.signatures;
+    section.ctors.forEach((ctor) => {
+      ctor.signatures = ctor.signatures || [{}];
+      ctor.signatures.forEach((sig) => {
+        sig.desc = ctor.desc;
+      });
+      sigs.push(...ctor.signatures);
+    });
+    delete section.ctors;
+  }
+
+  // properties are a bit special.
+  // their "type" is the type of object, not "property"
+  if (section.properties) {
+    section.properties.forEach((p) => {
+      if (p.typeof) p.type = p.typeof;
+      else delete p.type;
+      delete p.typeof;
+    });
+  }
+
+  // handle clones
+  if (section.clone) {
+    const clone = section.clone;
+    delete section.clone;
+    delete clone.clone;
+    deepCopy(section, clone);
+    finishSection(clone, parent);
+  }
+
+  let plur;
+  if (section.type.slice(-1) === 's') {
+    plur = `${section.type}es`;
+  } else if (section.type.slice(-1) === 'y') {
+    plur = section.type.replace(/y$/, 'ies');
+  } else {
+    plur = `${section.type}s`;
+  }
+
+  // if the parent's type is 'misc', then it's just a random
+  // collection of stuff, like the "globals" section.
+  // Make the children top-level items.
+  if (section.type === 'misc') {
+    Object.keys(section).forEach((k) => {
+      switch (k) {
+        case 'textRaw':
+        case 'name':
+        case 'type':
+        case 'desc':
+        case 'miscs':
+          return;
+        default:
+          if (parent.type === 'misc') {
+            return;
+          }
+          if (Array.isArray(k) && parent[k]) {
+            parent[k] = parent[k].concat(section[k]);
+          } else if (!parent[k]) {
+            parent[k] = section[k];
+          } else {
+            // parent already has, and it's not an array.
+            return;
+          }
+      }
+    });
+  }
+
+  parent[plur] = parent[plur] || [];
+  parent[plur].push(section);
+};
+
+
+// Not a general purpose deep copy.
+// But sufficient for these basic things.
+const deepCopy = (src, dest) => {
+  Object.keys(src).filter((k) => !Object.prototype.hasOwnProperty.call(dest, k)).forEach((k) => {
+    dest[k] = deepCopy_(src[k]);
+  });
+};
+
+const deepCopy_ = (src) => {
+  if (!src) return src;
+  if (Array.isArray(src)) {
+    const c = new Array(src.length);
+    src.forEach((v, i) => {
+      c[i] = deepCopy_(v);
+    });
+    return c;
+  }
+  if (typeof src === 'object') {
+    const c = {};
+    Object.keys(src).forEach((k) => {
+      c[k] = deepCopy_(src[k]);
+    });
+    return c;
+  }
+  return src;
+};
+
+
+// these parse out the contents of an H# tag
+const eventExpr = /^Event(?::|\s)+['"]?([^"']+).*$/i;
+const classExpr = /^Class:\s*([^ ]+).*?$/i;
+const propExpr = /^(?:property:?\s*)?[^.]+\.([^ .()]+)\s*?$/i;
+const braceExpr = /^(?:property:?\s*)?[^.[]+(\[[^\]]+\])\s*?$/i;
+const classMethExpr =
+  /^class\s*method\s*:?[^.]+\.([^ .()]+)\([^)]*\)\s*?$/i;
+const methExpr =
+  /^(?:method:?\s*)?(?:[^.]+\.)?([^ .()]+)\([^)]*\)\s*?$/i;
+const newExpr = /^new ([A-Z][a-z]+)\([^)]*\)\s*?$/;
+const paramExpr = /\((.*)\);?$/;
+
+const newSection = (tok) => {
+  const section = {};
+  // infer the type from the text.
+  const text = section.textRaw = tok.text;
+  if (text.match(eventExpr)) {
+    section.type = 'event';
+    section.name = text.replace(eventExpr, '$1');
+  } else if (text.match(classExpr)) {
+    section.type = 'class';
+    section.name = text.replace(classExpr, '$1');
+  } else if (text.match(braceExpr)) {
+    section.type = 'property';
+    section.name = text.replace(braceExpr, '$1');
+  } else if (text.match(propExpr)) {
+    section.type = 'property';
+    section.name = text.replace(propExpr, '$1');
+  } else if (text.match(classMethExpr)) {
+    section.type = 'classMethod';
+    section.name = text.replace(classMethExpr, '$1');
+  } else if (text.match(methExpr)) {
+    section.type = 'method';
+    section.name = text.replace(methExpr, '$1');
+  } else if (text.match(newExpr)) {
+    section.type = 'ctor';
+    section.name = text.replace(newExpr, '$1');
+  } else {
+    section.name = text;
+  }
+  return section;
+};

src/bin/doc/package-lock.json

@@ -0,0 +1,13 @@
+{
+  "name": "node-doc-generator",
+  "version": "0.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "marked": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.2.tgz",
+      "integrity": "sha512-EGwzEeCcLniFX51DhTpmTom+dSA/MG/OBUDjnWtHbEnjAH180VzUeAw+oE4+Zv+CoYBWyRlYOTR0N8SO9R1PVw=="
+    }
+  }
+}

src/bin/doc/package.json

@@ -0,0 +1,15 @@
+{
+  "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)",
+  "name": "node-doc-generator",
+  "description": "Internal tool for generating Node.js API docs",
+  "version": "0.0.0",
+  "engines": {
+    "node": ">=10.17.0"
+  },
+  "dependencies": {
+    "marked": "0.8.2"
+  },
+  "devDependencies": {},
+  "optionalDependencies": {},
+  "bin": "./generate.js"
+}

src/bin/extractPadData.js

@@ -0,0 +1,64 @@
+'use strict';
+
+/*
+ * This is a debug tool. It helps to extract all datas of a pad and move it from
+ * a productive environment and to a develop environment to reproduce bugs
+ * there. It outputs a dirtydb file
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 3) throw new Error('Use: node extractPadData.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // load extra modules
+  const dirtyDB = require('dirty');
+  const padManager = require('../node/db/PadManager');
+
+  // initialize output database
+  const dirty = dirtyDB(`${padId}.db`);
+
+  // Promise wrapped get and set function
+  const wrapped = db.db.db.wrappedDB;
+  const get = util.promisify(wrapped.get.bind(wrapped));
+  const set = util.promisify(dirty.set.bind(dirty));
+
+  // array in which required key values will be accumulated
+  const neededDBValues = [`pad:${padId}`];
+
+  // get the actual pad object
+  const pad = await padManager.getPad(padId);
+
+  // add all authors
+  neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`));
+
+  // add all revisions
+  for (let rev = 0; rev <= pad.head; ++rev) {
+    neededDBValues.push(`pad:${padId}:revs:${rev}`);
+  }
+
+  // add all chat values
+  for (let chat = 0; chat <= pad.chatHead; ++chat) {
+    neededDBValues.push(`pad:${padId}:chat:${chat}`);
+  }
+
+  for (const dbkey of neededDBValues) {
+    let dbvalue = await get(dbkey);
+    if (dbvalue && typeof dbvalue !== 'object') {
+      dbvalue = JSON.parse(dbvalue);
+    }
+    await set(dbkey, dbvalue);
+  }
+
+  console.log('finished');
+})();

src/bin/fastRun.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Run Etherpad directly, assuming all the dependencies are already installed.
+#
+# Useful for developers, or users that know what they are doing. If you just
+# upgraded Etherpad version, installed a new dependency, or are simply unsure
+# of what to do, please execute bin/installDeps.sh once before running this
+# script.
+
+set -eu
+
+# source: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself#246128
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+# Source constants and useful functions
+. ${DIR}/../bin/functions.sh
+
+echo "Running directly, without checking/installing dependencies"
+
+# move to the base Etherpad directory. This will be necessary until Etherpad
+# learns to run from arbitrary CWDs.
+cd "${DIR}/.."
+
+# run Etherpad main class
+node $(compute_node_args) "${DIR}/../node_modules/ep_etherpad-lite/node/server.js" "$@"

src/bin/functions.sh

@@ -0,0 +1,74 @@
+# minimum required node version
+REQUIRED_NODE_MAJOR=10
+REQUIRED_NODE_MINOR=13
+
+# minimum required npm version
+REQUIRED_NPM_MAJOR=5
+REQUIRED_NPM_MINOR=5
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
+
+get_program_version() {
+  PROGRAM="$1"
+  KIND="${2:-full}"
+  PROGRAM_VERSION_STRING=$($PROGRAM --version)
+  PROGRAM_VERSION_STRING=${PROGRAM_VERSION_STRING#"v"}
+
+  DETECTED_MAJOR=$(pecho "$PROGRAM_VERSION_STRING" | cut -s -d "." -f 1)
+  [ -n "$DETECTED_MAJOR" ] || fatal "Cannot extract $PROGRAM major version from version string \"$PROGRAM_VERSION_STRING\""
+  case "$DETECTED_MAJOR" in
+      ''|*[!0-9]*)
+        fatal "$PROGRAM_LABEL major version from \"$VERSION_STRING\" is not a number. Detected: \"$DETECTED_MAJOR\""
+        ;;
+  esac
+
+  DETECTED_MINOR=$(pecho "$PROGRAM_VERSION_STRING" | cut -s -d "." -f 2)
+  [ -n "$DETECTED_MINOR" ] || fatal "Cannot extract $PROGRAM minor version from version string \"$PROGRAM_VERSION_STRING\""
+  case "$DETECTED_MINOR" in
+      ''|*[!0-9]*)
+        fatal "$PROGRAM_LABEL minor version from \"$VERSION_STRING\" is not a number. Detected: \"$DETECTED_MINOR\""
+  esac
+
+  case $KIND in
+    major)
+      echo $DETECTED_MAJOR
+      exit;;
+    minor)
+      echo $DETECTED_MINOR
+      exit;;
+    *)
+      echo $DETECTED_MAJOR.$DETECTED_MINOR
+      exit;;
+  esac
+
+  echo $VERSION
+}
+
+
+compute_node_args() {
+  ARGS=""
+
+  NODE_MAJOR=$(get_program_version "node" "major")
+  [ "$NODE_MAJOR" -eq "10" ] && ARGS="$ARGS --experimental-worker"
+
+  echo $ARGS
+}
+
+
+require_minimal_version() {
+  PROGRAM_LABEL="$1"
+  VERSION="$2"
+  REQUIRED_MAJOR="$3"
+  REQUIRED_MINOR="$4"
+
+  VERSION_MAJOR=$(pecho "$VERSION" | cut -s -d "." -f 1)
+  VERSION_MINOR=$(pecho "$VERSION" | cut -s -d "." -f 2)
+
+  [ "$VERSION_MAJOR" -gt "$REQUIRED_MAJOR" ] || ([ "$VERSION_MAJOR" -eq "$REQUIRED_MAJOR" ] && [ "$VERSION_MINOR" -ge "$REQUIRED_MINOR" ]) \
+    || fatal "Your $PROGRAM_LABEL version \"$VERSION_MAJOR.$VERSION_MINOR\" is too old. $PROGRAM_LABEL $REQUIRED_MAJOR.$REQUIRED_MINOR.x or higher is required."
+}

src/bin/importSqlFile.js

@@ -0,0 +1,100 @@
+'use strict';
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const startTime = Date.now();
+
+const log = (str) => {
+  console.log(`${(Date.now() - startTime) / 1000}\t${str}`);
+};
+
+const unescape = (val) => {
+  // value is a string
+  if (val.substr(0, 1) === "'") {
+    val = val.substr(0, val.length - 1).substr(1);
+
+    return val.replace(/\\[0nrbtZ\\'"]/g, (s) => {
+      switch (s) {
+        case '\\0': return '\0';
+        case '\\n': return '\n';
+        case '\\r': return '\r';
+        case '\\b': return '\b';
+        case '\\t': return '\t';
+        case '\\Z': return '\x1a';
+        default: return s.substr(1);
+      }
+    });
+  }
+
+  // value is a boolean or NULL
+  if (val === 'NULL') {
+    return null;
+  }
+  if (val === 'true') {
+    return true;
+  }
+  if (val === 'false') {
+    return false;
+  }
+
+  // value is a number
+  return val;
+};
+
+(async () => {
+  const fs = require('fs');
+  const log4js = require('log4js');
+  const settings = require('../node/utils/Settings');
+  const ueberDB = require('ueberdb2');
+
+  const dbWrapperSettings = {
+    cache: 0,
+    writeInterval: 100,
+    json: false, // data is already json encoded
+  };
+  const db = new ueberDB.database( // eslint-disable-line new-cap
+      settings.dbType,
+      settings.dbSettings,
+      dbWrapperSettings,
+      log4js.getLogger('ueberDB'));
+
+  const sqlFile = process.argv[2];
+
+  // stop if the settings file is not set
+  if (!sqlFile) throw new Error('Use: node importSqlFile.js $SQLFILE');
+
+  log('initializing db');
+  await util.promisify(db.init.bind(db))();
+  log('done');
+
+  log('open output file...');
+  const lines = fs.readFileSync(sqlFile, 'utf8').split('\n');
+
+  const count = lines.length;
+  let keyNo = 0;
+
+  process.stdout.write(`Start importing ${count} keys...\n`);
+  lines.forEach((l) => {
+    if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') {
+      const pos = l.indexOf("', '");
+      const key = l.substr(28, pos - 28);
+      let value = l.substr(pos + 3);
+      value = value.substr(0, value.length - 2);
+      console.log(`key: ${key} val: ${value}`);
+      console.log(`unval: ${unescape(value)}`);
+      db.set(key, unescape(value), null);
+      keyNo++;
+      if (keyNo % 1000 === 0) {
+        process.stdout.write(` ${keyNo}/${count}\n`);
+      }
+    }
+  });
+  process.stdout.write('\n');
+  process.stdout.write('done. waiting for db to finish transaction. ' +
+                       'depended on dbms this may take some time..\n');
+
+  await util.promisify(db.close.bind(db))();
+  log(`finished, imported ${keyNo} keys.`);
+})();

src/bin/installDeps.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+# Is node installed?
+# Not checking io.js, default installation creates a symbolic link to node
+is_cmd node || fatal "Please install node.js ( https://nodejs.org )"
+
+# Is npm installed?
+is_cmd npm || fatal "Please install npm ( https://npmjs.org )"
+
+# Check npm version
+require_minimal_version "npm" $(get_program_version "npm") "$REQUIRED_NPM_MAJOR" "$REQUIRED_NPM_MINOR"
+
+# Check node version
+require_minimal_version "nodejs" $(get_program_version "node") "$REQUIRED_NODE_MAJOR" "$REQUIRED_NODE_MINOR"
+
+# Get the name of the settings file
+settings="settings.json"
+a='';
+for arg in "$@"; do
+  if [ "$a" = "--settings" ] || [ "$a" = "-s" ]; then settings=$arg; fi
+  a=$arg
+done
+
+# Does a $settings exist? if not copy the template
+if [ ! -f "$settings" ]; then
+  log "Copy the settings template to $settings..."
+  cp settings.json.template "$settings" || exit 1
+fi
+
+log "Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient."
+(
+  mkdir -p node_modules
+  cd node_modules
+  [ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
+  cd ep_etherpad-lite
+  npm ci
+) || {
+  rm -rf src/node_modules
+  exit 1
+}
+
+# Remove all minified data to force node creating it new
+log "Clearing minified cache..."
+rm -f var/minified*
+
+exit 0

src/bin/installOnWindows.bat

@@ -0,0 +1,34 @@
+@echo off
+
+:: Change directory to etherpad-lite root
+cd /D "%~dp0\.."
+
+:: Is node installed?
+cmd /C node -e "" || ( echo "Please install node.js ( https://nodejs.org )" && exit /B 1 )
+
+echo _
+echo Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient.
+
+mkdir node_modules
+cd /D node_modules
+mklink /D "ep_etherpad-lite" "..\src"
+
+cd /D "ep_etherpad-lite"
+cmd /C npm ci || exit /B 1
+
+cd /D "%~dp0\.."
+
+echo _
+echo Clearing cache...
+del /S var\minified*
+
+echo _
+echo Setting up settings.json...
+IF NOT EXIST settings.json (
+  echo Can't find settings.json.
+  echo Copying settings.json.template...
+  cmd /C copy settings.json.template settings.json || exit /B 1
+)
+
+echo _
+echo Installed Etherpad!  To run Etherpad type start.bat

src/bin/migrateDirtyDBtoRealDB.js

@@ -0,0 +1,56 @@
+'use strict';
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+(async () => {
+  // This script requires that you have modified your settings.json file
+  // to work with a real database.  Please make a backup of your dirty.db
+  // file before using this script, just to be safe.
+
+  // It might be necessary to run the script using more memory:
+  // `node --max-old-space-size=4096 bin/migrateDirtyDBtoRealDB.js`
+
+  const dirtyDb = require('dirty');
+  const log4js = require('log4js');
+  const settings = require('../node/utils/Settings');
+  const ueberDB = require('ueberdb2');
+  const util = require('util');
+
+  const dbWrapperSettings = {
+    cache: '0', // The cache slows things down when you're mostly writing.
+    writeInterval: 0, // Write directly to the database, don't buffer
+  };
+  const db = new ueberDB.database( // eslint-disable-line new-cap
+      settings.dbType,
+      settings.dbSettings,
+      dbWrapperSettings,
+      log4js.getLogger('ueberDB'));
+  await db.init();
+
+  console.log('Waiting for dirtyDB to parse its file.');
+  const dirty = dirtyDb(`${__dirname}/../../var/dirty.db`);
+  const length = await new Promise((resolve) => { dirty.once('load', resolve); });
+
+  console.log(`Found ${length} records, processing now.`);
+  const p = [];
+  let numWritten = 0;
+  dirty.forEach((key, value) => {
+    let bcb, wcb;
+    p.push(new Promise((resolve, reject) => {
+      bcb = (err) => { if (err != null) return reject(err); };
+      wcb = (err) => {
+        if (err != null) return reject(err);
+        if (++numWritten % 100 === 0) console.log(`Wrote record ${numWritten} of ${length}`);
+        resolve();
+      };
+    }));
+    db.set(key, value, bcb, wcb);
+  });
+  await Promise.all(p);
+  console.log(`Wrote all ${numWritten} records`);
+
+  await util.promisify(db.close.bind(db))();
+  console.log('Finished.');
+})();

src/bin/plugins/README.md

@@ -0,0 +1,53 @@
+The files in this folder are for Plugin developers.
+
+# Get suggestions to improve your Plugin
+
+This code will check your plugin for known usual issues and some suggestions for
+improvements. No changes will be made to your project.
+
+```
+node bin/plugins/checkPlugin.js $PLUGIN_NAME$
+```
+
+# Basic Example:
+
+```
+node bin/plugins/checkPlugin.js ep_webrtc
+```
+
+## Autofixing - will autofix any issues it can
+
+```
+node bin/plugins/checkPlugin.js ep_whatever autofix
+```
+
+## Autocommitting, push, npm minor patch and npm publish (highly dangerous)
+
+```
+node bin/plugins/checkPlugin.js ep_whatever autocommit
+```
+
+# All the plugins
+
+Replace johnmclear with your github username
+
+```
+# Clones
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+# autofixes and autocommits /pushes & npm publishes
+for dir in node_modules/ep_*; do
+  dir=${dir#node_modules/}
+  [ "$dir" != ep_etherpad-lite ] || continue
+  node bin/plugins/checkPlugin.js "$dir" autocommit
+done
+```
+
+# Automating update of ether organization plugins
+
+```
+getCorePlugins.sh
+updateCorePlugins.sh
+```

src/bin/plugins/checkPlugin.js

@@ -0,0 +1,476 @@
+'use strict';
+
+/*
+ * Usage -- see README.md
+ *
+ * Normal usage:                node bin/plugins/checkPlugin.js ep_whatever
+ * Auto fix the things it can:  node bin/plugins/checkPlugin.js ep_whatever autofix
+ * Auto commit, push and publish to npm (highly dangerous):
+ *                              node bin/plugins/checkPlugin.js ep_whatever autocommit
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const fs = require('fs');
+const childProcess = require('child_process');
+
+// get plugin name & path from user input
+const pluginName = process.argv[2];
+
+if (!pluginName) throw new Error('no plugin name specified');
+
+const pluginPath = `node_modules/${pluginName}`;
+
+console.log(`Checking the plugin: ${pluginName}`);
+
+const optArgs = process.argv.slice(3);
+const autoCommit = optArgs.indexOf('autocommit') !== -1;
+const autoFix = autoCommit || optArgs.indexOf('autofix') !== -1;
+
+const execSync = (cmd, opts = {}) => (childProcess.execSync(cmd, {
+  cwd: `${pluginPath}/`,
+  ...opts,
+}) || '').toString().replace(/\n+$/, '');
+
+const writePackageJson = (obj) => {
+  let s = JSON.stringify(obj, null, 2);
+  if (s.length && s.slice(s.length - 1) !== '\n') s += '\n';
+  return fs.writeFileSync(`${pluginPath}/package.json`, s);
+};
+
+const updateDeps = (parsedPackageJson, key, wantDeps) => {
+  const {[key]: deps = {}} = parsedPackageJson;
+  let changed = false;
+  for (const [pkg, verInfo] of Object.entries(wantDeps)) {
+    const {ver, overwrite = true} = typeof verInfo === 'string' ? {ver: verInfo} : verInfo;
+    if (deps[pkg] === ver) continue;
+    if (deps[pkg] == null) {
+      console.warn(`Missing dependency in ${key}: '${pkg}': '${ver}'`);
+    } else {
+      if (!overwrite) continue;
+      console.warn(`Dependency mismatch in ${key}: '${pkg}': '${ver}' (current: ${deps[pkg]})`);
+    }
+    if (autoFix) {
+      deps[pkg] = ver;
+      changed = true;
+    }
+  }
+  if (changed) {
+    parsedPackageJson[key] = deps;
+    writePackageJson(parsedPackageJson);
+  }
+};
+
+const prepareRepo = () => {
+  let branch = execSync('git symbolic-ref HEAD');
+  if (branch !== 'refs/heads/master' && branch !== 'refs/heads/main') {
+    throw new Error('master/main must be checked out');
+  }
+  branch = branch.replace(/^refs\/heads\//, '');
+  execSync('git rev-parse --verify -q HEAD^0 || ' +
+           `{ echo "Error: no commits on ${branch}" >&2; exit 1; }`);
+  execSync('git rev-parse --verify @{u}'); // Make sure there's a remote tracking branch.
+  const modified = execSync('git diff-files --name-status');
+  if (modified !== '') throw new Error(`working directory has modifications:\n${modified}`);
+  const untracked = execSync('git ls-files -o --exclude-standard');
+  if (untracked !== '') throw new Error(`working directory has untracked files:\n${untracked}`);
+  const indexStatus = execSync('git diff-index --cached --name-status HEAD');
+  if (indexStatus !== '') throw new Error(`uncommitted staged changes to files:\n${indexStatus}`);
+  execSync('git pull --ff-only', {stdio: 'inherit'});
+  if (execSync('git rev-list @{u}...') !== '') throw new Error('repo contains unpushed commits');
+  if (autoCommit) {
+    execSync('git config --get user.name');
+    execSync('git config --get user.email');
+  }
+};
+
+if (autoCommit) {
+  console.warn('Auto commit is enabled, I hope you know what you are doing...');
+}
+
+fs.readdir(pluginPath, (err, rootFiles) => {
+  // handling error
+  if (err) {
+    return console.log(`Unable to scan directory: ${err}`);
+  }
+
+  // rewriting files to lower case
+  const files = [];
+
+  // some files we need to know the actual file name.  Not compulsory but might help in the future.
+  let readMeFileName;
+  let repository;
+
+  for (let i = 0; i < rootFiles.length; i++) {
+    if (rootFiles[i].toLowerCase().indexOf('readme') !== -1) readMeFileName = rootFiles[i];
+    files.push(rootFiles[i].toLowerCase());
+  }
+
+  if (files.indexOf('.git') === -1) throw new Error('No .git folder, aborting');
+  prepareRepo();
+
+  try {
+    const path = `${pluginPath}/.github/workflows/npmpublish.yml`;
+    if (!fs.existsSync(path)) {
+      console.log('no .github/workflows/npmpublish.yml');
+      console.log('create one and set npm secret to auto publish to npm on commit');
+      if (autoFix) {
+        const npmpublish =
+            fs.readFileSync('bin/plugins/lib/npmpublish.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, npmpublish);
+        console.log("If you haven't already, setup autopublish for this plugin https://github.com/ether/etherpad-lite/wiki/Plugins:-Automatically-publishing-to-npm-on-commit-to-Github-Repo");
+      } else {
+        console.log('Setup autopublish for this plugin https://github.com/ether/etherpad-lite/wiki/Plugins:-Automatically-publishing-to-npm-on-commit-to-Github-Repo');
+      }
+    } else {
+      // autopublish exists, we should check the version..
+      // checkVersion takes two file paths and checks for a version string in them.
+      const currVersionFile = fs.readFileSync(path, {encoding: 'utf8', flag: 'r'});
+      const existingConfigLocation = currVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const existingValue = parseInt(
+          currVersionFile.substr(existingConfigLocation + 17, existingConfigLocation.length));
+
+      const reqVersionFile =
+          fs.readFileSync('bin/plugins/lib/npmpublish.yml', {encoding: 'utf8', flag: 'r'});
+      const reqConfigLocation = reqVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const reqValue =
+          parseInt(reqVersionFile.substr(reqConfigLocation + 17, reqConfigLocation.length));
+
+      if (!existingValue || (reqValue > existingValue)) {
+        const npmpublish =
+            fs.readFileSync('bin/plugins/lib/npmpublish.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, npmpublish);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+  }
+
+
+  try {
+    const path = `${pluginPath}/.github/workflows/backend-tests.yml`;
+    if (!fs.existsSync(path)) {
+      console.log('no .github/workflows/backend-tests.yml');
+      console.log('create one and set npm secret to auto publish to npm on commit');
+      if (autoFix) {
+        const backendTests =
+            fs.readFileSync('bin/plugins/lib/backend-tests.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, backendTests);
+      }
+    } else {
+      // autopublish exists, we should check the version..
+      // checkVersion takes two file paths and checks for a version string in them.
+      const currVersionFile = fs.readFileSync(path, {encoding: 'utf8', flag: 'r'});
+      const existingConfigLocation = currVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const existingValue = parseInt(
+          currVersionFile.substr(existingConfigLocation + 17, existingConfigLocation.length));
+
+      const reqVersionFile =
+          fs.readFileSync('bin/plugins/lib/backend-tests.yml', {encoding: 'utf8', flag: 'r'});
+      const reqConfigLocation = reqVersionFile.indexOf('##ETHERPAD_NPM_V=');
+      const reqValue =
+          parseInt(reqVersionFile.substr(reqConfigLocation + 17, reqConfigLocation.length));
+
+      if (!existingValue || (reqValue > existingValue)) {
+        const backendTests =
+            fs.readFileSync('bin/plugins/lib/backend-tests.yml', {encoding: 'utf8', flag: 'r'});
+        fs.mkdirSync(`${pluginPath}/.github/workflows`, {recursive: true});
+        fs.writeFileSync(path, backendTests);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+  }
+
+  if (files.indexOf('package.json') === -1) {
+    console.warn('no package.json, please create');
+  }
+
+  if (files.indexOf('package.json') !== -1) {
+    const packageJSON =
+        fs.readFileSync(`${pluginPath}/package.json`, {encoding: 'utf8', flag: 'r'});
+    const parsedPackageJSON = JSON.parse(packageJSON);
+    if (autoFix) {
+      let updatedPackageJSON = false;
+      if (!parsedPackageJSON.funding) {
+        updatedPackageJSON = true;
+        parsedPackageJSON.funding = {
+          type: 'individual',
+          url: 'https://etherpad.org/',
+        };
+      }
+      if (updatedPackageJSON) {
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+
+    if (packageJSON.toLowerCase().indexOf('repository') === -1) {
+      console.warn('No repository in package.json');
+      if (autoFix) {
+        console.warn('Repository not detected in package.json.  Add repository section.');
+      }
+    } else {
+      // useful for creating README later.
+      repository = parsedPackageJSON.repository.url;
+    }
+
+    updateDeps(parsedPackageJSON, 'devDependencies', {
+      'eslint': '^7.18.0',
+      'eslint-config-etherpad': '^1.0.24',
+      'eslint-plugin-eslint-comments': '^3.2.0',
+      'eslint-plugin-mocha': '^8.0.0',
+      'eslint-plugin-node': '^11.1.0',
+      'eslint-plugin-prefer-arrow': '^1.2.3',
+      'eslint-plugin-promise': '^4.2.1',
+      'eslint-plugin-you-dont-need-lodash-underscore': '^6.10.0',
+    });
+
+    updateDeps(parsedPackageJSON, 'peerDependencies', {
+      // Some plugins require a newer version of Etherpad so don't overwrite if already set.
+      'ep_etherpad-lite': {ver: '>=1.8.6', overwrite: false},
+    });
+
+    if (packageJSON.toLowerCase().indexOf('eslintconfig') === -1) {
+      console.warn('No esLintConfig in package.json');
+      if (autoFix) {
+        const eslintConfig = {
+          root: true,
+          extends: 'etherpad/plugin',
+        };
+        parsedPackageJSON.eslintConfig = eslintConfig;
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+
+    if (packageJSON.toLowerCase().indexOf('scripts') === -1) {
+      console.warn('No scripts in package.json');
+      if (autoFix) {
+        const scripts = {
+          'lint': 'eslint .',
+          'lint:fix': 'eslint --fix .',
+        };
+        parsedPackageJSON.scripts = scripts;
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+
+    if ((packageJSON.toLowerCase().indexOf('engines') === -1) || !parsedPackageJSON.engines.node) {
+      console.warn('No engines or node engine in package.json');
+      if (autoFix) {
+        const engines = {
+          node: '^10.17.0 || >=11.14.0',
+        };
+        parsedPackageJSON.engines = engines;
+        writePackageJson(parsedPackageJSON);
+      }
+    }
+  }
+
+  if (files.indexOf('package-lock.json') === -1) {
+    console.warn('package-lock.json not found');
+    if (!autoFix) {
+      console.warn('Run npm install in the plugin folder and commit the package-lock.json file.');
+    }
+  }
+
+  if (files.indexOf('readme') === -1 && files.indexOf('readme.md') === -1) {
+    console.warn('README.md file not found, please create');
+    if (autoFix) {
+      console.log('Autofixing missing README.md file');
+      console.log('please edit the README.md file further to include plugin specific details.');
+      let readme = fs.readFileSync('bin/plugins/lib/README.md', {encoding: 'utf8', flag: 'r'});
+      readme = readme.replace(/\[plugin_name\]/g, pluginName);
+      if (repository) {
+        const org = repository.split('/')[3];
+        const name = repository.split('/')[4];
+        readme = readme.replace(/\[org_name\]/g, org);
+        readme = readme.replace(/\[repo_url\]/g, name);
+        fs.writeFileSync(`${pluginPath}/README.md`, readme);
+      } else {
+        console.warn('Unable to find repository in package.json, aborting.');
+      }
+    }
+  }
+
+  if (files.indexOf('contributing') === -1 && files.indexOf('contributing.md') === -1) {
+    console.warn('CONTRIBUTING.md file not found, please create');
+    if (autoFix) {
+      console.log('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md ' +
+                  'file further to include plugin specific details.');
+      let contributing =
+          fs.readFileSync('bin/plugins/lib/CONTRIBUTING.md', {encoding: 'utf8', flag: 'r'});
+      contributing = contributing.replace(/\[plugin_name\]/g, pluginName);
+      fs.writeFileSync(`${pluginPath}/CONTRIBUTING.md`, contributing);
+    }
+  }
+
+
+  if (files.indexOf('readme') !== -1 && files.indexOf('readme.md') !== -1) {
+    const readme =
+        fs.readFileSync(`${pluginPath}/${readMeFileName}`, {encoding: 'utf8', flag: 'r'});
+    if (readme.toLowerCase().indexOf('license') === -1) {
+      console.warn('No license section in README');
+      if (autoFix) {
+        console.warn('Please add License section to README manually.');
+      }
+    }
+  }
+
+  if (files.indexOf('license') === -1 && files.indexOf('license.md') === -1) {
+    console.warn('LICENSE.md file not found, please create');
+    if (autoFix) {
+      console.log('Autofixing missing LICENSE.md file, including Apache 2 license.');
+      let license = fs.readFileSync('bin/plugins/lib/LICENSE.md', {encoding: 'utf8', flag: 'r'});
+      license = license.replace('[yyyy]', new Date().getFullYear());
+      license = license.replace('[name of copyright owner]', execSync('git config user.name'));
+      fs.writeFileSync(`${pluginPath}/LICENSE.md`, license);
+    }
+  }
+
+  let travisConfig = fs.readFileSync('bin/plugins/lib/travis.yml', {encoding: 'utf8', flag: 'r'});
+  travisConfig = travisConfig.replace(/\[plugin_name\]/g, pluginName);
+
+  if (files.indexOf('.travis.yml') === -1) {
+    console.warn('.travis.yml file not found, please create.  ' +
+                 '.travis.yml is used for automatically CI testing Etherpad.  ' +
+                 'It is useful to know if your plugin breaks another feature for example.');
+    // TODO: Make it check version of the .travis file to see if it needs an update.
+    if (autoFix) {
+      console.log('Autofixing missing .travis.yml file');
+      fs.writeFileSync(`${pluginPath}/.travis.yml`, travisConfig);
+      console.log('Travis file created, please sign into travis and enable this repository');
+    }
+  }
+  if (autoFix) {
+    // checks the file versioning of .travis and updates it to the latest.
+    const existingConfig =
+        fs.readFileSync(`${pluginPath}/.travis.yml`, {encoding: 'utf8', flag: 'r'});
+    const existingConfigLocation = existingConfig.indexOf('##ETHERPAD_TRAVIS_V=');
+    const existingValue =
+        parseInt(existingConfig.substr(existingConfigLocation + 20, existingConfig.length));
+
+    const newConfigLocation = travisConfig.indexOf('##ETHERPAD_TRAVIS_V=');
+    const newValue = parseInt(travisConfig.substr(newConfigLocation + 20, travisConfig.length));
+    if (existingConfigLocation === -1) {
+      console.warn('no previous .travis.yml version found so writing new.');
+      // we will write the newTravisConfig to the location.
+      fs.writeFileSync(`${pluginPath}/.travis.yml`, travisConfig);
+    } else if (newValue > existingValue) {
+      console.log('updating .travis.yml');
+      fs.writeFileSync(`${pluginPath}/.travis.yml`, travisConfig);
+    }//
+  }
+
+  if (files.indexOf('.gitignore') === -1) {
+    console.warn('.gitignore file not found, please create.  .gitignore files are useful to ' +
+                 "ensure files aren't incorrectly commited to a repository.");
+    if (autoFix) {
+      console.log('Autofixing missing .gitignore file');
+      const gitignore = fs.readFileSync('bin/plugins/lib/gitignore', {encoding: 'utf8', flag: 'r'});
+      fs.writeFileSync(`${pluginPath}/.gitignore`, gitignore);
+    }
+  } else {
+    let gitignore =
+        fs.readFileSync(`${pluginPath}/.gitignore`, {encoding: 'utf8', flag: 'r'});
+    if (gitignore.indexOf('node_modules/') === -1) {
+      console.warn('node_modules/ missing from .gitignore');
+      if (autoFix) {
+        gitignore += 'node_modules/';
+        fs.writeFileSync(`${pluginPath}/.gitignore`, gitignore);
+      }
+    }
+  }
+
+  // if we include templates but don't have translations...
+  if (files.indexOf('templates') !== -1 && files.indexOf('locales') === -1) {
+    console.warn('Translations not found, please create.  ' +
+                 'Translation files help with Etherpad accessibility.');
+  }
+
+
+  if (files.indexOf('.ep_initialized') !== -1) {
+    console.warn(
+        '.ep_initialized found, please remove.  .ep_initialized should never be commited to git ' +
+        'and should only exist once the plugin has been executed one time.');
+    if (autoFix) {
+      console.log('Autofixing incorrectly existing .ep_initialized file');
+      fs.unlinkSync(`${pluginPath}/.ep_initialized`);
+    }
+  }
+
+  if (files.indexOf('npm-debug.log') !== -1) {
+    console.warn('npm-debug.log found, please remove.  npm-debug.log should never be commited to ' +
+                 'your repository.');
+    if (autoFix) {
+      console.log('Autofixing incorrectly existing npm-debug.log file');
+      fs.unlinkSync(`${pluginPath}/npm-debug.log`);
+    }
+  }
+
+  if (files.indexOf('static') !== -1) {
+    fs.readdir(`${pluginPath}/static`, (errRead, staticFiles) => {
+      if (staticFiles.indexOf('tests') === -1) {
+        console.warn('Test files not found, please create tests.  https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin');
+      }
+    });
+  } else {
+    console.warn('Test files not found, please create tests.  https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin');
+  }
+
+  // Install dependencies so we can run ESLint. This should also create or update package-lock.json
+  // if autoFix is enabled.
+  const npmInstall = `npm install${autoFix ? '' : ' --no-package-lock'}`;
+  execSync(npmInstall, {stdio: 'inherit'});
+  // The ep_etherpad-lite peer dep must be installed last otherwise `npm install` will nuke it. An
+  // absolute path to etherpad-lite/src is used here so that pluginPath can be a symlink.
+  execSync(
+      `${npmInstall} --no-save ep_etherpad-lite@file:${__dirname}/../../`, {stdio: 'inherit'});
+
+  // linting begins
+  try {
+    console.log('Linting...');
+    const lintCmd = autoFix ? 'npx eslint --fix .' : 'npx eslint';
+    execSync(lintCmd, {stdio: 'inherit'});
+  } catch (e) {
+    // it is gonna throw an error anyway
+    console.log('Manual linting probably required, check with: npm run lint');
+  }
+  // linting ends.
+
+  if (autoFix) {
+    const unchanged = JSON.parse(execSync(
+        'untracked=$(git ls-files -o --exclude-standard) || exit 1; ' +
+        'git diff-files --quiet && [ -z "$untracked" ] && echo true || echo false'));
+    if (!unchanged) {
+      // Display a diff of changes. Git doesn't diff untracked files, so they must be added to the
+      // index. Use a temporary index file to avoid modifying Git's default index file.
+      execSync('git read-tree HEAD; git add -A && git diff-index -p --cached HEAD && echo ""', {
+        env: {...process.env, GIT_INDEX_FILE: '.git/checkPlugin.index'},
+        stdio: 'inherit',
+      });
+      fs.unlinkSync(`${pluginPath}/.git/checkPlugin.index`);
+
+      const cmd = [
+        'git add -A',
+        'git commit -m "autofixes from Etherpad checkPlugin.js"',
+        'git push',
+      ].join(' && ');
+      if (autoCommit) {
+        console.log('Attempting autocommit and auto publish to npm');
+        execSync(cmd, {stdio: 'inherit'});
+      } else {
+        console.log('Fixes applied. Check the above git diff then run the following command:');
+        console.log(`(cd node_modules/${pluginName} && ${cmd})`);
+      }
+    } else {
+      console.log('No changes.');
+    }
+  }
+
+  console.log('Finished');
+});

src/bin/plugins/getCorePlugins.sh

@@ -0,0 +1,4 @@
+cd node_modules/
+GHUSER=ether; curl "https://api.github.com/users/$GHUSER/repos?per_page=100" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=ether; curl "https://api.github.com/users/$GHUSER/repos?per_page=100&page=2&" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=ether; curl "https://api.github.com/users/$GHUSER/repos?per_page=100&page=3&" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone

src/bin/plugins/lib/CONTRIBUTING.md

@@ -0,0 +1,133 @@
+# Contributor Guidelines
+(Please talk to people on the mailing list before you change this page, see our section on [how to get in touch](https://github.com/ether/etherpad-lite#get-in-touch))
+
+## Pull requests
+
+* the commit series in the PR should be _linear_ (it **should not contain merge commits**). This is necessary because we want to be able to [bisect](https://en.wikipedia.org/wiki/Bisection_(software_engineering)) bugs easily. Rewrite history/perform a rebase if necessary
+* PRs should be issued against the **develop** branch: we never pull directly into **master**
+* PRs **should not have conflicts** with develop. If there are, please resolve them rebasing and force-pushing
+* when preparing your PR, please make sure that you have included the relevant **changes to the documentation** (preferably with usage examples)
+* contain meaningful and detailed **commit messages** in the form:
+  ```
+  submodule: description
+
+  longer description of the change you have made, eventually mentioning the
+  number of the issue that is being fixed, in the form: Fixes #someIssueNumber
+  ```
+* if the PR is a **bug fix**:
+  * the first commit in the series must be a test that shows the failure
+  * subsequent commits will fix the bug and make the test pass
+  * the final commit message should include the text `Fixes: #xxx` to link it to its bug report
+* think about stability: code has to be backwards compatible as much as possible. Always **assume your code will be run with an older version of the DB/config file**
+* if you want to remove a feature, **deprecate it instead**:
+  * write an issue with your deprecation plan
+  * output a `WARN` in the log informing that the feature is going to be removed
+  * remove the feature in the next version
+* if you want to add a new feature, put it under a **feature flag**:
+  * once the new feature has reached a minimal level of stability, do a PR for it, so it can be integrated early
+  * expose a mechanism for enabling/disabling the feature
+  * the new feature should be **disabled** by default. With the feature disabled, the code path should be exactly the same as before your contribution. This is a __necessary condition__ for early integration
+* think of the PR not as something that __you wrote__, but as something that __someone else is going to read__. The commit series in the PR should tell a novice developer the story of your thoughts when developing it
+
+## How to write a bug report
+
+* Please be polite, we all are humans and problems can occur.
+* Please add as much information as possible, for example
+  * client os(s) and version(s)
+    * browser(s) and version(s), is the problem reproducible on different clients
+    * special environments like firewalls or antivirus
+  * host os and version
+    * npm and nodejs version
+    * Logfiles if available
+  * steps to reproduce
+  * what you expected to happen
+  * what actually happened
+* Please format logfiles and code examples with markdown see github Markdown help below the issue textarea for more information.
+
+If you send logfiles, please set the loglevel switch DEBUG in your settings.json file:
+
+```
+/* The log level we are using, can be: DEBUG, INFO, WARN, ERROR */
+  "loglevel": "DEBUG",
+```
+
+The logfile location is defined in startup script or the log is directly shown in the commandline after you have started etherpad.
+
+## General goals of Etherpad
+To make sure everybody is going in the same direction:
+* easy to install for admins and easy to use for people
+* easy to integrate into other apps, but also usable as standalone
+* lightweight and scalable
+* extensible, as much functionality should be extendable with plugins so changes don't have to be done in core.
+Also, keep it maintainable. We don't wanna end up as the monster Etherpad was!
+
+## How to work with git?
+* Don't work in your master branch.
+* Make a new branch for every feature you're working on. (This ensures that you can work you can do lots of small, independent pull requests instead of one big one with complete different features)
+* Don't use the online edit function of github (this only creates ugly and not working commits!)
+* Try to make clean commits that are easy readable (including descriptive commit messages!)
+* Test before you push. Sounds easy, it isn't!
+* Don't check in stuff that gets generated during build or runtime
+* Make small pull requests that are easy to review but make sure they do add value by themselves / individually
+
+## Coding style
+* Do write comments. (You don't have to comment every line, but if you come up with something that's a bit complex/weird, just leave a comment. Bear in mind that you will probably leave the project at some point and that other people will read your code. Undocumented huge amounts of code are worthless!)
+* Never ever use tabs
+* Indentation: JS/CSS: 2 spaces; HTML: 4 spaces
+* Don't overengineer. Don't try to solve any possible problem in one step, but try to solve problems as easy as possible and improve the solution over time!
+* Do generalize sooner or later! (if an old solution, quickly hacked together, poses more problems than it solves today, refactor it!)
+* Keep it compatible. Do not introduce changes to the public API, db schema or configurations too lightly. Don't make incompatible changes without good reasons!
+* If you do make changes, document them! (see below)
+* Use protocol independent urls "//"
+
+## Branching model / git workflow
+see git flow http://nvie.com/posts/a-successful-git-branching-model/
+
+### `master` branch
+* the stable
+* This is the branch everyone should use for production stuff
+
+### `develop`branch
+* everything that is READY to go into master at some point in time
+* This stuff is tested and ready to go out
+
+### release branches
+* stuff that should go into master very soon
+* only bugfixes go into these (see http://nvie.com/posts/a-successful-git-branching-model/ for why)
+* we should not be blocking new features to develop, just because we feel that we should be releasing it to master soon. This is the situation that release branches solve/handle.
+
+### hotfix branches
+* fixes for bugs in master
+
+### feature branches (in your own repos)
+* these are the branches where you develop your features in
+* If it's ready to go out, it will be merged into develop
+
+Over the time we pull features from feature branches into the develop branch. Every month we pull from develop into master. Bugs in master get fixed in hotfix branches. These branches will get merged into master AND develop. There should never be commits in master that aren't in develop
+
+## Documentation
+The docs are in the `doc/` folder in the git repository, so people can easily find the suitable docs for the current git revision.
+
+Documentation should be kept up-to-date. This means, whenever you add a new API method, add a new hook or change the database model, pack the relevant changes to the docs in the same pull request.
+
+You can build the docs e.g. produce html, using `make docs`. At some point in the future we will provide an online documentation. The current documentation in the github wiki should always reflect the state of `master` (!), since there are no docs in master, yet.
+
+## Testing
+Front-end tests are found in the `tests/frontend/` folder in the repository. Run them by pointing your browser to `<yourdomainhere>/tests/frontend`.
+
+Back-end tests can be run from the `src` directory, via `npm test`.
+
+## Things you can help with
+Etherpad is much more than software.  So if you aren't a developer then worry not, there is still a LOT you can do!  A big part of what we do is community engagement.  You can help in the following ways
+ * Triage bugs (applying labels) and confirming their existence
+ * Testing fixes (simply applying them and seeing if it fixes your issue or not) - Some git experience required
+ * Notifying large site admins of new releases
+ * Writing Changelogs for releases
+ * Creating Windows packages
+ * Creating releases
+ * Bumping dependencies periodically and checking they don't break anything
+ * Write proposals for grants
+ * Co-Author and Publish CVEs
+ * Work with SFC to maintain legal side of project
+ * Maintain TODO page - https://github.com/ether/etherpad-lite/wiki/TODO#IMPORTANT_TODOS
+

src/bin/plugins/lib/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

src/bin/plugins/lib/README.md

@@ -0,0 +1,29 @@
+[![Travis (.com)](https://api.travis-ci.com/[org_name]/[repo_url].svg?branch=develop)](https://travis-ci.com/github/[org_name]/[repo_url])
+
+# My awesome plugin README example
+Explain what your plugin does and who it's useful for.
+
+## Example animated gif of usage if appropriate
+![screenshot](https://user-images.githubusercontent.com/220864/99979953-97841d80-2d9f-11eb-9782-5f65817c58f4.PNG)
+
+## Installing
+npm install [plugin_name]
+
+or Use the Etherpad ``/admin`` interface.
+
+## Settings
+Document settings if any
+
+## Testing
+Document how to run backend / frontend tests.
+
+### Frontend
+
+Visit http://whatever/tests/frontend/ to run the frontend tests.
+
+### backend
+
+Type ``cd src && npm run test`` to run the backend tests.
+
+## LICENSE
+Apache 2.0

src/bin/plugins/lib/backend-tests.yml

@@ -0,0 +1,51 @@
+# You need to change lines 38 and 46 in case the plugin's name on npmjs.com is different
+# from the repository name
+
+name: "Backend tests"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  withplugins:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: with Plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Install libreoffice
+      run: |
+        sudo add-apt-repository -y ppa:libreoffice/ppa
+        sudo apt update
+        sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+
+    # clone etherpad-lite
+    - name: Install etherpad core
+      uses: actions/checkout@v2
+      with:
+        repository: ether/etherpad-lite
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    # clone this repository into node_modules/ep_plugin-name
+    - name: Checkout plugin repository
+      uses: actions/checkout@v2
+      with:
+        path: ./node_modules/${{github.event.repository.name}}
+
+    - name: Install plugin dependencies
+      run: |
+        cd node_modules/${{github.event.repository.name}}
+        npm ci
+
+    # configures some settings and runs npm run test
+    - name: Run the backend tests
+      run: tests/frontend/travis/runnerBackend.sh
+
+##ETHERPAD_NPM_V=1
+## NPM configuration automatically created using bin/plugins/updateAllPluginsScript.sh

src/bin/plugins/lib/gitignore

@@ -0,0 +1,5 @@
+.ep_initialized
+.DS_Store
+node_modules/
+node_modules
+npm-debug.log

src/bin/plugins/lib/npmpublish.yml

@@ -0,0 +1,83 @@
+# This workflow will run tests using node and then publish a package to the npm registry when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: Node.js Package
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      # Clone ether/etherpad-lite to ../etherpad-lite so that ep_etherpad-lite
+      # can be "installed" in this plugin's node_modules. The checkout v2 action
+      # doesn't support cloning outside of $GITHUB_WORKSPACE (see
+      # https://github.com/actions/checkout/issues/197), so the repo is first
+      # cloned to etherpad-lite then moved to ../etherpad-lite. To avoid
+      # conflicts with this plugin's clone, etherpad-lite must be cloned and
+      # moved out before this plugin's repo is cloned to $GITHUB_WORKSPACE.
+      - uses: actions/checkout@v2
+        with:
+          repository: ether/etherpad-lite
+          path: etherpad-lite
+      - run: mv etherpad-lite ..
+      # etherpad-lite has been moved outside of $GITHUB_WORKSPACE, so it is now
+      # safe to clone this plugin's repo to $GITHUB_WORKSPACE.
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12
+      # All of ep_etherpad-lite's devDependencies are installed because the
+      # plugin might do `require('ep_etherpad-lite/node_modules/${devDep}')`.
+      # Eventually it would be nice to create an ESLint plugin that prohibits
+      # Etherpad plugins from piggybacking off of ep_etherpad-lite's
+      # devDependencies. If we had that, we could change this line to only
+      # install production dependencies.
+      - run: cd ../etherpad-lite/src && npm ci
+      - run: npm ci
+      # This runs some sanity checks and creates a symlink at
+      # node_modules/ep_etherpad-lite that points to ../../etherpad-lite/src.
+      # This step must be done after `npm ci` installs the plugin's dependencies
+      # because npm "helpfully" cleans up such symlinks. :( Installing
+      # ep_etherpad-lite in the plugin's node_modules prevents lint errors and
+      # unit test failures if the plugin does `require('ep_etherpad-lite/foo')`.
+      - run: npm install --no-save ep_etherpad-lite@file:../etherpad-lite/src
+      - run: npm test
+      - run: npm run lint
+
+  publish-npm:
+    if: github.event_name == 'push'
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12
+          registry-url: https://registry.npmjs.org/
+      - run: git config user.name 'github-actions[bot]'
+      - run: git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+      - run: npm ci
+      - run: npm version patch
+      - run: git push --follow-tags
+      # `npm publish` must come after `git push` otherwise there is a race
+      # condition: If two PRs are merged back-to-back then master/main will be
+      # updated with the commits from the second PR before the first PR's
+      # workflow has a chance to push the commit generated by `npm version
+      # patch`. This causes the first PR's `git push` step to fail after the
+      # package has already been published, which in turn will cause all future
+      # workflow runs to fail because they will all attempt to use the same
+      # already-used version number. By running `npm publish` after `git push`,
+      # back-to-back merges will cause the first merge's workflow to fail but
+      # the second's will succeed.
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
+
+##ETHERPAD_NPM_V=2
+## NPM configuration automatically created using bin/plugins/updateAllPluginsScript.sh

src/bin/plugins/lib/travis.yml

@@ -0,0 +1,70 @@
+language: node_js
+
+node_js:
+  - "lts/*"
+
+cache: false
+
+services:
+  - docker
+
+install:
+  - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+
+#script:
+#  - "tests/frontend/travis/runner.sh"
+
+env:
+  global:
+    - secure: "WMGxFkOeTTlhWB+ChMucRtIqVmMbwzYdNHuHQjKCcj8HBEPdZLfCuK/kf4rG\nVLcLQiIsyllqzNhBGVHG1nyqWr0/LTm8JRqSCDDVIhpyzp9KpCJQQJG2Uwjk\n6/HIJJh/wbxsEdLNV2crYU/EiVO3A4Bq0YTHUlbhUqG3mSCr5Ec="
+    - secure: "gejXUAHYscbR6Bodw35XexpToqWkv2ifeECsbeEmjaLkYzXmUUNWJGknKSu7\nEUsSfQV8w+hxApr1Z+jNqk9aX3K1I4btL3cwk2trnNI8XRAvu1c1Iv60eerI\nkE82Rsd5lwUaMEh+/HoL8ztFCZamVndoNgX7HWp5J/NRZZMmh4g="
+
+jobs:
+  include:
+    - name: "Lint test package-lock"
+      install:
+        - "npm install lockfile-lint"
+      script:
+        - npx lockfile-lint --path package-lock.json --validate-https --allowed-hosts npm
+    - name: "Run the Backend tests"
+      before_install:
+        - sudo add-apt-repository -y ppa:libreoffice/ppa
+        - sudo apt-get update
+        - sudo apt-get -y install libreoffice
+        - sudo apt-get -y install libreoffice-pdfimport
+      install:
+        - "npm install"
+        - "mkdir [plugin_name]"
+        - "mv !([plugin_name]) [plugin_name]"
+        - "git clone https://github.com/ether/etherpad-lite.git etherpad"
+        - "cd etherpad"
+        - "mkdir -p node_modules"
+        - "mv ../[plugin_name] node_modules"
+        - "bin/installDeps.sh"
+        - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+        - "cd src && npm install && cd -"
+      script:
+        - "tests/frontend/travis/runnerBackend.sh"
+    - name: "Test the Frontend"
+      before_script:
+        - "tests/frontend/travis/sauce_tunnel.sh"
+      install:
+        - "npm install"
+        - "mkdir [plugin_name]"
+        - "mv !([plugin_name]) [plugin_name]"
+        - "git clone https://github.com/ether/etherpad-lite.git etherpad"
+        - "cd etherpad"
+        - "mkdir -p node_modules"
+        - "mv ../[plugin_name] node_modules"
+        - "bin/installDeps.sh"
+        - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+      script:
+        - "tests/frontend/travis/runner.sh"
+
+notifications:
+  irc:
+    channels:
+      - "irc.freenode.org#etherpad-lite-dev"
+
+##ETHERPAD_TRAVIS_V=9
+## Travis configuration automatically created using bin/plugins/updateAllPluginsScript.sh

src/bin/plugins/reTestAllPlugins.sh

@@ -0,0 +1,14 @@
+echo "herp";
+for dir in `ls node_modules`;
+do
+  echo $dir
+  if [[ $dir == *"ep_"* ]]; then
+    if [[ $dir != "ep_etherpad-lite" ]]; then
+      # node bin/plugins/checkPlugin.js $dir autofix autocommit autoupdate
+      cd node_modules/$dir
+      git commit -m "Automatic update: bump update to re-run latest Etherpad tests" --allow-empty
+      git push origin master
+      cd ../..
+    fi
+  fi
+done

src/bin/plugins/updateAllPluginsScript.sh

@@ -0,0 +1,17 @@
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=2" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=3" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=4" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+for dir in `ls node_modules`;
+do
+  # echo $0
+  if [[ $dir == *"ep_"* ]]; then
+    if [[ $dir != "ep_etherpad-lite" ]]; then
+      node bin/plugins/checkPlugin.js $dir autofix autocommit autoupdate
+    fi
+  fi
+  # echo $dir
+done

src/bin/plugins/updateCorePlugins.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+for dir in node_modules/ep_*; do
+  dir=${dir#node_modules/}
+  [ "$dir" != ep_etherpad-lite ] || continue
+  node bin/plugins/checkPlugin.js "$dir" autofix autocommit autoupdate
+done

src/bin/rebuildPad.js

@@ -0,0 +1,84 @@
+'use strict';
+
+/*
+  This is a repair tool. It rebuilds an old pad at a new pad location up to a
+  known "good" revision.
+*/
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+if (process.argv.length !== 4 && process.argv.length !== 5) {
+  throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]');
+}
+
+const padId = process.argv[2];
+const newRevHead = process.argv[3];
+const newPadId = process.argv[4] || `${padId}-rebuilt`;
+
+(async () => {
+  const db = require('../node/db/DB');
+  await db.init();
+
+  const PadManager = require('../node/db/PadManager');
+  const Pad = require('../node/db/Pad').Pad;
+  // Validate the newPadId if specified and that a pad with that ID does
+  // not already exist to avoid overwriting it.
+  if (!PadManager.isValidPadId(newPadId)) {
+    throw new Error('Cannot create a pad with that id as it is invalid');
+  }
+  const exists = await PadManager.doesPadExist(newPadId);
+  if (exists) throw new Error('Cannot create a pad with that id as it already exists');
+
+  const oldPad = await PadManager.getPad(padId);
+  const newPad = new Pad(newPadId);
+
+  // Clone all Chat revisions
+  const chatHead = oldPad.chatHead;
+  await Promise.all([...Array(chatHead + 1).keys()].map(async (i) => {
+    const chat = await db.get(`pad:${padId}:chat:${i}`);
+    await db.set(`pad:${newPadId}:chat:${i}`, chat);
+    console.log(`Created: Chat Revision: pad:${newPadId}:chat:${i}`);
+  }));
+
+  // Rebuild Pad from revisions up to and including the new revision head
+  const AuthorManager = require('../node/db/AuthorManager');
+  const Changeset = require('../static/js/Changeset');
+  // Author attributes are derived from changesets, but there can also be
+  // non-author attributes with specific mappings that changesets depend on
+  // and, AFAICT, cannot be recreated any other way
+  newPad.pool.numToAttrib = oldPad.pool.numToAttrib;
+  for (let curRevNum = 0; curRevNum <= newRevHead; curRevNum++) {
+    const rev = await db.get(`pad:${padId}:revs:${curRevNum}`);
+    if (!rev || !rev.meta) throw new Error('The specified revision number could not be found.');
+    const newRevNum = ++newPad.head;
+    const newRevId = `pad:${newPad.id}:revs:${newRevNum}`;
+    await Promise.all([
+      db.set(newRevId, rev),
+      AuthorManager.addPad(rev.meta.author, newPad.id),
+    ]);
+    newPad.atext = Changeset.applyToAText(rev.changeset, newPad.atext, newPad.pool);
+    console.log(`Created: Revision: pad:${newPad.id}:revs:${newRevNum}`);
+  }
+
+  // Add saved revisions up to the new revision head
+  console.log(newPad.head);
+  const newSavedRevisions = [];
+  for (const savedRev of oldPad.savedRevisions) {
+    if (savedRev.revNum <= newRevHead) {
+      newSavedRevisions.push(savedRev);
+      console.log(`Added: Saved Revision: ${savedRev.revNum}`);
+    }
+  }
+  newPad.savedRevisions = newSavedRevisions;
+
+  // Save the source pad
+  await db.set(`pad:${newPadId}`, newPad);
+
+  console.log(`Created: Source Pad: pad:${newPadId}`);
+  await newPad.saveToDatabase();
+
+  await db.shutdown();
+  console.info('finished');
+})();

src/bin/release.js

@@ -0,0 +1,74 @@
+'use strict';
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+const fs = require('fs');
+const childProcess = require('child_process');
+const semver = require('semver');
+
+/*
+
+Usage
+
+node bin/release.js patch
+
+*/
+const usage = 'node bin/release.js [patch/minor/major] -- example: "node bin/release.js patch"';
+
+const release = process.argv[2];
+
+if (!release) {
+  console.log(usage);
+  throw new Error('No release type included');
+}
+
+const changelog = fs.readFileSync('CHANGELOG.md', {encoding: 'utf8', flag: 'r'});
+let packageJson = fs.readFileSync('./src/package.json', {encoding: 'utf8', flag: 'r'});
+packageJson = JSON.parse(packageJson);
+const currentVersion = packageJson.version;
+
+const newVersion = semver.inc(currentVersion, release);
+if (!newVersion) {
+  console.log(usage);
+  throw new Error('Unable to generate new version from input');
+}
+
+const changelogIncludesVersion = changelog.indexOf(newVersion) !== -1;
+
+if (!changelogIncludesVersion) {
+  throw new Error('No changelog record for ', newVersion, ' - please create changelog record');
+}
+
+console.log('Okay looks good, lets create the package.json and package-lock.json');
+
+packageJson.version = newVersion;
+
+fs.writeFileSync('src/package.json', JSON.stringify(packageJson, null, 2));
+
+// run npm version `release` where release is patch, minor or major
+childProcess.execSync('npm install --package-lock-only', {cwd: 'src/'});
+// run npm install --package-lock-only <-- required???
+
+childProcess.execSync(`git checkout -b release/${newVersion}`);
+childProcess.execSync('git add src/package.json');
+childProcess.execSync('git add src/package-lock.json');
+childProcess.execSync('git commit -m "bump version"');
+childProcess.execSync(`git push origin release/${newVersion}`);
+
+
+childProcess.execSync('make docs');
+childProcess.execSync('git clone git@github.com:ether/ether.github.com.git');
+childProcess.execSync(`cp -R out/doc/ ether.github.com/doc/v${newVersion}`);
+
+console.log('Once merged into master please run the following commands');
+console.log(`git tag -a ${newVersion} -m ${newVersion} && git push origin master`);
+console.log(`cd ether.github.com && git add . && git commit -m '${newVersion} docs'`);
+console.log('Build the windows zip');
+console.log('Visit https://github.com/ether/etherpad-lite/releases/new and create a new release ' +
+            `with 'master' as the target and the version is ${newVersion}.  Include the windows ` +
+            'zip as an asset');
+console.log(`Once the new docs are uploaded then modify the download
+   link on etherpad.org and then pull master onto develop`);
+console.log('Finally go public with an announcement via our comms channels :)');

src/bin/repairPad.js

@@ -0,0 +1,56 @@
+'use strict';
+
+/*
+ * This is a repair tool. It extracts all datas of a pad, removes and inserts them again.
+ */
+
+// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
+// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
+process.on('unhandledRejection', (err) => { throw err; });
+
+console.warn('WARNING: This script must not be used while etherpad is running!');
+
+if (process.argv.length !== 3) throw new Error('Use: node bin/repairPad.js $PADID');
+
+// get the padID
+const padId = process.argv[2];
+
+let valueCount = 0;
+
+(async () => {
+  // initialize database
+  require('../node/utils/Settings');
+  const db = require('../node/db/DB');
+  await db.init();
+
+  // get the pad
+  const padManager = require('../node/db/PadManager');
+  const pad = await padManager.getPad(padId);
+
+  // accumulate the required keys
+  const neededDBValues = [`pad:${padId}`];
+
+  // add all authors
+  neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`));
+
+  // add all revisions
+  for (let rev = 0; rev <= pad.head; ++rev) {
+    neededDBValues.push(`pad:${padId}:revs:${rev}`);
+  }
+
+  // add all chat values
+  for (let chat = 0; chat <= pad.chatHead; ++chat) {
+    neededDBValues.push(`pad:${padId}:chat:${chat}`);
+  }
+  // now fetch and reinsert every key
+  for (const key of neededDBValues) {
+    const value = await db.get(key);
+    // if it isn't a globalAuthor value which we want to ignore..
+    // console.log(`Key: ${key}, value: ${JSON.stringify(value)}`);
+    await db.remove(key);
+    await db.set(key, value);
+    valueCount++;
+  }
+
+  console.info(`Finished: Replaced ${valueCount} values in the database`);
+})();

src/bin/run.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Source constants and useful functions
+. bin/functions.sh
+
+ignoreRoot=0
+for ARG in "$@"; do
+  if [ "$ARG" = "--root" ]; then
+    ignoreRoot=1
+  fi
+done
+
+# Stop the script if it's started as root
+if [ "$(id -u)" -eq 0 ] && [ "$ignoreRoot" -eq 0 ]; then
+  cat <<EOF >&2
+You shouldn't start Etherpad as root!
+Please type 'Etherpad rocks my socks' (or restart with the '--root'
+argument) if you still want to start it as root:
+EOF
+  printf "> " >&2
+  read rocks
+  [ "$rocks" = "Etherpad rocks my socks" ] || fatal "Your input was incorrect"
+fi
+
+# Prepare the environment
+bin/installDeps.sh "$@" || exit 1
+
+# Move to the node folder and start
+log "Starting Etherpad..."
+
+SCRIPTPATH=$(pwd -P)
+exec node $(compute_node_args) "$SCRIPTPATH/node_modules/ep_etherpad-lite/node/server.js" "$@"

src/bin/safeRun.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+# This script ensures that ep-lite is automatically restarting after
+# an error happens
+
+# Handling Errors
+#   0 silent
+#   1 email
+ERROR_HANDLING=0
+# Your email address which should receive the error messages
+EMAIL_ADDRESS="no-reply@example.com"
+# Sets the minimum amount of time between the sending of error emails.
+# This ensures you do not get spammed during an endless reboot loop
+# It's the time in seconds
+TIME_BETWEEN_EMAILS=600 # 10 minutes
+
+# DON'T EDIT AFTER THIS LINE
+
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+
+LAST_EMAIL_SEND=0
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Check if a logfile parameter is set
+LOG="$1"
+[ -n "${LOG}" ] || fatal "Set a logfile as the first parameter"
+shift
+
+while true; do
+  # Try to touch the file if it doesn't exist
+  [ -f "${LOG}" ] || touch "${LOG}" || fatal "Logfile '${LOG}' is not writeable"
+
+  # Check if the file is writeable
+  [ -w "${LOG}" ] || fatal "Logfile '${LOG}' is not writeable"
+
+  # Start the application
+  bin/run.sh "$@" >>${LOG} 2>>${LOG}
+
+  TIME_FMT=$(date +%Y-%m-%dT%H:%M:%S%z)
+
+  # Send email
+  if [ "$ERROR_HANDLING" = 1 ]; then
+    TIME_NOW=$(date +%s)
+    TIME_SINCE_LAST_SEND=$(($TIME_NOW - $LAST_EMAIL_SEND))
+
+    if [ "$TIME_SINCE_LAST_SEND" -gt "$TIME_BETWEEN_EMAILS" ]; then
+      {
+        cat <<EOF
+Server was restarted at: ${TIME_FMT}
+The last 50 lines of the log before the server exited:
+
+EOF
+        tail -n 50 "${LOG}"
+      } | mail -s "Etherpad restarted" "$EMAIL_ADDRESS"
+
+      LAST_EMAIL_SEND=$TIME_NOW
+    fi
+  fi
+
+  pecho "RESTART! ${TIME_FMT}" >>${LOG}
+
+  # Sleep 10 seconds before restart
+  sleep 10
+done

src/bin/updatePlugins.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+#Move to the folder where ep-lite is installed
+cd $(dirname $0)
+
+#Was this script started in the bin folder? if yes move out
+if [ -d "../bin" ]; then
+  cd "../"
+fi
+
+# npm outdated --depth=0 | grep -v "^Package" | awk '{print $1}' | xargs npm install $1 --save-dev
+OUTDATED=$(npm outdated --depth=0 | grep -v "^Package" | awk '{print $1}')
+# echo $OUTDATED
+if test -n "$OUTDATED"; then
+  echo "Plugins require update, doing this now..."
+  echo "Updating $OUTDATED"
+  npm install $OUTDATED --save-dev
+else
+  echo "Plugins are all up to date"
+fi