Merge branch 'master' into develop

src/node/db/Pad.js

@@ -7,6 +7,7 @@
 const Changeset = require('../../static/js/Changeset');
 const ChatMessage = require('../../static/js/ChatMessage');
 const AttributePool = require('../../static/js/AttributePool');
+const assert = require('assert').strict;
 const db = require('./DB');
 const settings = require('../utils/Settings');
 const authorManager = require('./AuthorManager');
@@ -20,7 +21,7 @@ const hooks = require('../../static/js/pluginfw/hooks');
 const promises = require('../utils/promises');
 
 // serialization/deserialization attributes
-const attributeBlackList = ['id'];
+const attributeBlackList = ['_db', 'id'];
 const jsonableList = ['pool'];
 
 /**
@@ -33,7 +34,15 @@ exports.cleanText = (txt) => txt.replace(/\r\n/g, '\n')
     .replace(/\t/g, '        ')
     .replace(/\xa0/g, ' ');
 
-const Pad = function (id) {
+/**
+ * @param [database] - Database object to access this pad's records (and only this pad's records --
+ *     the shared global Etherpad database object is still used for all other pad accesses, such as
+ *     copying the pad). Defaults to the shared global Etherpad database object. This parameter can
+ *     be used to shard pad storage across multiple database backends, to put each pad in its own
+ *     database table, or to validate imported pad data before it is written to the database.
+ */
+const Pad = function (id, database = db) {
+  this._db = database;
   this.atext = Changeset.makeAText('\n');
   this.pool = new AttributePool();
   this.head = -1;
@@ -94,7 +103,7 @@ Pad.prototype.appendRevision = async function (aChangeset, author) {
   }
 
   const p = [
-    db.set(`pad:${this.id}:revs:${newRev}`, newRevData),
+    this._db.set(`pad:${this.id}:revs:${newRev}`, newRevData),
     this.saveToDatabase(),
   ];
 
@@ -127,25 +136,25 @@ Pad.prototype.saveToDatabase = async function () {
     }
   }
 
-  await db.set(`pad:${this.id}`, dbObject);
+  await this._db.set(`pad:${this.id}`, dbObject);
 };
 
 // get time of last edit (changeset application)
-Pad.prototype.getLastEdit = function () {
+Pad.prototype.getLastEdit = async function () {
   const revNum = this.getHeadRevisionNumber();
-  return db.getSub(`pad:${this.id}:revs:${revNum}`, ['meta', 'timestamp']);
+  return await this._db.getSub(`pad:${this.id}:revs:${revNum}`, ['meta', 'timestamp']);
 };
 
-Pad.prototype.getRevisionChangeset = function (revNum) {
-  return db.getSub(`pad:${this.id}:revs:${revNum}`, ['changeset']);
+Pad.prototype.getRevisionChangeset = async function (revNum) {
+  return await this._db.getSub(`pad:${this.id}:revs:${revNum}`, ['changeset']);
 };
 
-Pad.prototype.getRevisionAuthor = function (revNum) {
-  return db.getSub(`pad:${this.id}:revs:${revNum}`, ['meta', 'author']);
+Pad.prototype.getRevisionAuthor = async function (revNum) {
+  return await this._db.getSub(`pad:${this.id}:revs:${revNum}`, ['meta', 'author']);
 };
 
-Pad.prototype.getRevisionDate = function (revNum) {
-  return db.getSub(`pad:${this.id}:revs:${revNum}`, ['meta', 'timestamp']);
+Pad.prototype.getRevisionDate = async function (revNum) {
+  return await this._db.getSub(`pad:${this.id}:revs:${revNum}`, ['meta', 'timestamp']);
 };
 
 Pad.prototype.getAllAuthors = function () {
@@ -172,7 +181,7 @@ Pad.prototype.getInternalRevisionAText = async function (targetRev) {
   // get all needed data out of the database
 
   // start to get the atext of the key revision
-  const p_atext = db.getSub(`pad:${this.id}:revs:${keyRev}`, ['meta', 'atext']);
+  const p_atext = this._db.getSub(`pad:${this.id}:revs:${keyRev}`, ['meta', 'atext']);
 
   // get all needed changesets
   const changesets = [];
@@ -195,8 +204,8 @@ Pad.prototype.getInternalRevisionAText = async function (targetRev) {
   return atext;
 };
 
-Pad.prototype.getRevision = function (revNum) {
-  return db.get(`pad:${this.id}:revs:${revNum}`);
+Pad.prototype.getRevision = async function (revNum) {
+  return await this._db.get(`pad:${this.id}:revs:${revNum}`);
 };
 
 Pad.prototype.getAllAuthorColors = async function () {
@@ -292,7 +301,7 @@ Pad.prototype.appendChatMessage = async function (msgOrText, authorId = null, ti
     // Don't save the display name in the database because the user can change it at any time. The
     // `displayName` property will be populated with the current value when the message is read from
     // the database.
-    db.set(`pad:${this.id}:chat:${this.chatHead}`, {...msg, displayName: undefined}),
+    this._db.set(`pad:${this.id}:chat:${this.chatHead}`, {...msg, displayName: undefined}),
     this.saveToDatabase(),
   ]);
 };
@@ -302,7 +311,7 @@ Pad.prototype.appendChatMessage = async function (msgOrText, authorId = null, ti
  * @returns {?ChatMessage}
  */
 Pad.prototype.getChatMessage = async function (entryNum) {
-  const entry = await db.get(`pad:${this.id}:chat:${entryNum}`);
+  const entry = await this._db.get(`pad:${this.id}:chat:${entryNum}`);
   if (entry == null) return null;
   const message = ChatMessage.fromObject(entry);
   message.displayName = await authorManager.getAuthorName(message.authorId);
@@ -339,7 +348,7 @@ Pad.prototype.init = async function (text) {
   }
 
   // try to load the pad
-  const value = await db.get(`pad:${this.id}`);
+  const value = await this._db.get(`pad:${this.id}`);
 
   // if this pad exists, load it
   if (value != null) {
@@ -357,13 +366,9 @@ Pad.prototype.init = async function (text) {
 
     await this.appendRevision(firstChangeset, '');
   }
-
-  hooks.callAll('padLoad', {pad: this});
 };
 
 Pad.prototype.copy = async function (destinationID, force) {
-  const sourceID = this.id;
-
   // Kick everyone from this pad.
   // This was commented due to https://github.com/ether/etherpad-lite/issues/3183.
   // Do we really need to kick everyone out?
@@ -379,7 +384,7 @@ Pad.prototype.copy = async function (destinationID, force) {
   await this.removePadIfForceIsTrueAndAlreadyExist(destinationID, force);
 
   // copy the 'pad' entry
-  const pad = await db.get(`pad:${sourceID}`);
+  const pad = await this._db.get(`pad:${this.id}`);
   db.set(`pad:${destinationID}`, pad);
 
   // copy all relations in parallel
@@ -388,7 +393,7 @@ Pad.prototype.copy = async function (destinationID, force) {
   // copy all chat messages
   const chatHead = this.chatHead;
   for (let i = 0; i <= chatHead; ++i) {
-    const p = db.get(`pad:${sourceID}:chat:${i}`)
+    const p = this._db.get(`pad:${this.id}:chat:${i}`)
         .then((chat) => db.set(`pad:${destinationID}:chat:${i}`, chat));
     promises.push(p);
   }
@@ -396,7 +401,7 @@ Pad.prototype.copy = async function (destinationID, force) {
   // copy all revisions
   const revHead = this.head;
   for (let i = 0; i <= revHead; ++i) {
-    const p = db.get(`pad:${sourceID}:revs:${i}`)
+    const p = this._db.get(`pad:${this.id}:revs:${i}`)
         .then((rev) => db.set(`pad:${destinationID}:revs:${i}`, rev));
     promises.push(p);
   }
@@ -552,12 +557,12 @@ Pad.prototype.remove = async function () {
 
   // delete all chat messages
   p.push(promises.timesLimit(this.chatHead + 1, 500, async (i) => {
-    await db.remove(`pad:${padID}:chat:${i}`, null);
+    await this._db.remove(`pad:${this.id}:chat:${i}`, null);
   }));
 
   // delete all revisions
   p.push(promises.timesLimit(this.head + 1, 500, async (i) => {
-    await db.remove(`pad:${padID}:revs:${i}`, null);
+    await this._db.remove(`pad:${this.id}:revs:${i}`, null);
   }));
 
   // remove pad from all authors who contributed
@@ -601,3 +606,127 @@ Pad.prototype.addSavedRevision = async function (revNum, savedById, label) {
 Pad.prototype.getSavedRevisions = function () {
   return this.savedRevisions;
 };
+
+/**
+ * Asserts that all pad data is consistent. Throws if inconsistent.
+ */
+Pad.prototype.check = async function () {
+  assert(this.id != null);
+  assert.equal(typeof this.id, 'string');
+
+  const head = this.getHeadRevisionNumber();
+  assert(Number.isInteger(head));
+  assert(head >= -1);
+
+  const savedRevisionsList = this.getSavedRevisionsList();
+  assert(Array.isArray(savedRevisionsList));
+  assert.equal(this.getSavedRevisionsNumber(), savedRevisionsList.length);
+  let prevSavedRev = null;
+  for (const rev of savedRevisionsList) {
+    assert(Number.isInteger(rev));
+    assert(rev >= 0);
+    assert(rev <= head);
+    assert(prevSavedRev == null || rev > prevSavedRev);
+    prevSavedRev = rev;
+  }
+  const savedRevisions = this.getSavedRevisions();
+  assert(Array.isArray(savedRevisions));
+  assert.equal(savedRevisions.length, savedRevisionsList.length);
+  const savedRevisionsIds = new Set();
+  for (const savedRev of savedRevisions) {
+    assert(savedRev != null);
+    assert.equal(typeof savedRev, 'object');
+    assert(savedRevisionsList.includes(savedRev.revNum));
+    assert(savedRev.id != null);
+    assert.equal(typeof savedRev.id, 'string');
+    assert(!savedRevisionsIds.has(savedRev.id));
+    savedRevisionsIds.add(savedRev.id);
+  }
+
+  const pool = this.apool();
+  assert(pool instanceof AttributePool);
+  await pool.check();
+
+  const decodeAttribString = function* (str) {
+    const re = /\*([0-9a-z]+)|./gy;
+    let match;
+    while ((match = re.exec(str)) != null) {
+      const [m, n] = match;
+      if (n == null) throw new Error(`invalid character in attribute string: ${m}`);
+      yield Number.parseInt(n, 36);
+    }
+  };
+
+  const authors = new Set();
+  pool.eachAttrib((k, v) => {
+    if (k === 'author' && v) authors.add(v);
+  });
+  let atext = Changeset.makeAText('\n');
+  let r;
+  try {
+    for (r = 0; r <= head; ++r) {
+      const [changeset, author, timestamp] = await Promise.all([
+        this.getRevisionChangeset(r),
+        this.getRevisionAuthor(r),
+        this.getRevisionDate(r),
+      ]);
+      assert(author != null);
+      assert.equal(typeof author, 'string');
+      if (author) authors.add(author);
+      assert(timestamp != null);
+      assert.equal(typeof timestamp, 'number');
+      assert(timestamp > 0);
+      assert(changeset != null);
+      assert.equal(typeof changeset, 'string');
+      Changeset.checkRep(changeset);
+      const unpacked = Changeset.unpack(changeset);
+      let text = atext.text;
+      const iter = Changeset.opIterator(unpacked.ops);
+      while (iter.hasNext()) {
+        const op = iter.next();
+        if (['=', '-'].includes(op.opcode)) {
+          assert(text.length >= op.chars);
+          const consumed = text.slice(0, op.chars);
+          const nlines = (consumed.match(/\n/g) || []).length;
+          assert.equal(op.lines, nlines);
+          if (op.lines > 0) assert(consumed.endsWith('\n'));
+          text = text.slice(op.chars);
+        }
+        let prevK = null;
+        for (const n of decodeAttribString(op.attribs)) {
+          const attrib = pool.getAttrib(n);
+          assert(attrib != null);
+          const [k] = attrib;
+          assert(prevK == null || prevK < k);
+          prevK = k;
+        }
+      }
+      atext = Changeset.applyToAText(changeset, atext, pool);
+      assert.deepEqual(await this.getInternalRevisionAText(r), atext);
+    }
+  } catch (err) {
+    const pfx = `(pad ${this.id} revision ${r}) `;
+    if (err.stack) err.stack = pfx + err.stack;
+    err.message = pfx + err.message;
+    throw err;
+  }
+  assert.equal(this.text(), atext.text);
+  assert.deepEqual(this.atext, atext);
+  assert.deepEqual(this.getAllAuthors().sort(), [...authors].sort());
+
+  assert(Number.isInteger(this.chatHead));
+  assert(this.chatHead >= -1);
+  let c;
+  try {
+    for (c = 0; c <= this.chatHead; ++c) {
+      const msg = await this.getChatMessage(c);
+      assert(msg != null);
+      assert(msg instanceof ChatMessage);
+    }
+  } catch (err) {
+    const pfx = `(pad ${this.id} chat message ${c}) `;
+    if (err.stack) err.stack = pfx + err.stack;
+    err.message = pfx + err.message;
+    throw err;
+  }
+};

src/node/db/PadManager.js

@@ -20,8 +20,9 @@
  */
 
 const CustomError = require('../utils/customError');
-const Pad = require('../db/Pad').Pad;
+const Pad = require('../db/Pad');
 const db = require('./DB');
+const hooks = require('../../static/js/pluginfw/hooks');
 
 /**
  * A cache of all loaded Pads.
@@ -137,10 +138,11 @@ exports.getPad = async (id, text) => {
   }
 
   // try to load pad
-  pad = new Pad(id);
+  pad = new Pad.Pad(id);
 
   // initialize the pad
   await pad.init(text);
+  hooks.callAll('padLoad', {pad});
   globalPads.set(id, pad);
   padList.addPad(id);
 

src/node/utils/ImportEtherpad.js

@@ -1,5 +1,5 @@
-// 'use strict';
-// Uncommenting above breaks tests.
+'use strict';
+
 /**
  * 2014 John McLear (Etherpad Foundation / McLear Ltd)
  *
@@ -16,6 +16,10 @@
  * limitations under the License.
  */
 
+const AttributePool = require('../../static/js/AttributePool');
+const {Pad} = require('../db/Pad');
+const async = require('async');
+const authorManager = require('../db/AuthorManager');
 const db = require('../db/DB');
 const hooks = require('../../static/js/pluginfw/hooks');
 const log4js = require('log4js');
@@ -23,7 +27,7 @@ const supportedElems = require('../../static/js/contentcollector').supportedElem
 
 const logger = log4js.getLogger('ImportEtherpad');
 
-exports.setPadRaw = (padId, r) => {
+exports.setPadRaw = async (padId, r) => {
   const records = JSON.parse(r);
 
   // get supported block Elements from plugins, we will use this later.
@@ -31,72 +35,86 @@ exports.setPadRaw = (padId, r) => {
     supportedElems.add(element);
   });
 
-  const unsupportedElements = new Set();
+  // DB key prefixes for pad records. Each key is expected to have the form `${prefix}:${padId}` or
+  // `${prefix}:${padId}:${otherstuff}`.
+  const padKeyPrefixes = [
+    ...await hooks.aCallAll('exportEtherpadAdditionalContent'),
+    'pad',
+  ];
 
-  Object.keys(records).forEach(async (key) => {
-    let value = records[key];
+  let originalPadId = null;
+  const checkOriginalPadId = (padId) => {
+    if (originalPadId == null) originalPadId = padId;
+    if (originalPadId !== padId) throw new Error('unexpected pad ID in record');
+  };
 
+  // Limit the number of in-flight database queries so that the queries do not time out when
+  // importing really large files.
+  const q = async.queue(async (task) => await task(), 100);
+
+  // First validate and transform values. Do not commit any records to the database yet in case
+  // there is a problem with the data.
+
+  const dbRecords = new Map();
+  const existingAuthors = new Set();
+  await Promise.all(Object.entries(records).map(([key, value]) => q.pushAsync(async () => {
     if (!value) {
       return;
     }
-
-    let newKey;
-
-    if (value.padIDs) {
-      // Author data - rewrite author pad ids
-      value.padIDs[padId] = 1;
-      newKey = key;
-
-      // Does this author already exist?
-      const author = await db.get(key);
-
-      if (author) {
-        // Yes, add the padID to the author
-        if (Object.prototype.toString.call(author) === '[object Array]') {
-          author.padIDs.push(padId);
-        }
-
-        value = author;
-      } else {
-        // No, create a new array with the author info in
-        value.padIDs = [padId];
+    const keyParts = key.split(':');
+    const [prefix, id] = keyParts;
+    if (prefix === 'globalAuthor' && keyParts.length === 2) {
+      // In the database, the padIDs subkey is an object (which is used as a set) that records every
+      // pad the author has worked on. When exported, that object becomes a single string containing
+      // the exported pad's ID.
+      if (typeof value.padIDs !== 'string') {
+        throw new TypeError('globalAuthor padIDs subkey is not a string');
       }
+      checkOriginalPadId(value.padIDs);
+      if (await authorManager.doesAuthorExist(id)) {
+        existingAuthors.add(id);
+        return;
+      }
+      value.padIDs = {[padId]: 1};
+    } else if (padKeyPrefixes.includes(prefix)) {
+      checkOriginalPadId(id);
+      if (prefix === 'pad' && keyParts.length === 2) {
+        const pool = new AttributePool().fromJsonable(value.pool);
+        const unsupportedElements = new Set();
+        pool.eachAttrib((k, v) => {
+          if (!supportedElems.has(k)) unsupportedElements.add(k);
+        });
+        if (unsupportedElements.size) {
+          logger.warn(`(pad ${padId}) unsupported attributes (try installing a plugin): ` +
+                      `${[...unsupportedElements].join(', ')}`);
+        }
+      }
+      keyParts[1] = padId;
+      key = keyParts.join(':');
     } else {
-      // Not author data, probably pad data
-      // we can split it to look to see if it's pad data
-
-      // is this an attribute we support or not?  If not, tell the admin
-      if (value.pool) {
-        for (const attrib of Object.keys(value.pool.numToAttrib)) {
-          const attribName = value.pool.numToAttrib[attrib][0];
-          if (!supportedElems.has(attribName)) unsupportedElements.add(attribName);
-        }
-      }
-      const oldPadId = key.split(':');
-
-      // we know it's pad data
-      if (oldPadId[0] === 'pad') {
-        // so set the new pad id for the author
-        oldPadId[1] = padId;
-
-        // and create the value
-        newKey = oldPadId.join(':'); // create the new key
-      }
-
-      // is this a key that is supported through a plugin?
-      // get content that has a different prefix IE comments:padId:foo
-      // a plugin would return something likle ['comments', 'cakes']
-      for (const prefix of await hooks.aCallAll('exportEtherpadAdditionalContent')) {
-        if (prefix === oldPadId[0]) newKey = `${prefix}:${padId}`;
-      }
+      logger.warn(`(pad ${padId}) Ignoring record with unsupported key: ${key}`);
+      return;
     }
+    dbRecords.set(key, value);
+  })));
 
-    // Write the value to the server
-    await db.set(newKey, value);
+  const pad = new Pad(padId, {
+    // Only fetchers are needed to check the pad's integrity.
+    get: async (k) => dbRecords.get(k),
+    getSub: async (k, sub) => {
+      let v = dbRecords.get(k);
+      for (const sk of sub) {
+        if (v == null) return null;
+        v = v[sk];
+      }
+      return v;
+    },
   });
+  await pad.init();
+  await pad.check();
 
-  if (unsupportedElements.size) {
-    logger.warn('Ignoring unsupported elements (you might want to install a plugin): ' +
-                `${[...unsupportedElements].join(', ')}`);
-  }
+  await Promise.all([
+    ...[...dbRecords].map(([k, v]) => q.pushAsync(() => db.set(k, v))),
+    ...[...existingAuthors].map((a) => q.pushAsync(() => authorManager.addPad(a, padId))),
+  ]);
 };