mirrors/etherpad-lite
1
0
Fork 0
mirror of https://github.com/ether/etherpad-lite.git synced 2025-04-23 08:56:17 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

resolve conflict

Browse source
This commit is contained in:
John McLear 2021-01-30 08:11:10 +00:00
commit 96520a3f31
13 changed files with 458 additions and 394 deletions
Download patch file Download diff file Expand all files Collapse all files

142
bin/checkAllPads.js
View file

@ -7,89 +7,87 @@
// unhandled rejection into an uncaught exception, which does cause Node.js to exit. // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; }); process.on('unhandledRejection', (err) => { throw err; });
const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util');
if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js'); if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js');
// load and initialize NPM (async () => {
const npm = require('ep_etherpad-lite/node_modules/npm'); await util.promisify(npm.load)({});
npm.load({}, async () => {
try {
// initialize the database
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// load modules // initialize the database
const Changeset = require('ep_etherpad-lite/static/js/Changeset'); require('ep_etherpad-lite/node/utils/Settings');
const padManager = require('ep_etherpad-lite/node/db/PadManager'); const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
let revTestedCount = 0; // load modules
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const padManager = require('ep_etherpad-lite/node/db/PadManager');
// get all pads let revTestedCount = 0;
const res = await padManager.listAllPads();
for (const padId of res.padIDs) {
const pad = await padManager.getPad(padId);
// check if the pad has a pool // get all pads
if (pad.pool === undefined) { const res = await padManager.listAllPads();
console.error(`[${pad.id}] Missing attribute pool`); for (const padId of res.padIDs) {
const pad = await padManager.getPad(padId);
// check if the pad has a pool
if (pad.pool == null) {
console.error(`[${pad.id}] Missing attribute pool`);
continue;
}
// create an array with key kevisions
// key revisions always save the full pad atext
const head = pad.getHeadRevisionNumber();
const keyRevisions = [];
for (let rev = 0; rev < head; rev += 100) {
keyRevisions.push(rev);
}
// run through all key revisions
for (const keyRev of keyRevisions) {
// create an array of revisions we need till the next keyRevision or the End
const revisionsNeeded = [];
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
revisionsNeeded.push(rev);
}
// this array will hold all revision changesets
const revisions = [];
// run through all needed revisions and get them from the database
for (const revNum of revisionsNeeded) {
const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
revisions[revNum] = revision;
}
// check if the revision exists
if (revisions[keyRev] == null) {
console.error(`[${pad.id}] Missing revision ${keyRev}`);
continue; continue;
} }
// create an array with key kevisions
// key revisions always save the full pad atext // check if there is a atext in the keyRevisions
const head = pad.getHeadRevisionNumber(); let {meta: {atext} = {}} = revisions[keyRev];
const keyRevisions = []; if (atext == null) {
for (let rev = 0; rev < head; rev += 100) { console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
keyRevisions.push(rev); continue;
} }
// run through all key revisions const apool = pad.pool;
for (const keyRev of keyRevisions) { for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
// create an array of revisions we need till the next keyRevision or the End try {
const revisionsNeeded = []; const cs = revisions[rev].changeset;
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) { atext = Changeset.applyToAText(cs, atext, apool);
revisionsNeeded.push(rev); revTestedCount++;
} } catch (e) {
console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
// this array will hold all revision changesets
const revisions = [];
// run through all needed revisions and get them from the database
for (const revNum of revisionsNeeded) {
const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
revisions[revNum] = revision;
}
// check if the revision exists
if (revisions[keyRev] == null) {
console.error(`[${pad.id}] Missing revision ${keyRev}`);
continue;
}
// check if there is a atext in the keyRevisions
if (revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) {
console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
continue;
}
const apool = pad.pool;
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
try {
const cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
revTestedCount++;
} catch (e) {
console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
}
} }
} }
} }
if (revTestedCount === 0) {
throw new Error('No revisions tested');
}
console.log(`Finished: Tested ${revTestedCount} revisions`);
} catch (err) {
console.trace(err);
throw err;
} }
}); if (revTestedCount === 0) {
throw new Error('No revisions tested');
}
console.log(`Finished: Tested ${revTestedCount} revisions`);
})();

120
bin/checkPad.js
View file

@ -7,85 +7,81 @@
// unhandled rejection into an uncaught exception, which does cause Node.js to exit. // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; }); process.on('unhandledRejection', (err) => { throw err; });
const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util');
if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID'); if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID');
// get the padID // get the padID
const padId = process.argv[2]; const padId = process.argv[2];
let checkRevisionCount = 0; let checkRevisionCount = 0;
// load and initialize NPM; (async () => {
const npm = require('ep_etherpad-lite/node_modules/npm'); await util.promisify(npm.load)({});
npm.load({}, async () => {
try {
// initialize database
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// load modules // initialize database
const Changeset = require('ep_etherpad-lite/static/js/Changeset'); require('ep_etherpad-lite/node/utils/Settings');
const padManager = require('ep_etherpad-lite/node/db/PadManager'); const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
const exists = await padManager.doesPadExists(padId); // load modules
if (!exists) throw new Error('Pad does not exist'); const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const padManager = require('ep_etherpad-lite/node/db/PadManager');
// get the pad const exists = await padManager.doesPadExists(padId);
const pad = await padManager.getPad(padId); if (!exists) throw new Error('Pad does not exist');
// create an array with key revisions // get the pad
// key revisions always save the full pad atext const pad = await padManager.getPad(padId);
const head = pad.getHeadRevisionNumber();
const keyRevisions = []; // create an array with key revisions
for (let rev = 0; rev < head; rev += 100) { // key revisions always save the full pad atext
keyRevisions.push(rev); const head = pad.getHeadRevisionNumber();
const keyRevisions = [];
for (let rev = 0; rev < head; rev += 100) {
keyRevisions.push(rev);
}
// run through all key revisions
for (let keyRev of keyRevisions) {
keyRev = parseInt(keyRev);
// create an array of revisions we need till the next keyRevision or the End
const revisionsNeeded = [];
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
revisionsNeeded.push(rev);
} }
// run through all key revisions // this array will hold all revision changesets
for (let keyRev of keyRevisions) { const revisions = [];
keyRev = parseInt(keyRev);
// create an array of revisions we need till the next keyRevision or the End
const revisionsNeeded = [];
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
revisionsNeeded.push(rev);
}
// this array will hold all revision changesets // run through all needed revisions and get them from the database
const revisions = []; for (const revNum of revisionsNeeded) {
const revision = await db.get(`pad:${padId}:revs:${revNum}`);
revisions[revNum] = revision;
}
// run through all needed revisions and get them from the database // check if the pad has a pool
for (const revNum of revisionsNeeded) { if (pad.pool == null) throw new Error('Attribute pool is missing');
const revision = await db.get(`pad:${padId}:revs:${revNum}`);
revisions[revNum] = revision;
}
// check if the pad has a pool // check if there is an atext in the keyRevisions
if (pad.pool === undefined) throw new Error('Attribute pool is missing'); let {meta: {atext} = {}} = revisions[keyRev] || {};
if (atext == null) {
console.error(`No atext in key revision ${keyRev}`);
continue;
}
// check if there is an atext in the keyRevisions const apool = pad.pool;
if (revisions[keyRev] === undefined ||
revisions[keyRev].meta === undefined || for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
revisions[keyRev].meta.atext === undefined) { checkRevisionCount++;
console.error(`No atext in key revision ${keyRev}`); try {
const cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
} catch (e) {
console.error(`Bad changeset at revision ${rev} - ${e.message}`);
continue; continue;
} }
const apool = pad.pool;
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
checkRevisionCount++;
try {
const cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
} catch (e) {
console.error(`Bad changeset at revision ${rev} - ${e.message}`);
continue;
}
}
console.log(`Finished: Checked ${checkRevisionCount} revisions`);
} }
} catch (err) { console.log(`Finished: Checked ${checkRevisionCount} revisions`);
console.trace(err);
throw err;
} }
}); })();

14
bin/checkPadDeltas.js
View file

@ -12,12 +12,14 @@ if (process.argv.length !== 3) throw new Error('Use: node bin/checkPadDeltas.js
// get the padID // get the padID
const padId = process.argv[2]; const padId = process.argv[2];
// load and initialize NPM;
const expect = require('../tests/frontend/lib/expect'); const expect = require('../tests/frontend/lib/expect');
const diff = require('ep_etherpad-lite/node_modules/diff'); const diff = require('ep_etherpad-lite/node_modules/diff');
const npm = require('ep_etherpad-lite/node_modules/npm'); const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util');
(async () => {
await util.promisify(npm.load)({});
npm.load({}, async () => {
// initialize database // initialize database
require('ep_etherpad-lite/node/utils/Settings'); require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB'); const db = require('ep_etherpad-lite/node/db/DB');
@ -54,10 +56,8 @@ npm.load({}, async () => {
// console.log('Fetching', revNum) // console.log('Fetching', revNum)
const revision = await db.get(`pad:${padId}:revs:${revNum}`); const revision = await db.get(`pad:${padId}:revs:${revNum}`);
// check if there is a atext in the keyRevisions // check if there is a atext in the keyRevisions
if (~keyRevisions.indexOf(revNum) && const {meta: {atext: revAtext} = {}} = revision || {};
(revision === undefined || if (~keyRevisions.indexOf(revNum) && revAtext == null) {
revision.meta === undefined ||
revision.meta.atext === undefined)) {
console.error(`No atext in key revision ${revNum}`); console.error(`No atext in key revision ${revNum}`);
continue; continue;
} }
@ -104,4 +104,4 @@ npm.load({}, async () => {
} }
})); }));
} }
}); })();

87
bin/extractPadData.js
View file

@ -16,59 +16,54 @@ if (process.argv.length !== 3) throw new Error('Use: node extractPadData.js $PAD
const padId = process.argv[2]; const padId = process.argv[2];
const npm = require('ep_etherpad-lite/node_modules/npm'); const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util');
npm.load({}, async (err) => { (async () => {
if (err) throw err; await util.promisify(npm.load)({});
try { // initialize database
// initialize database require('ep_etherpad-lite/node/utils/Settings');
require('ep_etherpad-lite/node/utils/Settings'); const db = require('ep_etherpad-lite/node/db/DB');
const db = require('ep_etherpad-lite/node/db/DB'); await db.init();
await db.init();
// load extra modules // load extra modules
const dirtyDB = require('ep_etherpad-lite/node_modules/dirty'); const dirtyDB = require('ep_etherpad-lite/node_modules/dirty');
const padManager = require('ep_etherpad-lite/node/db/PadManager'); const padManager = require('ep_etherpad-lite/node/db/PadManager');
const util = require('util');
// initialize output database // initialize output database
const dirty = dirtyDB(`${padId}.db`); const dirty = dirtyDB(`${padId}.db`);
// Promise wrapped get and set function // Promise wrapped get and set function
const wrapped = db.db.db.wrappedDB; const wrapped = db.db.db.wrappedDB;
const get = util.promisify(wrapped.get.bind(wrapped)); const get = util.promisify(wrapped.get.bind(wrapped));
const set = util.promisify(dirty.set.bind(dirty)); const set = util.promisify(dirty.set.bind(dirty));
// array in which required key values will be accumulated // array in which required key values will be accumulated
const neededDBValues = [`pad:${padId}`]; const neededDBValues = [`pad:${padId}`];
// get the actual pad object // get the actual pad object
const pad = await padManager.getPad(padId); const pad = await padManager.getPad(padId);
// add all authors // add all authors
neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`)); neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`));
// add all revisions // add all revisions
for (let rev = 0; rev <= pad.head; ++rev) { for (let rev = 0; rev <= pad.head; ++rev) {
neededDBValues.push(`pad:${padId}:revs:${rev}`); neededDBValues.push(`pad:${padId}:revs:${rev}`);
}
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push(`pad:${padId}:chat:${chat}`);
}
for (const dbkey of neededDBValues) {
let dbvalue = await get(dbkey);
if (dbvalue && typeof dbvalue !== 'object') {
dbvalue = JSON.parse(dbvalue);
}
await set(dbkey, dbvalue);
}
console.log('finished');
} catch (err) {
console.error(err);
throw err;
} }
});
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push(`pad:${padId}:chat:${chat}`);
}
for (const dbkey of neededDBValues) {
let dbvalue = await get(dbkey);
if (dbvalue && typeof dbvalue !== 'object') {
dbvalue = JSON.parse(dbvalue);
}
await set(dbkey, dbvalue);
}
console.log('finished');
})();

68
bin/importSqlFile.js
View file

@ -4,6 +4,9 @@
// unhandled rejection into an uncaught exception, which does cause Node.js to exit. // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; }); process.on('unhandledRejection', (err) => { throw err; });
const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util');
const startTime = Date.now(); const startTime = Date.now();
const log = (str) => { const log = (str) => {
@ -43,10 +46,10 @@ const unescape = (val) => {
return val; return val;
}; };
(async () => {
await util.promisify(npm.load)({});
require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => {
const fs = require('fs'); const fs = require('fs');
const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2'); const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2');
const settings = require('ep_etherpad-lite/node/utils/Settings'); const settings = require('ep_etherpad-lite/node/utils/Settings');
const log4js = require('ep_etherpad-lite/node_modules/log4js'); const log4js = require('ep_etherpad-lite/node_modules/log4js');
@ -68,42 +71,35 @@ require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => {
if (!sqlFile) throw new Error('Use: node importSqlFile.js $SQLFILE'); if (!sqlFile) throw new Error('Use: node importSqlFile.js $SQLFILE');
log('initializing db'); log('initializing db');
db.init((err) => { await util.promisify(db.init.bind(db))();
// there was an error while initializing the database, output it and stop log('done');
if (err) {
throw err;
} else {
log('done');
log('open output file...'); log('open output file...');
const lines = fs.readFileSync(sqlFile, 'utf8').split('\n'); const lines = fs.readFileSync(sqlFile, 'utf8').split('\n');
const count = lines.length; const count = lines.length;
let keyNo = 0; let keyNo = 0;
process.stdout.write(`Start importing ${count} keys...\n`); process.stdout.write(`Start importing ${count} keys...\n`);
lines.forEach((l) => { lines.forEach((l) => {
if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') { if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') {
const pos = l.indexOf("', '"); const pos = l.indexOf("', '");
const key = l.substr(28, pos - 28); const key = l.substr(28, pos - 28);
let value = l.substr(pos + 3); let value = l.substr(pos + 3);
value = value.substr(0, value.length - 2); value = value.substr(0, value.length - 2);
console.log(`key: ${key} val: ${value}`); console.log(`key: ${key} val: ${value}`);
console.log(`unval: ${unescape(value)}`); console.log(`unval: ${unescape(value)}`);
db.set(key, unescape(value), null); db.set(key, unescape(value), null);
keyNo++; keyNo++;
if (keyNo % 1000 === 0) { if (keyNo % 1000 === 0) {
process.stdout.write(` ${keyNo}/${count}\n`); process.stdout.write(` ${keyNo}/${count}\n`);
} }
}
});
process.stdout.write('\n');
process.stdout.write('done. waiting for db to finish transaction. ' +
'depended on dbms this may take some time..\n');
db.close(() => {
log(`finished, imported ${keyNo} keys.`);
});
} }
}); });
}); process.stdout.write('\n');
process.stdout.write('done. waiting for db to finish transaction. ' +
'depended on dbms this may take some time..\n');
await util.promisify(db.close.bind(db))();
log(`finished, imported ${keyNo} keys.`);
})();

7
bin/migrateDirtyDBtoRealDB.js
View file

@ -4,9 +4,12 @@
// unhandled rejection into an uncaught exception, which does cause Node.js to exit. // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; }); process.on('unhandledRejection', (err) => { throw err; });
const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util'); const util = require('util');
require('ep_etherpad-lite/node_modules/npm').load({}, async (er, npm) => { (async () => {
await util.promisify(npm.load)({});
process.chdir(`${npm.root}/..`); process.chdir(`${npm.root}/..`);
// This script requires that you have modified your settings.json file // This script requires that you have modified your settings.json file
@ -56,4 +59,4 @@ require('ep_etherpad-lite/node_modules/npm').load({}, async (er, npm) => {
await util.promisify(db.close.bind(db))(); await util.promisify(db.close.bind(db))();
console.log('Finished.'); console.log('Finished.');
}); })();

154
bin/rebuildPad.js
View file

@ -13,7 +13,6 @@ if (process.argv.length !== 4 && process.argv.length !== 5) {
throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]'); throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]');
} }
const async = require('ep_etherpad-lite/node_modules/async');
const npm = require('ep_etherpad-lite/node_modules/npm'); const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util'); const util = require('util');
@ -21,95 +20,70 @@ const padId = process.argv[2];
const newRevHead = process.argv[3]; const newRevHead = process.argv[3];
const newPadId = process.argv[4] || `${padId}-rebuilt`; const newPadId = process.argv[4] || `${padId}-rebuilt`;
let db, oldPad, newPad; (async () => {
let Pad, PadManager; await util.promisify(npm.load)({});
async.series([ const db = require('ep_etherpad-lite/node/db/DB');
(callback) => npm.load({}, callback), await db.init();
(callback) => {
// Get a handle into the database const PadManager = require('ep_etherpad-lite/node/db/PadManager');
db = require('ep_etherpad-lite/node/db/DB'); const Pad = require('ep_etherpad-lite/node/db/Pad').Pad;
db.init(callback); // Validate the newPadId if specified and that a pad with that ID does
}, // not already exist to avoid overwriting it.
(callback) => { if (!PadManager.isValidPadId(newPadId)) {
Pad = require('ep_etherpad-lite/node/db/Pad').Pad; throw new Error('Cannot create a pad with that id as it is invalid');
PadManager = require('ep_etherpad-lite/node/db/PadManager'); }
// Get references to the original pad and to a newly created pad const exists = await PadManager.doesPadExist(newPadId);
// HACK: This is a standalone script, so we want to write everything if (exists) throw new Error('Cannot create a pad with that id as it already exists');
// out to the database immediately. The only problem with this is
// that a driver (like the mysql driver) can hardcode these values. const oldPad = await PadManager.getPad(padId);
db.db.db.settings = {cache: 0, writeInterval: 0, json: true}; const newPad = new Pad(newPadId);
// Validate the newPadId if specified and that a pad with that ID does
// not already exist to avoid overwriting it. // Clone all Chat revisions
if (!PadManager.isValidPadId(newPadId)) { const chatHead = oldPad.chatHead;
throw new Error('Cannot create a pad with that id as it is invalid'); await Promise.all([...Array(chatHead + 1).keys()].map(async (i) => {
const chat = await db.get(`pad:${padId}:chat:${i}`);
await db.set(`pad:${newPadId}:chat:${i}`, chat);
console.log(`Created: Chat Revision: pad:${newPadId}:chat:${i}`);
}));
// Rebuild Pad from revisions up to and including the new revision head
const AuthorManager = require('ep_etherpad-lite/node/db/AuthorManager');
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
// Author attributes are derived from changesets, but there can also be
// non-author attributes with specific mappings that changesets depend on
// and, AFAICT, cannot be recreated any other way
newPad.pool.numToAttrib = oldPad.pool.numToAttrib;
for (let curRevNum = 0; curRevNum <= newRevHead; curRevNum++) {
const rev = await db.get(`pad:${padId}:revs:${curRevNum}`);
if (!rev || !rev.meta) throw new Error('The specified revision number could not be found.');
const newRevNum = ++newPad.head;
const newRevId = `pad:${newPad.id}:revs:${newRevNum}`;
await Promise.all([
db.set(newRevId, rev),
AuthorManager.addPad(rev.meta.author, newPad.id),
]);
newPad.atext = Changeset.applyToAText(rev.changeset, newPad.atext, newPad.pool);
console.log(`Created: Revision: pad:${newPad.id}:revs:${newRevNum}`);
}
// Add saved revisions up to the new revision head
console.log(newPad.head);
const newSavedRevisions = [];
for (const savedRev of oldPad.savedRevisions) {
if (savedRev.revNum <= newRevHead) {
newSavedRevisions.push(savedRev);
console.log(`Added: Saved Revision: ${savedRev.revNum}`);
} }
PadManager.doesPadExists(newPadId, (err, exists) => { }
if (exists) throw new Error('Cannot create a pad with that id as it already exists'); newPad.savedRevisions = newSavedRevisions;
});
PadManager.getPad(padId, (err, pad) => { // Save the source pad
oldPad = pad; await db.set(`pad:${newPadId}`, newPad);
newPad = new Pad(newPadId);
callback(); console.log(`Created: Source Pad: pad:${newPadId}`);
}); await newPad.saveToDatabase();
},
(callback) => { await db.shutdown();
// Clone all Chat revisions
const chatHead = oldPad.chatHead;
for (let i = 0, curHeadNum = 0; i <= chatHead; i++) {
db.db.get(`pad:${padId}:chat:${i}`, (err, chat) => {
db.db.set(`pad:${newPadId}:chat:${curHeadNum++}`, chat);
console.log(`Created: Chat Revision: pad:${newPadId}:chat:${curHeadNum}`);
});
}
callback();
},
(callback) => {
// Rebuild Pad from revisions up to and including the new revision head
const AuthorManager = require('ep_etherpad-lite/node/db/AuthorManager');
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
// Author attributes are derived from changesets, but there can also be
// non-author attributes with specific mappings that changesets depend on
// and, AFAICT, cannot be recreated any other way
newPad.pool.numToAttrib = oldPad.pool.numToAttrib;
for (let curRevNum = 0; curRevNum <= newRevHead; curRevNum++) {
db.db.get(`pad:${padId}:revs:${curRevNum}`, (err, rev) => {
if (rev.meta) {
throw new Error('The specified revision number could not be found.');
}
const newRevNum = ++newPad.head;
const newRevId = `pad:${newPad.id}:revs:${newRevNum}`;
db.db.set(newRevId, rev);
AuthorManager.addPad(rev.meta.author, newPad.id);
newPad.atext = Changeset.applyToAText(rev.changeset, newPad.atext, newPad.pool);
console.log(`Created: Revision: pad:${newPad.id}:revs:${newRevNum}`);
if (newRevNum === newRevHead) {
callback();
}
});
}
},
(callback) => {
// Add saved revisions up to the new revision head
console.log(newPad.head);
const newSavedRevisions = [];
for (const savedRev of oldPad.savedRevisions) {
if (savedRev.revNum <= newRevHead) {
newSavedRevisions.push(savedRev);
console.log(`Added: Saved Revision: ${savedRev.revNum}`);
}
}
newPad.savedRevisions = newSavedRevisions;
callback();
},
(callback) => {
// Save the source pad
db.db.set(`pad:${newPadId}`, newPad, (err) => {
console.log(`Created: Source Pad: pad:${newPadId}`);
util.callbackify(newPad.saveToDatabase.bind(newPad))(callback);
});
},
], (err) => {
if (err) throw err;
console.info('finished'); console.info('finished');
}); })();

8
bin/repairPad.js
View file

@ -18,8 +18,10 @@ const padId = process.argv[2];
let valueCount = 0; let valueCount = 0;
const npm = require('ep_etherpad-lite/node_modules/npm'); const npm = require('ep_etherpad-lite/node_modules/npm');
npm.load({}, async (err) => { const util = require('util');
if (err) throw err;
(async () => {
await util.promisify(npm.load)({});
// intialize database // intialize database
require('ep_etherpad-lite/node/utils/Settings'); require('ep_etherpad-lite/node/utils/Settings');
@ -56,4 +58,4 @@ npm.load({}, async (err) => {
} }
console.info(`Finished: Replaced ${valueCount} values in the database`); console.info(`Finished: Replaced ${valueCount} values in the database`);
}); })();

8
package-lock.json generated
View file

@ -870,7 +870,8 @@
"tinycon": "0.0.1", "tinycon": "0.0.1",
"ueberdb2": "^1.2.5", "ueberdb2": "^1.2.5",
"underscore": "1.8.3", "underscore": "1.8.3",
"unorm": "1.4.1" "unorm": "1.4.1",
"wtfnode": "^0.8.4"
}, },
"dependencies": { "dependencies": {
"@apidevtools/json-schema-ref-parser": { "@apidevtools/json-schema-ref-parser": {
@ -10577,6 +10578,11 @@
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
}, },
"wtfnode": {
"version": "0.8.4",
"resolved": "https://registry.npmjs.org/wtfnode/-/wtfnode-0.8.4.tgz",
"integrity": "sha512-64GEKtMt/MUBuAm+8kHqP74ojjafzu00aT0JKsmkIwYmjRQ/odO0yhbzKLm+Z9v1gMla+8dwITRKzTAlHsB+Og=="
},
"yallist": { "yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

234
src/node/server.js
View file

@ -27,6 +27,10 @@
const log4js = require('log4js'); const log4js = require('log4js');
log4js.replaceConsole(); log4js.replaceConsole();
// wtfnode should be loaded after log4js.replaceConsole() so that it uses log4js for logging, and it
// should be above everything else so that it can hook in before resources are used.
const wtfnode = require('wtfnode');
/* /*
* early check for version compatibility before calling * early check for version compatibility before calling
* any modules that require newer versions of NodeJS * any modules that require newer versions of NodeJS
@ -44,111 +48,195 @@ const plugins = require('../static/js/pluginfw/plugins');
const settings = require('./utils/Settings'); const settings = require('./utils/Settings');
const util = require('util'); const util = require('util');
let started = false; const State = {
let stopped = false; INITIAL: 1,
STARTING: 2,
RUNNING: 3,
STOPPING: 4,
STOPPED: 5,
EXITING: 6,
WAITING_FOR_EXIT: 7,
};
let state = State.INITIAL;
const removeSignalListener = (signal, listener) => {
console.debug(`Removing ${signal} listener because it might interfere with shutdown tasks. ` +
`Function code:\n${listener.toString()}\n` +
`Current stack:\n${(new Error()).stack.split('\n').slice(1).join('\n')}`);
process.off(signal, listener);
};
const runningCallbacks = [];
exports.start = async () => { exports.start = async () => {
if (started) return express.server; switch (state) {
started = true; case State.INITIAL:
if (stopped) throw new Error('restart not supported'); break;
case State.STARTING:
await new Promise((resolve) => runningCallbacks.push(resolve));
// fall through
case State.RUNNING:
return express.server;
case State.STOPPING:
case State.STOPPED:
case State.EXITING:
case State.WAITING_FOR_EXIT:
throw new Error('restart not supported');
default:
throw new Error(`unknown State: ${state.toString()}`);
}
console.log('Starting Etherpad...');
state = State.STARTING;
// Check if Etherpad version is up-to-date // Check if Etherpad version is up-to-date
UpdateCheck.check(); UpdateCheck.check();
// start up stats counting system // start up stats counting system
const stats = require('./stats'); const stats = require('./stats');
let startDurations = {}; const startDurations = {};
stats.gauge('startDurations', () => startDurations); stats.gauge('startDurations', () => startDurations);
stats.gauge('memoryUsage', () => process.memoryUsage().rss); stats.gauge('memoryUsage', () => process.memoryUsage().rss);
stats.gauge('memoryUsageHeap', () => process.memoryUsage().heapUsed); stats.gauge('memoryUsageHeap', () => process.memoryUsage().heapUsed);
// Performance stats gauges process.on('uncaughtException', exports.exit);
// We use gauges because a reload might replace the value // As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
for (const signal of ['SIGINT', 'SIGTERM']) {
// Forcibly remove other signal listeners to prevent them from terminating node before we are
// done cleaning up. See https://github.com/andywer/threads.js/pull/329 for an example of a
// problematic listener. This means that exports.exit is solely responsible for performing all
// necessary cleanup tasks.
for (const listener of process.listeners(signal)) {
removeSignalListener(signal, listener);
}
process.on(signal, exports.exit);
// Prevent signal listeners from being added in the future.
process.on('newListener', (event, listener) => {
if (event !== signal) return;
removeSignalListener(signal, listener);
});
}
const preNpmLoad = Date.now(); const preNpmLoad = Date.now();
await util.promisify(npm.load)(); await util.promisify(npm.load)();
startDurations.npmLoad = Date.now() - preNpmLoad; startDurations.npmLoad = Date.now() - preNpmLoad;
try { const preDbInit = Date.now();
const preDbInit = Date.now(); await db.init();
await db.init(); startDurations.dbInit = Date.now() - preDbInit;
startDurations.dbInit = Date.now() - preDbInit;
const prePluginsUpdate = Date.now(); const prePluginsUpdate = Date.now();
await plugins.update(); await plugins.update();
startDurations.loadPlugins = Date.now() - prePluginsUpdate; startDurations.loadPlugins = Date.now() - prePluginsUpdate;
console.info(`Installed plugins: ${plugins.formatPluginsWithVersion()}`); console.info(`Installed plugins: ${plugins.formatPluginsWithVersion()}`);
console.debug(`Installed parts:\n${plugins.formatParts()}`); console.debug(`Installed parts:\n${plugins.formatParts()}`);
console.debug(`Installed hooks:\n${plugins.formatHooks()}`); console.debug(`Installed hooks:\n${plugins.formatHooks()}`);
const preLoadSettings = Date.now(); const preLoadSettings = Date.now();
await hooks.aCallAll('loadSettings', {settings}); await hooks.aCallAll('loadSettings', {settings});
startDurations.loadSettings = Date.now() - preLoadSettings; startDurations.loadSettings = Date.now() - preLoadSettings;
const preCreateServer = Date.now(); await hooks.aCallAll('createServer');
await hooks.aCallAll('createServer');
startDurations.createSettings = Date.now() - preCreateServer;
} catch (e) {
console.error(`exception thrown: ${e.message}`);
if (e.stack) console.log(e.stack);
process.exit(1);
}
process.on('uncaughtException', exports.exit); console.log('Etherpad is running');
state = State.RUNNING;
/* while (runningCallbacks.length > 0) setImmediate(runningCallbacks.pop());
* Connect graceful shutdown with sigint and uncaught exception
*
* Until Etherpad 1.7.5, process.on('SIGTERM') and process.on('SIGINT') were
* not hooked up under Windows, because old nodejs versions did not support
* them.
*
* According to nodejs 6.x documentation, it is now safe to do so. This
* allows to gracefully close the DB connection when hitting CTRL+C under
* Windows, for example.
*
* Source: https://nodejs.org/docs/latest-v6.x/api/process.html#process_signal_events
*
* - SIGTERM is not supported on Windows, it can be listened on.
* - SIGINT from the terminal is supported on all platforms, and can usually
* be generated with <Ctrl>+C (though this may be configurable). It is not
* generated when terminal raw mode is enabled.
*/
process.on('SIGINT', exports.exit);
// When running as PID1 (e.g. in docker container) allow graceful shutdown on SIGTERM c.f. #3265.
// Pass undefined to exports.exit because this is not an abnormal termination.
process.on('SIGTERM', () => exports.exit());
// Return the HTTP server to make it easier to write tests. // Return the HTTP server to make it easier to write tests.
return express.server; return express.server;
}; };
const stoppedCallbacks = [];
exports.stop = async () => { exports.stop = async () => {
if (stopped) return; switch (state) {
stopped = true; case State.STARTING:
await exports.start();
// Don't fall through to State.RUNNING in case another caller is also waiting for startup.
return await exports.stop();
case State.RUNNING:
break;
case State.STOPPING:
await new Promise((resolve) => stoppedCallbacks.push(resolve));
// fall through
case State.INITIAL:
case State.STOPPED:
case State.EXITING:
case State.WAITING_FOR_EXIT:
return;
default:
throw new Error(`unknown State: ${state.toString()}`);
}
console.log('Stopping Etherpad...'); console.log('Stopping Etherpad...');
await new Promise(async (resolve, reject) => { state = State.STOPPING;
const id = setTimeout(() => reject(new Error('Timed out waiting for shutdown tasks')), 3000); let timeout = null;
await hooks.aCallAll('shutdown'); await Promise.race([
clearTimeout(id); hooks.aCallAll('shutdown'),
resolve(); new Promise((resolve, reject) => {
}); timeout = setTimeout(() => reject(new Error('Timed out waiting for shutdown tasks')), 3000);
}),
]);
clearTimeout(timeout);
console.log('Etherpad stopped');
state = State.STOPPED;
while (stoppedCallbacks.length > 0) setImmediate(stoppedCallbacks.pop());
}; };
exports.exit = async (err) => { const exitCallbacks = [];
let exitCode = 0; let exitCalled = false;
if (err) { exports.exit = async (err = null) => {
exitCode = 1; /* eslint-disable no-process-exit */
console.error(err.stack ? err.stack : err); if (err === 'SIGTERM') {
// Termination from SIGTERM is not treated as an abnormal termination.
console.log('Received SIGTERM signal');
err = null;
} else if (err != null) {
console.error(err.stack || err.toString());
process.exitCode = 1;
if (exitCalled) {
console.error('Error occurred while waiting to exit. Forcing an immediate unclean exit...');
process.exit(1);
}
} }
try { exitCalled = true;
await exports.stop(); switch (state) {
} catch (err) { case State.STARTING:
exitCode = 1; case State.RUNNING:
console.error(err.stack ? err.stack : err); case State.STOPPING:
await exports.stop();
// Don't fall through to State.STOPPED in case another caller is also waiting for stop().
// Don't pass err to exports.exit() because this err has already been processed. (If err is
// passed again to exit() then exit() will think that a second error occurred while exiting.)
return await exports.exit();
case State.INITIAL:
case State.STOPPED:
break;
case State.EXITING:
await new Promise((resolve) => exitCallbacks.push(resolve));
// fall through
case State.WAITING_FOR_EXIT:
return;
default:
throw new Error(`unknown State: ${state.toString()}`);
} }
process.exit(exitCode); console.log('Exiting...');
state = State.EXITING;
while (exitCallbacks.length > 0) setImmediate(exitCallbacks.pop());
// Node.js should exit on its own without further action. Add a timeout to force Node.js to exit
// just in case something failed to get cleaned up during the shutdown hook. unref() is called on
// the timeout so that the timeout itself does not prevent Node.js from exiting.
setTimeout(() => {
console.error('Something that should have been cleaned up during the shutdown hook (such as ' +
'a timer, worker thread, or open connection) is preventing Node.js from exiting');
wtfnode.dump();
console.error('Forcing an unclean exit...');
process.exit(1);
}, 5000).unref();
console.log('Waiting for Node.js to exit...');
state = State.WAITING_FOR_EXIT;
/* eslint-enable no-process-exit */
}; };
if (require.main === module) exports.start(); if (require.main === module) exports.start();

5
src/package-lock.json generated
View file

@ -8697,6 +8697,11 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.2.tgz", "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.2.tgz",
"integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==" "integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA=="
}, },
"wtfnode": {
"version": "0.8.4",
"resolved": "https://registry.npmjs.org/wtfnode/-/wtfnode-0.8.4.tgz",
"integrity": "sha512-64GEKtMt/MUBuAm+8kHqP74ojjafzu00aT0JKsmkIwYmjRQ/odO0yhbzKLm+Z9v1gMla+8dwITRKzTAlHsB+Og=="
},
"xml2js": { "xml2js": {
"version": "0.4.23", "version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",

3
src/package.json
View file

@ -72,7 +72,8 @@
"tinycon": "0.0.1", "tinycon": "0.0.1",
"ueberdb2": "^1.2.5", "ueberdb2": "^1.2.5",
"underscore": "1.8.3", "underscore": "1.8.3",
"unorm": "1.4.1" "unorm": "1.4.1",
"wtfnode": "^0.8.4"
}, },
"bin": { "bin": {
"etherpad-lite": "node/server.js" "etherpad-lite": "node/server.js"

2
tests/backend/common.js
View file

@ -50,10 +50,10 @@ exports.init = async function () {
after(async function () { after(async function () {
webaccess.authnFailureDelayMs = backups.authnFailureDelayMs; webaccess.authnFailureDelayMs = backups.authnFailureDelayMs;
await server.stop();
// Note: This does not unset settings that were added. // Note: This does not unset settings that were added.
Object.assign(settings, backups.settings); Object.assign(settings, backups.settings);
log4js.setGlobalLogLevel(logLevel); log4js.setGlobalLogLevel(logLevel);
await server.exit();
}); });
return exports.agent; return exports.agent;