From 9497ee734f415daa266e015ea4b5deb45033aa38 Mon Sep 17 00:00:00 2001 From: muxator Date: Fri, 8 Feb 2019 23:20:57 +0100 Subject: [PATCH 01/58] prepare to async: trivial reformatting This change is only cosmetic. Its aim is do make it easier to understand the async changes that are going to be merged later on. It was extracted from the original work from Ray Bellis. To verify that nothing has changed, you can run the following command on each file touched by this commit: npm install uglify-es diff --unified <(uglify-js --beautify bracketize ) <(uglify-js --beautify bracketize ) This is a complete script that does the same automatically (works from a mercurial clone): ```bash #!/usr/bin/env bash set -eu REVISION= PARENT_REV=$(hg identify --rev "${REVISION}" --template '{p1rev}') FILE_LIST=$(hg status --no-status --change ${REVISION}) UGLIFYJS="node_modules/uglify-es/bin/uglifyjs" for FILE_NAME in ${FILE_LIST[@]}; do echo "Checking ${FILE_NAME}" diff --unified \ <("${UGLIFYJS}" --beautify bracketize <(hg cat --rev "${PARENT_REV}" "${FILE_NAME}")) \ <("${UGLIFYJS}" --beautify bracketize <(hg cat --rev "${REVISION}" "${FILE_NAME}")) done ``` --- bin/checkAllPads.js | 207 +++-- bin/checkPad.js | 130 ++- bin/deletePad.js | 50 +- bin/extractPadData.js | 118 ++- src/node/db/API.js | 762 +++++++--------- src/node/db/AuthorManager.js | 163 ++-- src/node/db/DB.js | 27 +- src/node/db/GroupManager.js | 293 +++---- src/node/db/Pad.js | 463 +++++----- src/node/db/PadManager.js | 177 ++-- src/node/db/ReadOnlyManager.js | 49 +- src/node/db/SecurityManager.js | 350 ++++---- src/node/db/SessionManager.js | 248 +++--- src/node/db/SessionStore.js | 42 +- src/node/handler/APIHandler.js | 96 +- src/node/handler/ExportHandler.js | 117 ++- src/node/handler/ImportHandler.js | 152 ++-- src/node/handler/PadMessageHandler.js | 1010 +++++++++++----------- src/node/handler/SocketIORouter.js | 104 ++- src/node/hooks/express/adminplugins.js | 103 ++- src/node/hooks/express/errorhandling.js | 15 +- src/node/hooks/express/importexport.js | 14 +- src/node/hooks/express/padreadonly.js | 39 +- src/node/hooks/express/padurlsanitize.js | 19 +- src/node/hooks/express/tests.js | 51 +- src/node/padaccess.js | 10 +- src/node/server.js | 24 +- src/node/utils/ExportTxt.js | 165 ++-- src/node/utils/ImportEtherpad.js | 43 +- src/node/utils/ImportHtml.js | 26 +- src/node/utils/padDiff.js | 498 ++++++----- src/static/js/pluginfw/installer.js | 77 +- src/static/js/pluginfw/plugins.js | 7 +- 33 files changed, 2706 insertions(+), 2943 deletions(-) diff --git a/bin/checkAllPads.js b/bin/checkAllPads.js index a94c38d23..c4467fa79 100644 --- a/bin/checkAllPads.js +++ b/bin/checkAllPads.js @@ -1,144 +1,133 @@ /* - This is a debug tool. It checks all revisions for data corruption -*/ + * This is a debug tool. It checks all revisions for data corruption + */ -if(process.argv.length != 2) -{ +if (process.argv.length != 2) { console.error("Use: node bin/checkAllPads.js"); process.exit(1); } -//initialize the variables +// initialize the variables var db, settings, padManager; -var npm = require("../src/node_modules/npm"); -var async = require("../src/node_modules/async"); +var npm = require('../src/node_modules/npm'); +var async = require('../src/node_modules/async'); -var Changeset = require("../src/static/js/Changeset"); +var Changeset = require('../src/static/js/Changeset'); async.series([ - //load npm + // load npm function(callback) { npm.load({}, callback); }, - //load modules + + // load modules function(callback) { settings = require('../src/node/utils/Settings'); db = require('../src/node/db/DB'); - //initialize the database + // initialize the database db.init(callback); }, - //load pads - function (callback) - { + + // load pads + function (callback) { padManager = require('../src/node/db/PadManager'); - - padManager.listAllPads(function(err, res) - { + + padManager.listAllPads(function(err, res) { padIds = res.padIDs; callback(err); }); }, - function (callback) - { - async.forEach(padIds, function(padId, callback) - { - padManager.getPad(padId, function(err, pad) { + + function (callback) { + async.forEach(padIds, function(padId, callback) { + padManager.getPad(padId, function(err, pad) { + if (err) { + callback(err); + } + + // check if the pad has a pool + if (pad.pool === undefined ) { + console.error("[" + pad.id + "] Missing attribute pool"); + callback(); + + return; + } + + // create an array with key kevisions + // key revisions always save the full pad atext + var head = pad.getHeadRevisionNumber(); + var keyRevisions = []; + for (var i = 0; i < head; i += 100) { + keyRevisions.push(i); + } + + // run through all key revisions + async.forEachSeries(keyRevisions, function(keyRev, callback) { + // create an array of revisions we need till the next keyRevision or the End + var revisionsNeeded = []; + + for(var i = keyRev; i <= keyRev + 100 && i <= head; i++) { + revisionsNeeded.push(i); + } + + // this array will hold all revision changesets + var revisions = []; + + // run through all needed revisions and get them from the database + async.forEach(revisionsNeeded, function(revNum, callback) { + db.db.get("pad:" + pad.id + ":revs:" + revNum, function(err, revision) { + revisions[revNum] = revision; + callback(err); + }); + }, + + function(err) { if (err) { - callback(err); + callback(err); + return; } - - //check if the pad has a pool - if(pad.pool === undefined ) - { - console.error("[" + pad.id + "] Missing attribute pool"); + + // check if the revision exists + if (revisions[keyRev] == null) { + console.error("[" + pad.id + "] Missing revision " + keyRev); + callback(); + return; + } + + // check if there is a atext in the keyRevisions + if (revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) { + console.error("[" + pad.id + "] Missing atext in revision " + keyRev); + callback(); + return; + } + + var apool = pad.pool; + var atext = revisions[keyRev].meta.atext; + + for(var i = keyRev + 1; i <= keyRev + 100 && i <= head; i++) { + try { + // console.log("[" + pad.id + "] check revision " + i); + var cs = revisions[i].changeset; + atext = Changeset.applyToAText(cs, atext, apool); + } catch(e) { + console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message); callback(); return; + } } - //create an array with key kevisions - //key revisions always save the full pad atext - var head = pad.getHeadRevisionNumber(); - var keyRevisions = []; - for(var i=0;i pad.getHeadRevisionNumber()) - { - callback(new customError("rev is higher than the head revision of the pad","apierror")); + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + + // the client asked for a special revision + if (rev !== undefined) { + // check if this is a valid revision + if (rev > pad.getHeadRevisionNumber()) { + callback(new customError("rev is higher than the head revision of the pad", "apierror")); return; } - - //get the changeset for this revision - pad.getRevisionChangeset(rev, function(err, changeset) - { - if(ERR(err, callback)) return; - + + // get the changeset for this revision + pad.getRevisionChangeset(rev, function(err, changeset) { + if (ERR(err, callback)) return; + callback(null, changeset); }) return; } - //the client wants the latest changeset, lets return it to him - pad.getRevisionChangeset(pad.getHeadRevisionNumber(), function(err, changeset) - { - if(ERR(err, callback)) return; + // the client wants the latest changeset, lets return it to him + pad.getRevisionChangeset(pad.getHeadRevisionNumber(), function(err, changeset) { + if (ERR(err, callback)) return; callback(null, changeset); }) @@ -191,7 +182,7 @@ exports.getRevisionChangeset = function(padID, rev, callback) } /** -getText(padID, [rev]) returns the text of a pad +getText(padID, [rev]) returns the text of a pad Example returns: @@ -200,69 +191,61 @@ Example returns: */ exports.getText = function(padID, rev, callback) { - //check if rev is a number - if(rev !== undefined && typeof rev != "number") - { - //try to parse the number - if(isNaN(parseInt(rev))) - { + // check if rev is a number + if (rev !== undefined && typeof rev != "number") { + // try to parse the number + if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); return; } rev = parseInt(rev); } - - //ensure this is not a negativ number - if(rev !== undefined && rev < 0) - { - callback(new customError("rev is a negativ number","apierror")); + + // ensure this is not a negative number + if (rev !== undefined && rev < 0) { + callback(new customError("rev is a negativ number", "apierror")); return; } - - //ensure this is not a float value - if(rev !== undefined && !is_int(rev)) - { - callback(new customError("rev is a float value","apierror")); + + // ensure this is not a float value + if (rev !== undefined && !is_int(rev)) { + callback(new customError("rev is a float value", "apierror")); return; } - - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - - //the client asked for a special revision - if(rev !== undefined) - { - //check if this is a valid revision - if(rev > pad.getHeadRevisionNumber()) - { - callback(new customError("rev is higher than the head revision of the pad","apierror")); + + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + + // the client asked for a special revision + if (rev !== undefined) { + // check if this is a valid revision + if (rev > pad.getHeadRevisionNumber()) { + callback(new customError("rev is higher than the head revision of the pad", "apierror")); return; } - - //get the text of this revision - pad.getInternalRevisionAText(rev, function(err, atext) - { - if(ERR(err, callback)) return; - + + // get the text of this revision + pad.getInternalRevisionAText(rev, function(err, atext) { + if (ERR(err, callback)) return; + var data = {text: atext.text}; - + callback(null, data); }) return; } - //the client wants the latest text, lets return it to him + // the client wants the latest text, lets return it to him var padText = exportTxt.getTXTFromAtext(pad, pad.atext); callback(null, {"text": padText}); }); } /** -setText(padID, text) sets the text of a pad +setText(padID, text) sets the text of a pad Example returns: @@ -271,23 +254,21 @@ Example returns: {code: 1, message:"text too long", data: null} */ exports.setText = function(padID, text, callback) -{ - //text is required - if(typeof text != "string") - { - callback(new customError("text is no string","apierror")); +{ + // text is required + if (typeof text != "string") { + callback(new customError("text is no string", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - - //set the text + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + + // set the text pad.setText(text); - - //update the clients on the pad + + // update the clients on the pad padMessageHandler.updatePadClients(pad, callback); }); } @@ -303,29 +284,25 @@ Example returns: */ exports.appendText = function(padID, text, callback) { - //text is required - if(typeof text != "string") - { - callback(new customError("text is no string","apierror")); + // text is required + if (typeof text != "string") { + callback(new customError("text is no string", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; pad.appendText(text); - //update the clients on the pad + // update the clients on the pad padMessageHandler.updatePadClients(pad, callback); }); }; - - /** -getHTML(padID, [rev]) returns the html of a pad +getHTML(padID, [rev]) returns the html of a pad Example returns: @@ -334,47 +311,39 @@ Example returns: */ exports.getHTML = function(padID, rev, callback) { - if (rev !== undefined && typeof rev != "number") - { - if (isNaN(parseInt(rev))) - { - callback(new customError("rev is not a number","apierror")); + if (rev !== undefined && typeof rev != "number") { + if (isNaN(parseInt(rev))) { + callback(new customError("rev is not a number", "apierror")); return; } rev = parseInt(rev); } - if(rev !== undefined && rev < 0) - { - callback(new customError("rev is a negative number","apierror")); + if (rev !== undefined && rev < 0) { + callback(new customError("rev is a negative number", "apierror")); return; } - if(rev !== undefined && !is_int(rev)) - { - callback(new customError("rev is a float value","apierror")); + if (rev !== undefined && !is_int(rev)) { + callback(new customError("rev is a float value", "apierror")); return; } - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - - //the client asked for a special revision - if(rev !== undefined) - { - //check if this is a valid revision - if(rev > pad.getHeadRevisionNumber()) - { - callback(new customError("rev is higher than the head revision of the pad","apierror")); + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + + // the client asked for a special revision + if (rev !== undefined) { + // check if this is a valid revision + if (rev > pad.getHeadRevisionNumber()) { + callback(new customError("rev is higher than the head revision of the pad", "apierror")); return; } - - //get the html of this revision - exportHtml.getPadHTML(pad, rev, function(err, html) - { - if(ERR(err, callback)) return; + + // get the html of this revision + exportHtml.getPadHTML(pad, rev, function(err, html) { + if (ERR(err, callback)) return; html = "" +html; // adds HTML head html += ""; var data = {html: html}; @@ -384,10 +353,9 @@ exports.getHTML = function(padID, rev, callback) return; } - //the client wants the latest text, lets return it to him - exportHtml.getPadHTML(pad, undefined, function (err, html) - { - if(ERR(err, callback)) return; + // the client wants the latest text, lets return it to him + exportHtml.getPadHTML(pad, undefined, function(err, html) { + if (ERR(err, callback)) return; html = "" +html; // adds HTML head html += ""; var data = {html: html}; @@ -406,26 +374,24 @@ Example returns: */ exports.setHTML = function(padID, html, callback) { - //html is required - if(typeof html != "string") - { - callback(new customError("html is no string","apierror")); + // html is required + if (typeof html != "string") { + callback(new customError("html is no string", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; // add a new changeset with the new html to the pad - importHtml.setPadHTML(pad, cleanText(html), function(e){ - if(e){ - callback(new customError("HTML is malformed","apierror")); + importHtml.setPadHTML(pad, cleanText(html), function(e) { + if (e) { + callback(new customError("HTML is malformed", "apierror")); return; } - //update the clients on the pad + // update the clients on the pad padMessageHandler.updatePadClients(pad, callback); }); }); @@ -449,54 +415,46 @@ Example returns: */ exports.getChatHistory = function(padID, start, end, callback) { - if(start && end) - { - if(start < 0) - { - callback(new customError("start is below zero","apierror")); + if (start && end) { + if (start < 0) { + callback(new customError("start is below zero", "apierror")); return; } - if(end < 0) - { - callback(new customError("end is below zero","apierror")); + if (end < 0) { + callback(new customError("end is below zero", "apierror")); return; } - if(start > end) - { - callback(new customError("start is higher than end","apierror")); + if (start > end) { + callback(new customError("start is higher than end", "apierror")); return; } } - - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + var chatHead = pad.chatHead; - + // fall back to getting the whole chat-history if a parameter is missing - if(!start || !end) - { + if (!start || !end) { start = 0; end = pad.chatHead; } - - if(start > chatHead) - { - callback(new customError("start is higher than the current chatHead","apierror")); + + if (start > chatHead) { + callback(new customError("start is higher than the current chatHead", "apierror")); return; } - if(end > chatHead) - { - callback(new customError("end is higher than the current chatHead","apierror")); + if (end > chatHead) { + callback(new customError("end is higher than the current chatHead", "apierror")); return; } - + // the the whole message-log and return it to the client pad.getChatMessages(start, end, - function(err, msgs) - { - if(ERR(err, callback)) return; + function(err, msgs) { + if (ERR(err, callback)) return; callback(null, {messages: msgs}); }); }); @@ -512,16 +470,14 @@ Example returns: */ exports.appendChatMessage = function(padID, text, authorID, time, callback) { - //text is required - if(typeof text != "string") - { - callback(new customError("text is no string","apierror")); + // text is required + if (typeof text != "string") { + callback(new customError("text is no string", "apierror")); return; } - + // if time is not an integer value - if(time === undefined || !is_int(time)) - { + if (time === undefined || !is_int(time)) { // set time to current timestamp time = Date.now(); } @@ -537,7 +493,7 @@ exports.appendChatMessage = function(padID, text, authorID, time, callback) /*****************/ /** -getRevisionsCount(padID) returns the number of revisions of this pad +getRevisionsCount(padID) returns the number of revisions of this pad Example returns: @@ -546,11 +502,10 @@ Example returns: */ exports.getRevisionsCount = function(padID, callback) { - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + callback(null, {revisions: pad.getHeadRevisionNumber()}); }); } @@ -565,10 +520,9 @@ Example returns: */ exports.getSavedRevisionsCount = function(padID, callback) { - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; callback(null, {savedRevisions: pad.getSavedRevisionsNumber()}); }); @@ -584,10 +538,9 @@ Example returns: */ exports.listSavedRevisions = function(padID, callback) { - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; callback(null, {savedRevisions: pad.getSavedRevisionsList()}); }); @@ -603,12 +556,10 @@ Example returns: */ exports.saveRevision = function(padID, rev, callback) { - //check if rev is a number - if(rev !== undefined && typeof rev != "number") - { - //try to parse the number - if(isNaN(parseInt(rev))) - { + // check if rev is a number + if (rev !== undefined && typeof rev != "number") { + // try to parse the number + if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); return; } @@ -616,32 +567,27 @@ exports.saveRevision = function(padID, rev, callback) rev = parseInt(rev); } - //ensure this is not a negativ number - if(rev !== undefined && rev < 0) - { - callback(new customError("rev is a negativ number","apierror")); + // ensure this is not a negative number + if (rev !== undefined && rev < 0) { + callback(new customError("rev is a negativ number", "apierror")); return; } - //ensure this is not a float value - if(rev !== undefined && !is_int(rev)) - { - callback(new customError("rev is a float value","apierror")); + // ensure this is not a float value + if (rev !== undefined && !is_int(rev)) { + callback(new customError("rev is a float value", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; - //the client asked for a special revision - if(rev !== undefined) - { - //check if this is a valid revision - if(rev > pad.getHeadRevisionNumber()) - { - callback(new customError("rev is higher than the head revision of the pad","apierror")); + // the client asked for a special revision + if (rev !== undefined) { + // check if this is a valid revision + if (rev > pad.getHeadRevisionNumber()) { + callback(new customError("rev is higher than the head revision of the pad", "apierror")); return; } } else { @@ -649,7 +595,7 @@ exports.saveRevision = function(padID, rev, callback) } authorManager.createAuthor('API', function(err, author) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; pad.addSavedRevision(rev, author.authorID, 'Saved through API call'); callback(); @@ -667,19 +613,19 @@ Example returns: */ exports.getLastEdited = function(padID, callback) { - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + pad.getLastEdit(function(err, value) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; callback(null, {lastEdited: value}); }); }); } /** -createPad(padName [, text]) creates a new pad in this group +createPad(padName [, text]) creates a new pad in this group Example returns: @@ -687,34 +633,30 @@ Example returns: {code: 1, message:"pad does already exist", data: null} */ exports.createPad = function(padID, text, callback) -{ - //ensure there is no $ in the padID - if(padID) - { - if(padID.indexOf("$") != -1) - { - callback(new customError("createPad can't create group pads","apierror")); +{ + if (padID) { + // ensure there is no $ in the padID + if (padID.indexOf("$") != -1) { + callback(new customError("createPad can't create group pads", "apierror")); return; } - //check for url special characters - if(padID.match(/(\/|\?|&|#)/)) - { - callback(new customError("malformed padID: Remove special characters","apierror")); + // check for url special characters + if (padID.match(/(\/|\?|&|#)/)) { + callback(new customError("malformed padID: Remove special characters", "apierror")); return; } } - //create pad - getPadSafe(padID, false, text, function(err) - { - if(ERR(err, callback)) return; + // create pad + getPadSafe(padID, false, text, function(err) { + if (ERR(err, callback)) return; callback(); }); } /** -deletePad(padID) deletes a pad +deletePad(padID) deletes a pad Example returns: @@ -723,10 +665,9 @@ Example returns: */ exports.deletePad = function(padID, callback) { - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + pad.remove(callback); }); } @@ -738,14 +679,12 @@ exports.deletePad = function(padID, callback) {code:0, message:"ok", data:null} {code: 1, message:"padID does not exist", data: null} */ -exports.restoreRevision = function (padID, rev, callback) +exports.restoreRevision = function(padID, rev, callback) { - //check if rev is a number - if (rev !== undefined && typeof rev != "number") - { - //try to parse the number - if (isNaN(parseInt(rev))) - { + // check if rev is a number + if (rev !== undefined && typeof rev != "number") { + // try to parse the number + if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); return; } @@ -753,51 +692,43 @@ exports.restoreRevision = function (padID, rev, callback) rev = parseInt(rev); } - //ensure this is not a negativ number - if (rev !== undefined && rev < 0) - { + // ensure this is not a negative number + if (rev !== undefined && rev < 0) { callback(new customError("rev is a negativ number", "apierror")); return; } - //ensure this is not a float value - if (rev !== undefined && !is_int(rev)) - { + // ensure this is not a float value + if (rev !== undefined && !is_int(rev)) { callback(new customError("rev is a float value", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function (err, pad) - { + // get the pad + getPadSafe(padID, true, function(err, pad) { if (ERR(err, callback)) return; - //check if this is a valid revision - if (rev > pad.getHeadRevisionNumber()) - { + // check if this is a valid revision + if (rev > pad.getHeadRevisionNumber()) { callback(new customError("rev is higher than the head revision of the pad", "apierror")); return; } - pad.getInternalRevisionAText(rev, function (err, atext) - { + pad.getInternalRevisionAText(rev, function(err, atext) { if (ERR(err, callback)) return; var oldText = pad.text(); atext.text += "\n"; - function eachAttribRun(attribs, func) - { + function eachAttribRun(attribs, func) { var attribsIter = Changeset.opIterator(attribs); var textIndex = 0; var newTextStart = 0; var newTextEnd = atext.text.length; - while (attribsIter.hasNext()) - { + while (attribsIter.hasNext()) { var op = attribsIter.next(); var nextIndex = textIndex + op.chars; - if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) - { + if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) { func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs); } textIndex = nextIndex; @@ -808,29 +739,25 @@ exports.restoreRevision = function (padID, rev, callback) var builder = Changeset.builder(oldText.length); // assemble each line into the builder - eachAttribRun(atext.attribs, function (start, end, attribs) - { + eachAttribRun(atext.attribs, function(start, end, attribs) { builder.insert(atext.text.substring(start, end), attribs); }); var lastNewlinePos = oldText.lastIndexOf('\n'); - if (lastNewlinePos < 0) - { + if (lastNewlinePos < 0) { builder.remove(oldText.length - 1, 0); - } else - { + } else { builder.remove(lastNewlinePos, oldText.match(/\n/g).length - 1); builder.remove(oldText.length - lastNewlinePos - 1, 0); } var changeset = builder.toString(); - //append the changeset + // append the changeset pad.appendRevision(changeset); - // - padMessageHandler.updatePadClients(pad, function () - { - }); + + // update the clients on the pad + padMessageHandler.updatePadClients(pad, function() {}); callback(null, null); }); @@ -838,7 +765,7 @@ exports.restoreRevision = function (padID, rev, callback) }; /** -copyPad(sourceID, destinationID[, force=false]) copies a pad. If force is true, +copyPad(sourceID, destinationID[, force=false]) copies a pad. If force is true, the destination will be overwritten if it exists. Example returns: @@ -848,16 +775,15 @@ Example returns: */ exports.copyPad = function(sourceID, destinationID, force, callback) { - getPadSafe(sourceID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + getPadSafe(sourceID, true, function(err, pad) { + if (ERR(err, callback)) return; + pad.copy(destinationID, force, callback); }); } /** -movePad(sourceID, destinationID[, force=false]) moves a pad. If force is true, +movePad(sourceID, destinationID[, force=false]) moves a pad. If force is true, the destination will be overwritten if it exists. Example returns: @@ -867,18 +793,17 @@ Example returns: */ exports.movePad = function(sourceID, destinationID, force, callback) { - getPadSafe(sourceID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + getPadSafe(sourceID, true, function(err, pad) { + if (ERR(err, callback)) return; + pad.copy(destinationID, force, function(err) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; pad.remove(callback); }); }); } /** -getReadOnlyLink(padID) returns the read only link of a pad +getReadOnlyLink(padID) returns the read only link of a pad Example returns: @@ -887,15 +812,13 @@ Example returns: */ exports.getReadOnlyID = function(padID, callback) { - //we don't need the pad object, but this function does all the security stuff for us - getPadSafe(padID, true, function(err) - { - if(ERR(err, callback)) return; - - //get the readonlyId - readOnlyManager.getReadOnlyId(padID, function(err, readOnlyId) - { - if(ERR(err, callback)) return; + // we don't need the pad object, but this function does all the security stuff for us + getPadSafe(padID, true, function(err) { + if (ERR(err, callback)) return; + + // get the readonlyId + readOnlyManager.getReadOnlyId(padID, function(err, readOnlyId) { + if (ERR(err, callback)) return; callback(null, {readOnlyID: readOnlyId}); }); }); @@ -911,14 +834,12 @@ Example returns: */ exports.getPadID = function(roID, callback) { - //get the PadId - readOnlyManager.getPadId(roID, function(err, retrievedPadID) - { - if(ERR(err, callback)) return; + // get the PadId + readOnlyManager.getPadId(roID, function(err, retrievedPadID) { + if (ERR(err, callback)) return; - if(retrievedPadID == null) - { - callback(new customError("padID does not exist","apierror")); + if (retrievedPadID == null) { + callback(new customError("padID does not exist", "apierror")); return; } @@ -927,7 +848,7 @@ exports.getPadID = function(roID, callback) } /** -setPublicStatus(padID, publicStatus) sets a boolean for the public status of a pad +setPublicStatus(padID, publicStatus) sets a boolean for the public status of a pad Example returns: @@ -936,31 +857,29 @@ Example returns: */ exports.setPublicStatus = function(padID, publicStatus, callback) { - //ensure this is a group pad - if(padID && padID.indexOf("$") == -1) - { - callback(new customError("You can only get/set the publicStatus of pads that belong to a group","apierror")); + // ensure this is a group pad + if (padID && padID.indexOf("$") == -1) { + callback(new customError("You can only get/set the publicStatus of pads that belong to a group", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - - //convert string to boolean - if(typeof publicStatus == "string") + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + + // convert string to boolean + if (typeof publicStatus == "string") publicStatus = publicStatus == "true" ? true : false; - - //set the password + + // set the password pad.setPublicStatus(publicStatus); - + callback(); }); } /** -getPublicStatus(padID) return true of false +getPublicStatus(padID) return true of false Example returns: @@ -969,24 +888,22 @@ Example returns: */ exports.getPublicStatus = function(padID, callback) { - //ensure this is a group pad - if(padID && padID.indexOf("$") == -1) - { - callback(new customError("You can only get/set the publicStatus of pads that belong to a group","apierror")); + // ensure this is a group pad + if (padID && padID.indexOf("$") == -1) { + callback(new customError("You can only get/set the publicStatus of pads that belong to a group", "apierror")); return; } - - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + callback(null, {publicStatus: pad.getPublicStatus()}); }); } /** -setPassword(padID, password) returns ok or a error message +setPassword(padID, password) returns ok or a error message Example returns: @@ -995,27 +912,25 @@ Example returns: */ exports.setPassword = function(padID, password, callback) { - //ensure this is a group pad - if(padID && padID.indexOf("$") == -1) - { - callback(new customError("You can only get/set the password of pads that belong to a group","apierror")); + // ensure this is a group pad + if (padID && padID.indexOf("$") == -1) { + callback(new customError("You can only get/set the password of pads that belong to a group", "apierror")); return; } - - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - - //set the password + + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + + // set the password pad.setPassword(password == "" ? null : password); - + callback(); }); } /** -isPasswordProtected(padID) returns true or false +isPasswordProtected(padID) returns true or false Example returns: @@ -1024,24 +939,22 @@ Example returns: */ exports.isPasswordProtected = function(padID, callback) { - //ensure this is a group pad - if(padID && padID.indexOf("$") == -1) - { - callback(new customError("You can only get/set the password of pads that belong to a group","apierror")); + // ensure this is a group pad + if (padID && padID.indexOf("$") == -1) { + callback(new customError("You can only get/set the password of pads that belong to a group", "apierror")); return; } - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + callback(null, {isPasswordProtected: pad.isPasswordProtected()}); }); } /** -listAuthorsOfPad(padID) returns an array of authors who contributed to this pad +listAuthorsOfPad(padID) returns an array of authors who contributed to this pad Example returns: @@ -1050,11 +963,10 @@ Example returns: */ exports.listAuthorsOfPad = function(padID, callback) { - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; - + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + callback(null, {authorIDs: pad.getAllAuthors()}); }); } @@ -1082,8 +994,8 @@ Example returns: {code: 1, message:"padID does not exist"} */ -exports.sendClientsMessage = function (padID, msg, callback) { - getPadSafe(padID, true, function (err, pad) { +exports.sendClientsMessage = function(padID, msg, callback) { + getPadSafe(padID, true, function(err, pad) { if (ERR(err, callback)) { return; } @@ -1115,10 +1027,10 @@ Example returns: */ exports.getChatHead = function(padID, callback) { - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(ERR(err, callback)) return; + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (ERR(err, callback)) return; + callback(null, {chatHead: pad.chatHead}); }); } @@ -1131,69 +1043,64 @@ Example returns: {"code":0,"message":"ok","data":{"html":"Welcome to Etherpad!

This pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!

Get involved with Etherpad at http://etherpad.org
aw

","authors":["a.HKIv23mEbachFYfH",""]}} {"code":4,"message":"no or wrong API Key","data":null} */ -exports.createDiffHTML = function(padID, startRev, endRev, callback){ - //check if rev is a number - if(startRev !== undefined && typeof startRev != "number") - { - //try to parse the number - if(isNaN(parseInt(startRev))) - { +exports.createDiffHTML = function(padID, startRev, endRev, callback) { + // check if startRev is a number + if (startRev !== undefined && typeof startRev != "number") { + // try to parse the number + if (isNaN(parseInt(startRev))) { callback({stop: "startRev is not a number"}); return; } startRev = parseInt(startRev, 10); } - - //check if rev is a number - if(endRev !== undefined && typeof endRev != "number") - { - //try to parse the number - if(isNaN(parseInt(endRev))) - { + + // check if endRev is a number + if (endRev !== undefined && typeof endRev != "number") { + // try to parse the number + if (isNaN(parseInt(endRev))) { callback({stop: "endRev is not a number"}); return; } endRev = parseInt(endRev, 10); } - - //get the pad - getPadSafe(padID, true, function(err, pad) - { - if(err){ + + // get the pad + getPadSafe(padID, true, function(err, pad) { + if (err) { return callback(err); } - + try { var padDiff = new PadDiff(pad, startRev, endRev); } catch(e) { return callback({stop:e.message}); } var html, authors; - + async.series([ - function(callback){ - padDiff.getHtml(function(err, _html){ - if(err){ + function(callback) { + padDiff.getHtml(function(err, _html) { + if (err) { return callback(err); } - + html = _html; callback(); }); }, - function(callback){ - padDiff.getAuthors(function(err, _authors){ - if(err){ + function(callback) { + padDiff.getAuthors(function(err, _authors) { + if (err) { return callback(err); } - + authors = _authors; callback(); }); } - ], function(err){ + ], function(err) { callback(err, {html: html, authors: authors}) }); }); @@ -1203,53 +1110,44 @@ exports.createDiffHTML = function(padID, startRev, endRev, callback){ /** INTERNAL HELPER FUNCTIONS */ /******************************/ -//checks if a number is an int +// checks if a number is an int function is_int(value) -{ +{ return (parseFloat(value) == parseInt(value)) && !isNaN(value) } -//gets a pad safe +// gets a pad safe function getPadSafe(padID, shouldExist, text, callback) { - if(typeof text == "function") - { + if (typeof text == "function") { callback = text; text = null; } - //check if padID is a string - if(typeof padID != "string") - { - callback(new customError("padID is not a string","apierror")); + // check if padID is a string + if (typeof padID != "string") { + callback(new customError("padID is not a string", "apierror")); return; } - - //check if the padID maches the requirements - if(!padManager.isValidPadId(padID)) - { - callback(new customError("padID did not match requirements","apierror")); + + // check if the padID maches the requirements + if (!padManager.isValidPadId(padID)) { + callback(new customError("padID did not match requirements", "apierror")); return; } - - //check if the pad exists - padManager.doesPadExists(padID, function(err, exists) - { - if(ERR(err, callback)) return; - - //does not exist, but should - if(exists == false && shouldExist == true) - { - callback(new customError("padID does not exist","apierror")); - } - //does exists, but shouldn't - else if(exists == true && shouldExist == false) - { - callback(new customError("padID does already exist","apierror")); - } - //pad exists, let's get it - else - { + + // check if the pad exists + padManager.doesPadExists(padID, function(err, exists) { + if (ERR(err, callback)) return; + + // does not exist, but should + if (exists == false && shouldExist == true) { + callback(new customError("padID does not exist", "apierror")); + } else if (exists == true && shouldExist == false) { + // does exist, but shouldn't + callback(new customError("padID does already exist", "apierror")); + } else { + // pad exists, let's get it padManager.getPad(padID, text, callback); } }); diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index bcb6d393d..818eaab28 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -18,25 +18,33 @@ * limitations under the License. */ - var ERR = require("async-stacktrace"); var db = require("./DB").db; var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; -exports.getColorPalette = function(){ - return ["#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1", "#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5", "#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6", "#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9", "#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8", "#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7", "#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8", "#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6"]; +exports.getColorPalette = function() { + return [ + "#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1", + "#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5", + "#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6", + "#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9", + "#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8", + "#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7", + "#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8", + "#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6" + ]; }; /** * Checks if the author exists */ -exports.doesAuthorExists = function (authorID, callback) +exports.doesAuthorExists = function(authorID, callback) { - //check if the database entry of this author exists - db.get("globalAuthor:" + authorID, function (err, author) - { - if(ERR(err, callback)) return; + // check if the database entry of this author exists + db.get("globalAuthor:" + authorID, function(err, author) { + if (ERR(err, callback)) return; + callback(null, author != null); }); } @@ -46,12 +54,12 @@ exports.doesAuthorExists = function (authorID, callback) * @param {String} token The token * @param {Function} callback callback (err, author) */ -exports.getAuthor4Token = function (token, callback) +exports.getAuthor4Token = function(token, callback) { - mapAuthorWithDBKey("token2author", token, function(err, author) - { - if(ERR(err, callback)) return; - //return only the sub value authorID + mapAuthorWithDBKey("token2author", token, function(err, author) { + if (ERR(err, callback)) return; + + // return only the sub value authorID callback(null, author ? author.authorID : author); }); } @@ -62,17 +70,17 @@ exports.getAuthor4Token = function (token, callback) * @param {String} name The name of the author (optional) * @param {Function} callback callback (err, author) */ -exports.createAuthorIfNotExistsFor = function (authorMapper, name, callback) +exports.createAuthorIfNotExistsFor = function(authorMapper, name, callback) { - mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author) - { - if(ERR(err, callback)) return; + mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author) { + if (ERR(err, callback)) return; - //set the name of this author - if(name) + if (name) { + // set the name of this author exports.setAuthorName(author.authorID, name); + } - //return the authorID + // return the authorID callback(null, author); }); } @@ -86,33 +94,30 @@ exports.createAuthorIfNotExistsFor = function (authorMapper, name, callback) */ function mapAuthorWithDBKey (mapperkey, mapper, callback) { - //try to map to an author - db.get(mapperkey + ":" + mapper, function (err, author) - { - if(ERR(err, callback)) return; + // try to map to an author + db.get(mapperkey + ":" + mapper, function(err, author) { + if (ERR(err, callback)) return; - //there is no author with this mapper, so create one - if(author == null) - { - exports.createAuthor(null, function(err, author) - { - if(ERR(err, callback)) return; + if (author == null) { + // there is no author with this mapper, so create one + exports.createAuthor(null, function(err, author) { + if (ERR(err, callback)) return; - //create the token2author relation + // create the token2author relation db.set(mapperkey + ":" + mapper, author.authorID); - //return the author + // return the author callback(null, author); }); return; } - //there is a author with this mapper - //update the timestamp of this author + // there is an author with this mapper + // update the timestamp of this author db.setSub("globalAuthor:" + author, ["timestamp"], Date.now()); - //return the author + // return the author callback(null, {authorID: author}); }); } @@ -123,13 +128,17 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) */ exports.createAuthor = function(name, callback) { - //create the new author name + // create the new author name var author = "a." + randomString(16); - //create the globalAuthors db entry - var authorObj = {"colorId" : Math.floor(Math.random()*(exports.getColorPalette().length)), "name": name, "timestamp": Date.now()}; + // create the globalAuthors db entry + var authorObj = { + "colorId": Math.floor(Math.random() * (exports.getColorPalette().length)), + "name": name, + "timestamp": Date.now() + }; - //set the global author db entry + // set the global author db entry db.set("globalAuthor:" + author, authorObj); callback(null, {authorID: author}); @@ -140,19 +149,17 @@ exports.createAuthor = function(name, callback) * @param {String} author The id of the author * @param {Function} callback callback(err, authorObj) */ -exports.getAuthor = function (author, callback) +exports.getAuthor = function(author, callback) { db.get("globalAuthor:" + author, callback); } - - /** * Returns the color Id of the author * @param {String} author The id of the author * @param {Function} callback callback(err, colorId) */ -exports.getAuthorColorId = function (author, callback) +exports.getAuthorColorId = function(author, callback) { db.getSub("globalAuthor:" + author, ["colorId"], callback); } @@ -163,7 +170,7 @@ exports.getAuthorColorId = function (author, callback) * @param {String} colorId The color id of the author * @param {Function} callback (optional) */ -exports.setAuthorColorId = function (author, colorId, callback) +exports.setAuthorColorId = function(author, colorId, callback) { db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback); } @@ -173,7 +180,7 @@ exports.setAuthorColorId = function (author, colorId, callback) * @param {String} author The id of the author * @param {Function} callback callback(err, name) */ -exports.getAuthorName = function (author, callback) +exports.getAuthorName = function(author, callback) { db.getSub("globalAuthor:" + author, ["name"], callback); } @@ -184,7 +191,7 @@ exports.getAuthorName = function (author, callback) * @param {String} name The name of the author * @param {Function} callback (optional) */ -exports.setAuthorName = function (author, name, callback) +exports.setAuthorName = function(author, name, callback) { db.setSub("globalAuthor:" + author, ["name"], name, callback); } @@ -194,33 +201,33 @@ exports.setAuthorName = function (author, name, callback) * @param {String} author The id of the author * @param {Function} callback (optional) */ -exports.listPadsOfAuthor = function (authorID, callback) +exports.listPadsOfAuthor = function(authorID, callback) { /* There are two other places where this array is manipulated: * (1) When the author is added to a pad, the author object is also updated * (2) When a pad is deleted, each author of that pad is also updated */ - //get the globalAuthor - db.get("globalAuthor:" + authorID, function(err, author) - { - if(ERR(err, callback)) return; - //author does not exists - if(author == null) - { - callback(new customError("authorID does not exist","apierror")) + // get the globalAuthor + db.get("globalAuthor:" + authorID, function(err, author) { + if (ERR(err, callback)) return; + + if (author == null) { + // author does not exist + callback(new customError("authorID does not exist", "apierror")); + return; } - //everything is fine, return the pad IDs + // everything is fine, return the pad IDs var pads = []; - if(author.padIDs != null) - { - for (var padId in author.padIDs) - { + + if (author.padIDs != null) { + for (var padId in author.padIDs) { pads.push(padId); } } + callback(null, {padIDs: pads}); }); } @@ -230,24 +237,22 @@ exports.listPadsOfAuthor = function (authorID, callback) * @param {String} author The id of the author * @param {String} padID The id of the pad the author contributes to */ -exports.addPad = function (authorID, padID) +exports.addPad = function(authorID, padID) { - //get the entry - db.get("globalAuthor:" + authorID, function(err, author) - { - if(ERR(err)) return; - if(author == null) return; + // get the entry + db.get("globalAuthor:" + authorID, function(err, author) { + if (ERR(err)) return; + if (author == null) return; - //the entry doesn't exist so far, let's create it - if(author.padIDs == null) - { + if (author.padIDs == null) { + // the entry doesn't exist so far, let's create it author.padIDs = {}; } - //add the entry for this pad - author.padIDs[padID] = 1;// anything, because value is not used + // add the entry for this pad + author.padIDs[padID] = 1; // anything, because value is not used - //save the new element back + // save the new element back db.set("globalAuthor:" + authorID, author); }); } @@ -257,16 +262,14 @@ exports.addPad = function (authorID, padID) * @param {String} author The id of the author * @param {String} padID The id of the pad the author contributes to */ -exports.removePad = function (authorID, padID) +exports.removePad = function(authorID, padID) { - db.get("globalAuthor:" + authorID, function (err, author) - { - if(ERR(err)) return; - if(author == null) return; + db.get("globalAuthor:" + authorID, function(err, author) { + if (ERR(err)) return; + if (author == null) return; - if(author.padIDs != null) - { - //remove pad from author + if (author.padIDs != null) { + // remove pad from author delete author.padIDs[padID]; db.set("globalAuthor:" + authorID, author); } diff --git a/src/node/db/DB.js b/src/node/db/DB.js index 3c65d5cdb..93848b1bd 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -1,5 +1,5 @@ /** - * The DB Module provides a database initalized with the settings + * The DB Module provides a database initalized with the settings * provided by the settings module */ @@ -23,7 +23,7 @@ var ueberDB = require("ueberdb2"); var settings = require("../utils/Settings"); var log4js = require('log4js'); -//set database settings +// set database settings var db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB")); /** @@ -33,24 +33,19 @@ exports.db = null; /** * Initalizes the database with the settings provided by the settings module - * @param {Function} callback + * @param {Function} callback */ -exports.init = function(callback) -{ - //initalize the database async - db.init(function(err) - { - //there was an error while initializing the database, output it and stop - if(err) - { +exports.init = function(callback) { + // initalize the database async + db.init(function(err) { + if (err) { + // there was an error while initializing the database, output it and stop console.error("ERROR: Problem while initalizing the database"); console.error(err.stack ? err.stack : err); process.exit(1); - } - //everything ok - else - { - exports.db = db; + } else { + // everything ok + exports.db = db; callback(null); } }); diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index 0c9be1221..5f8183ea9 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -17,7 +17,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); @@ -29,117 +28,112 @@ var sessionManager = require("./SessionManager"); exports.listAllGroups = function(callback) { db.get("groups", function (err, groups) { - if(ERR(err, callback)) return; - - // there are no groups - if(groups == null) { + if (ERR(err, callback)) return; + + if (groups == null) { + // there are no groups callback(null, {groupIDs: []}); + return; } - + var groupIDs = []; - for ( var groupID in groups ) { + for (var groupID in groups) { groupIDs.push(groupID); } callback(null, {groupIDs: groupIDs}); }); } - + exports.deleteGroup = function(groupID, callback) { var group; async.series([ - //ensure group exists - function (callback) - { - //try to get the group entry - db.get("group:" + groupID, function (err, _group) - { - if(ERR(err, callback)) return; - - //group does not exist - if(_group == null) - { - callback(new customError("groupID does not exist","apierror")); + // ensure group exists + function (callback) { + // try to get the group entry + db.get("group:" + groupID, function (err, _group) { + if (ERR(err, callback)) return; + + if (_group == null) { + // group does not exist + callback(new customError("groupID does not exist", "apierror")); return; } - //group exists, everything is fine + // group exists, everything is fine group = _group; callback(); }); }, - //iterate trough all pads of this groups and delete them - function(callback) - { - //collect all padIDs in an array, that allows us to use async.forEach + + // iterate through all pads of this group and delete them + function(callback) { + // collect all padIDs in an array, that allows us to use async.forEach var padIDs = []; - for(var i in group.pads) - { + for(var i in group.pads) { padIDs.push(i); } - - //loop trough all pads and delete them - async.forEach(padIDs, function(padID, callback) - { - padManager.getPad(padID, function(err, pad) - { - if(ERR(err, callback)) return; - + + // loop through all pads and delete them + async.forEach(padIDs, function(padID, callback) { + padManager.getPad(padID, function(err, pad) { + if (ERR(err, callback)) return; + pad.remove(callback); }); }, callback); }, - //iterate trough group2sessions and delete all sessions + + // iterate through group2sessions and delete all sessions function(callback) { - //try to get the group entry - db.get("group2sessions:" + groupID, function (err, group2sessions) - { - if(ERR(err, callback)) return; - - //skip if there is no group2sessions entry - if(group2sessions == null) {callback(); return} - - //collect all sessions in an array, that allows us to use async.forEach + // try to get the group entry + db.get("group2sessions:" + groupID, function (err, group2sessions) { + if (ERR(err, callback)) return; + + // skip if there is no group2sessions entry + if (group2sessions == null) { callback(); return } + + // collect all sessions in an array, that allows us to use async.forEach var sessions = []; - for(var i in group2sessions.sessionsIDs) - { + for (var i in group2sessions.sessionsIDs) { sessions.push(i); } - - //loop trough all sessions and delete them - async.forEach(sessions, function(session, callback) - { + + // loop through all sessions and delete them + async.forEach(sessions, function(session, callback) { sessionManager.deleteSession(session, callback); }, callback); }); }, - //remove group and group2sessions entry - function(callback) - { + + // remove group and group2sessions entry + function(callback) { db.remove("group2sessions:" + groupID); db.remove("group:" + groupID); callback(); }, - //unlist the group - function(callback) - { + + // unlist the group + function(callback) { exports.listAllGroups(function(err, groups) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; groups = groups? groups.groupIDs : []; - // it's not listed - if(groups.indexOf(groupID) == -1) { + if (groups.indexOf(groupID) == -1) { + // it's not listed callback(); + return; } + // remove from the list groups.splice(groups.indexOf(groupID), 1); - - // store empty groupe list - if(groups.length == 0) { + + // store empty group list + if (groups.length == 0) { db.set("groups", {}); callback(); return; @@ -150,50 +144,51 @@ exports.deleteGroup = function(groupID, callback) async.forEach(groups, function(group, cb) { newGroups[group] = 1; cb(); - },function() { + }, + function() { db.set("groups", newGroups); callback(); }); }); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; callback(); }); } - + exports.doesGroupExist = function(groupID, callback) { - //try to get the group entry - db.get("group:" + groupID, function (err, group) - { - if(ERR(err, callback)) return; + // try to get the group entry + db.get("group:" + groupID, function (err, group) { + if (ERR(err, callback)) return; callback(null, group != null); }); } exports.createGroup = function(callback) { - //search for non existing groupID + // search for non existing groupID var groupID = "g." + randomString(16); - - //create the group + + // create the group db.set("group:" + groupID, {pads: {}}); - - //list the group + + // list the group exports.listAllGroups(function(err, groups) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; + groups = groups? groups.groupIDs : []; - groups.push(groupID); - + // regenerate group list var newGroups = {}; async.forEach(groups, function(group, cb) { newGroups[group] = 1; cb(); - },function() { + }, + function() { db.set("groups", newGroups); callback(null, {groupID: groupID}); }); @@ -202,129 +197,121 @@ exports.createGroup = function(callback) exports.createGroupIfNotExistsFor = function(groupMapper, callback) { - //ensure mapper is optional - if(typeof groupMapper != "string") - { - callback(new customError("groupMapper is no string","apierror")); + // ensure mapper is optional + if (typeof groupMapper != "string") { + callback(new customError("groupMapper is no string", "apierror")); return; } - - //try to get a group for this mapper - db.get("mapper2group:"+groupMapper, function(err, groupID) - { + + // try to get a group for this mapper + db.get("mapper2group:" + groupMapper, function(err, groupID) { function createGroupForMapper(cb) { - exports.createGroup(function(err, responseObj) - { - if(ERR(err, cb)) return; - - //create the mapper entry for this group - db.set("mapper2group:"+groupMapper, responseObj.groupID); - + exports.createGroup(function(err, responseObj) { + if (ERR(err, cb)) return; + + // create the mapper entry for this group + db.set("mapper2group:" + groupMapper, responseObj.groupID); + cb(null, responseObj); }); } - if(ERR(err, callback)) return; - - // there is a group for this mapper - if(groupID) { + if (ERR(err, callback)) return; + + if (groupID) { + // there is a group for this mapper exports.doesGroupExist(groupID, function(err, exists) { - if(ERR(err, callback)) return; - if(exists) return callback(null, {groupID: groupID}); + if (ERR(err, callback)) return; + + if (exists) return callback(null, {groupID: groupID}); // hah, the returned group doesn't exist, let's create one createGroupForMapper(callback) - }) + }); return; } - //there is no group for this mapper, let's create a group + // there is no group for this mapper, let's create a group createGroupForMapper(callback) }); } exports.createGroupPad = function(groupID, padName, text, callback) { - //create the padID + // create the padID var padID = groupID + "$" + padName; async.series([ - //ensure group exists - function (callback) - { - exports.doesGroupExist(groupID, function(err, exists) - { - if(ERR(err, callback)) return; - - //group does not exist - if(exists == false) - { - callback(new customError("groupID does not exist","apierror")); + // ensure group exists + function (callback) { + exports.doesGroupExist(groupID, function(err, exists) { + if (ERR(err, callback)) return; + + if (exists == false) { + // group does not exist + callback(new customError("groupID does not exist", "apierror")); return; } - //group exists, everything is fine + // group exists, everything is fine callback(); }); }, - //ensure pad does not exists - function (callback) - { - padManager.doesPadExists(padID, function(err, exists) - { - if(ERR(err, callback)) return; - - //pad exists already - if(exists == true) - { - callback(new customError("padName does already exist","apierror")); + + // ensure pad doesn't exist already + function (callback) { + padManager.doesPadExists(padID, function(err, exists) { + if (ERR(err, callback)) return; + + if (exists == true) { + // pad exists already + callback(new customError("padName does already exist", "apierror")); return; } - //pad does not exist, everything is fine + // pad does not exist, everything is fine callback(); }); }, - //create the pad - function (callback) - { - padManager.getPad(padID, text, function(err) - { - if(ERR(err, callback)) return; + + // create the pad + function (callback) { + padManager.getPad(padID, text, function(err) { + if (ERR(err, callback)) return; + callback(); }); }, - //create an entry in the group for this pad - function (callback) - { + + // create an entry in the group for this pad + function (callback) { db.setSub("group:" + groupID, ["pads", padID], 1); callback(); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; + callback(null, {padID: padID}); }); } exports.listPads = function(groupID, callback) { - exports.doesGroupExist(groupID, function(err, exists) - { - if(ERR(err, callback)) return; - - //group does not exist - if(exists == false) - { - callback(new customError("groupID does not exist","apierror")); + exports.doesGroupExist(groupID, function(err, exists) { + if (ERR(err, callback)) return; + + // ensure the group exists + if (exists == false) { + callback(new customError("groupID does not exist", "apierror")); return; } - //group exists, let's get the pads - db.getSub("group:" + groupID, ["pads"], function(err, result) - { - if(ERR(err, callback)) return; + // group exists, let's get the pads + db.getSub("group:" + groupID, ["pads"], function(err, result) { + if (ERR(err, callback)) return; + var pads = []; for ( var padId in result ) { pads.push(padId); diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index b74de5228..c5a75a89f 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -19,7 +19,7 @@ var crypto = require("crypto"); var randomString = require("../utils/randomstring"); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); -//serialization/deserialization attributes +// serialization/deserialization attributes var attributeBlackList = ["id"]; var jsonableList = ["pool"]; @@ -33,7 +33,6 @@ exports.cleanText = function (txt) { var Pad = function Pad(id) { - this.atext = Changeset.makeAText("\n"); this.pool = new AttributePool(); this.head = -1; @@ -60,7 +59,7 @@ Pad.prototype.getSavedRevisionsNumber = function getSavedRevisionsNumber() { Pad.prototype.getSavedRevisionsList = function getSavedRevisionsList() { var savedRev = new Array(); - for(var rev in this.savedRevisions){ + for (var rev in this.savedRevisions) { savedRev.push(this.savedRevisions[rev].revNum); } savedRev.sort(function(a, b) { @@ -74,8 +73,9 @@ Pad.prototype.getPublicStatus = function getPublicStatus() { }; Pad.prototype.appendRevision = function appendRevision(aChangeset, author) { - if(!author) + if (!author) { author = ''; + } var newAText = Changeset.applyToAText(aChangeset, this.atext, this.pool); Changeset.copyAText(newAText, this.atext); @@ -88,21 +88,22 @@ Pad.prototype.appendRevision = function appendRevision(aChangeset, author) { newRevData.meta.author = author; newRevData.meta.timestamp = Date.now(); - //ex. getNumForAuthor - if(author != '') + // ex. getNumForAuthor + if (author != '') { this.pool.putAttrib(['author', author || '']); + } - if(newRev % 100 == 0) - { + if (newRev % 100 == 0) { newRevData.meta.atext = this.atext; } - db.set("pad:"+this.id+":revs:"+newRev, newRevData); + db.set("pad:" + this.id + ":revs:" + newRev, newRevData); this.saveToDatabase(); // set the author to pad - if(author) + if (author) { authorManager.addPad(author, this.id); + } if (this.head == 0) { hooks.callAll("padCreate", {'pad':this, 'author': author}); @@ -111,49 +112,47 @@ Pad.prototype.appendRevision = function appendRevision(aChangeset, author) { } }; -//save all attributes to the database -Pad.prototype.saveToDatabase = function saveToDatabase(){ +// save all attributes to the database +Pad.prototype.saveToDatabase = function saveToDatabase() { var dbObject = {}; - for(var attr in this){ - if(typeof this[attr] === "function") continue; - if(attributeBlackList.indexOf(attr) !== -1) continue; + for (var attr in this) { + if (typeof this[attr] === "function") continue; + if (attributeBlackList.indexOf(attr) !== -1) continue; dbObject[attr] = this[attr]; - if(jsonableList.indexOf(attr) !== -1){ + if (jsonableList.indexOf(attr) !== -1) { dbObject[attr] = dbObject[attr].toJsonable(); } } - db.set("pad:"+this.id, dbObject); + db.set("pad:" + this.id, dbObject); } // get time of last edit (changeset application) -Pad.prototype.getLastEdit = function getLastEdit(callback){ +Pad.prototype.getLastEdit = function getLastEdit(callback) { var revNum = this.getHeadRevisionNumber(); - db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "timestamp"], callback); + db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); } Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum, callback) { - db.getSub("pad:"+this.id+":revs:"+revNum, ["changeset"], callback); + db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"], callback); }; Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum, callback) { - db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "author"], callback); + db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"], callback); }; Pad.prototype.getRevisionDate = function getRevisionDate(revNum, callback) { - db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "timestamp"], callback); + db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); }; Pad.prototype.getAllAuthors = function getAllAuthors() { var authors = []; - for(var key in this.pool.numToAttrib) - { - if(this.pool.numToAttrib[key][0] == "author" && this.pool.numToAttrib[key][1] != "") - { + for(var key in this.pool.numToAttrib) { + if (this.pool.numToAttrib[key][0] == "author" && this.pool.numToAttrib[key][1] != "") { authors.push(this.pool.numToAttrib[key][1]); } } @@ -168,26 +167,22 @@ Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targe var atext; var changesets = []; - //find out which changesets are needed + // find out which changesets are needed var neededChangesets = []; var curRev = keyRev; - while (curRev < targetRev) - { + while (curRev < targetRev) { curRev++; neededChangesets.push(curRev); } async.series([ - //get all needed data out of the database - function(callback) - { + // get all needed data out of the database + function(callback) { async.parallel([ - //get the atext of the key revision - function (callback) - { - db.getSub("pad:"+_this.id+":revs:"+keyRev, ["meta", "atext"], function(err, _atext) - { - if(ERR(err, callback)) return; + // get the atext of the key revision + function (callback) { + db.getSub("pad:" + _this.id + ":revs:" + keyRev, ["meta", "atext"], function(err, _atext) { + if (ERR(err, callback)) return; try { atext = Changeset.cloneAText(_atext); } catch (e) { @@ -197,14 +192,12 @@ Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targe callback(); }); }, - //get all needed changesets - function (callback) - { - async.forEach(neededChangesets, function(item, callback) - { - _this.getRevisionChangeset(item, function(err, changeset) - { - if(ERR(err, callback)) return; + + // get all needed changesets + function (callback) { + async.forEach(neededChangesets, function(item, callback) { + _this.getRevisionChangeset(item, function(err, changeset) { + if (ERR(err, callback)) return; changesets[item] = changeset; callback(); }); @@ -212,52 +205,53 @@ Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targe } ], callback); }, - //apply all changesets to the key changeset - function(callback) - { + + // apply all changesets to the key changeset + function(callback) { var apool = _this.apool(); var curRev = keyRev; - while (curRev < targetRev) - { + while (curRev < targetRev) { curRev++; var cs = changesets[curRev]; - try{ + try { atext = Changeset.applyToAText(cs, atext, apool); - }catch(e) { + } catch(e) { return callback(e) } } callback(null); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; callback(null, atext); }); }; Pad.prototype.getRevision = function getRevisionChangeset(revNum, callback) { - db.get("pad:"+this.id+":revs:"+revNum, callback); + db.get("pad:" + this.id + ":revs:" + revNum, callback); }; -Pad.prototype.getAllAuthorColors = function getAllAuthorColors(callback){ +Pad.prototype.getAllAuthorColors = function getAllAuthorColors(callback) { var authors = this.getAllAuthors(); var returnTable = {}; var colorPalette = authorManager.getColorPalette(); - async.forEach(authors, function(author, callback){ - authorManager.getAuthorColorId(author, function(err, colorId){ - if(err){ + async.forEach(authors, function(author, callback) { + authorManager.getAuthorColorId(author, function(err, colorId) { + if (err) { return callback(err); } - //colorId might be a hex color or an number out of the palette - returnTable[author]=colorPalette[colorId] || colorId; + + // colorId might be a hex color or an number out of the palette + returnTable[author] = colorPalette[colorId] || colorId; callback(); }); - }, function(err){ + }, + function(err) { callback(err, returnTable); }); }; @@ -266,15 +260,18 @@ Pad.prototype.getValidRevisionRange = function getValidRevisionRange(startRev, e startRev = parseInt(startRev, 10); var head = this.getHeadRevisionNumber(); endRev = endRev ? parseInt(endRev, 10) : head; - if(isNaN(startRev) || startRev < 0 || startRev > head) { + + if (isNaN(startRev) || startRev < 0 || startRev > head) { startRev = null; } - if(isNaN(endRev) || endRev < startRev) { + + if (isNaN(endRev) || endRev < startRev) { endRev = null; - } else if(endRev > head) { + } else if (endRev > head) { endRev = head; } - if(startRev !== null && endRev !== null) { + + if (startRev !== null && endRev !== null) { return { startRev: startRev , endRev: endRev } } return null; @@ -289,12 +286,12 @@ Pad.prototype.text = function text() { }; Pad.prototype.setText = function setText(newText) { - //clean the new text + // clean the new text newText = exports.cleanText(newText); var oldText = this.text(); - //create the changeset + // create the changeset // We want to ensure the pad still ends with a \n, but otherwise keep // getText() and setText() consistent. var changeset; @@ -304,27 +301,27 @@ Pad.prototype.setText = function setText(newText) { changeset = Changeset.makeSplice(oldText, 0, oldText.length-1, newText); } - //append the changeset + // append the changeset this.appendRevision(changeset); }; Pad.prototype.appendText = function appendText(newText) { - //clean the new text + // clean the new text newText = exports.cleanText(newText); var oldText = this.text(); - //create the changeset + // create the changeset var changeset = Changeset.makeSplice(oldText, oldText.length, 0, newText); - //append the changeset + // append the changeset this.appendRevision(changeset); }; Pad.prototype.appendChatMessage = function appendChatMessage(text, userId, time) { this.chatHead++; - //save the chat entry in the database - db.set("pad:"+this.id+":chat:"+this.chatHead, {"text": text, "userId": userId, "time": time}); + // save the chat entry in the database + db.set("pad:" + this.id + ":chat:" + this.chatHead, { "text": text, "userId": userId, "time": time }); this.saveToDatabase(); }; @@ -333,77 +330,70 @@ Pad.prototype.getChatMessage = function getChatMessage(entryNum, callback) { var entry; async.series([ - //get the chat entry - function(callback) - { - db.get("pad:"+_this.id+":chat:"+entryNum, function(err, _entry) - { - if(ERR(err, callback)) return; + // get the chat entry + function(callback) { + db.get("pad:" + _this.id + ":chat:" + entryNum, function(err, _entry) { + if (ERR(err, callback)) return; entry = _entry; callback(); }); }, - //add the authorName - function(callback) - { - //this chat message doesn't exist, return null - if(entry == null) - { + + // add the authorName + function(callback) { + // this chat message doesn't exist, return null + if (entry == null) { callback(); return; } - //get the authorName - authorManager.getAuthorName(entry.userId, function(err, authorName) - { - if(ERR(err, callback)) return; + // get the authorName + authorManager.getAuthorName(entry.userId, function(err, authorName) { + if (ERR(err, callback)) return; entry.userName = authorName; callback(); }); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; callback(null, entry); }); }; Pad.prototype.getChatMessages = function getChatMessages(start, end, callback) { - //collect the numbers of chat entries and in which order we need them + // collect the numbers of chat entries and in which order we need them var neededEntries = []; var order = 0; - for(var i=start;i<=end; i++) - { - neededEntries.push({entryNum:i, order: order}); + for (var i = start; i <= end; i++) { + neededEntries.push({ entryNum: i, order: order }); order++; } var _this = this; - //get all entries out of the database + // get all entries out of the database var entries = []; - async.forEach(neededEntries, function(entryObject, callback) - { - _this.getChatMessage(entryObject.entryNum, function(err, entry) - { - if(ERR(err, callback)) return; + async.forEach(neededEntries, function(entryObject, callback) { + _this.getChatMessage(entryObject.entryNum, function(err, entry) { + if (ERR(err, callback)) return; entries[entryObject.order] = entry; callback(); }); - }, function(err) - { - if(ERR(err, callback)) return; + }, + function(err) { + if (ERR(err, callback)) return; - //sort out broken chat entries - //it looks like in happend in the past that the chat head was - //incremented, but the chat message wasn't added + // sort out broken chat entries + // it looks like in happened in the past that the chat head was + // incremented, but the chat message wasn't added var cleanedEntries = []; - for(var i=0;i delete the entry of this pad in the group - function(callback) - { - if(padID.indexOf("$") === -1) - { + // is it a group pad? -> delete the entry of this pad in the group + function(callback) { + if (padID.indexOf("$") === -1) { // it isn't a group pad, nothing to do here callback(); return; } // it is a group pad - var groupID = padID.substring(0,padID.indexOf("$")); + var groupID = padID.substring(0, padID.indexOf("$")); - db.get("group:" + groupID, function (err, group) - { - if(ERR(err, callback)) return; + db.get("group:" + groupID, function (err, group) { + if (ERR(err, callback)) return; - //remove the pad entry + // remove the pad entry delete group.pads[padID]; - //set the new value + // set the new value db.set("group:" + groupID, group); callback(); }); }, - //remove the readonly entries - function(callback) - { - readOnlyManager.getReadOnlyId(padID, function(err, readonlyID) - { - if(ERR(err, callback)) return; + + // remove the readonly entries + function(callback) { + readOnlyManager.getReadOnlyId(padID, function(err, readonlyID) { + if (ERR(err, callback)) return; db.remove("pad2readonly:" + padID); db.remove("readonly2pad:" + readonlyID); @@ -656,37 +629,34 @@ Pad.prototype.remove = function remove(callback) { callback(); }); }, - //delete all chat messages - function(callback) - { + + // delete all chat messages + function(callback) { var chatHead = _this.chatHead; - for(var i=0;i<=chatHead;i++) - { - db.remove("pad:"+padID+":chat:"+i); + for (var i = 0; i <= chatHead; i++) { + db.remove("pad:" + padID + ":chat:" + i); } callback(); }, - //delete all revisions - function(callback) - { + + // delete all revisions + function(callback) { var revHead = _this.head; - for(var i=0;i<=revHead;i++) - { - db.remove("pad:"+padID+":revs:"+i); + for (var i = 0; i <= revHead; i++) { + db.remove("pad:" + padID + ":revs:" + i); } callback(); }, - //remove pad from all authors who contributed - function(callback) - { + + // remove pad from all authors who contributed + function(callback) { var authorIDs = _this.getAllAuthors(); - authorIDs.forEach(function (authorID) - { + authorIDs.forEach(function (authorID) { authorManager.removePad(authorID, padID); }); @@ -694,20 +664,21 @@ Pad.prototype.remove = function remove(callback) { } ], callback); }, - //delete the pad entry and delete pad from padManager - function(callback) - { + + // delete the pad entry and delete pad from padManager + function(callback) { padManager.removePad(padID); - hooks.callAll("padRemove", {'padID':padID}); + hooks.callAll("padRemove", { 'padID': padID }); callback(); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; callback(); }); }; - //set in db + +// set in db Pad.prototype.setPublicStatus = function setPublicStatus(publicStatus) { this.publicStatus = publicStatus; this.saveToDatabase(); @@ -727,14 +698,14 @@ Pad.prototype.isPasswordProtected = function isPasswordProtected() { }; Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, label) { - //if this revision is already saved, return silently - for(var i in this.savedRevisions){ - if(this.savedRevisions[i] && this.savedRevisions[i].revNum === revNum){ + // if this revision is already saved, return silently + for (var i in this.savedRevisions) { + if (this.savedRevisions[i] && this.savedRevisions[i].revNum === revNum) { return; } } - //build the saved revision object + // build the saved revision object var savedRevision = {}; savedRevision.revNum = revNum; savedRevision.savedById = savedById; @@ -742,7 +713,7 @@ Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, la savedRevision.timestamp = Date.now(); savedRevision.id = randomString(10); - //save this new saved revision + // save this new saved revision this.savedRevisions.push(savedRevision); this.saveToDatabase(); }; @@ -753,19 +724,17 @@ Pad.prototype.getSavedRevisions = function getSavedRevisions() { /* Crypto helper methods */ -function hash(password, salt) -{ +function hash(password, salt) { var shasum = crypto.createHash('sha512'); shasum.update(password + salt); + return shasum.digest("hex") + "$" + salt; } -function generateSalt() -{ +function generateSalt() { return randomString(86); } -function compare(hashStr, password) -{ +function compare(hashStr, password) { return hash(password, hashStr.split("$")[1]) === hashStr; } diff --git a/src/node/db/PadManager.js b/src/node/db/PadManager.js index 035ef3e5e..b198de81c 100644 --- a/src/node/db/PadManager.js +++ b/src/node/db/PadManager.js @@ -23,7 +23,7 @@ var customError = require("../utils/customError"); var Pad = require("../db/Pad").Pad; var db = require("./DB").db; -/** +/** * A cache of all loaded Pads. * * Provides "get" and "set" functions, @@ -35,12 +35,11 @@ var db = require("./DB").db; * that's defined somewhere more sensible. */ var globalPads = { - get: function (name) { return this[':'+name]; }, - set: function (name, value) - { + get: function(name) { return this[':'+name]; }, + set: function(name, value) { this[':'+name] = value; }, - remove: function (name) { + remove: function(name) { delete this[':'+name]; } }; @@ -54,56 +53,63 @@ var padList = { list: [], sorted : false, initiated: false, - init: function(cb) - { - db.findKeys("pad:*", "*:*:*", function(err, dbData) - { - if(ERR(err, cb)) return; - if(dbData != null){ + init: function(cb) { + db.findKeys("pad:*", "*:*:*", function(err, dbData) { + if (ERR(err, cb)) return; + + if (dbData != null) { padList.initiated = true - dbData.forEach(function(val){ + dbData.forEach(function(val) { padList.addPad(val.replace(/pad:/,""),false); }); - cb && cb() + + cb && cb(); } }); + return this; }, load: function(cb) { - if(this.initiated) cb && cb() - else this.init(cb) + if (this.initiated) { + cb && cb(); + } else { + this.init(cb); + } }, /** * Returns all pads in alphabetical order as array. */ - getPads: function(cb){ + getPads: function(cb) { this.load(function() { - if(!padList.sorted){ + if (!padList.sorted) { padList.list = padList.list.sort(); padList.sorted = true; } + cb && cb(padList.list); }) }, - addPad: function(name) - { - if(!this.initiated) return; - if(this.list.indexOf(name) == -1){ + addPad: function(name) { + if (!this.initiated) return; + + if (this.list.indexOf(name) == -1) { this.list.push(name); - this.sorted=false; + this.sorted = false; } }, - removePad: function(name) - { - if(!this.initiated) return; + removePad: function(name) { + if (!this.initiated) return; + var index = this.list.indexOf(name); - if(index>-1){ - this.list.splice(index,1); - this.sorted=false; + + if (index > -1) { + this.list.splice(index, 1); + this.sorted = false; } } }; -//initialises the allknowing data structure + +// initialises the all-knowing data structure /** * An array of padId transformations. These represent changes in pad name policy over @@ -117,58 +123,56 @@ var padIdTransforms = [ /** * Returns a Pad Object with the callback * @param id A String with the id of the pad - * @param {Function} callback + * @param {Function} callback */ exports.getPad = function(id, text, callback) -{ - //check if this is a valid padId - if(!exports.isValidPadId(id)) - { - callback(new customError(id + " is not a valid padId","apierror")); +{ + // check if this is a valid padId + if (!exports.isValidPadId(id)) { + callback(new customError(id + " is not a valid padId", "apierror")); + return; } - - //make text an optional parameter - if(typeof text == "function") - { + + // make text an optional parameter + if (typeof text == "function") { callback = text; text = null; } - - //check if this is a valid text - if(text != null) - { - //check if text is a string - if(typeof text != "string") - { - callback(new customError("text is not a string","apierror")); + + // check if this is a valid text + if (text != null) { + // check if text is a string + if (typeof text != "string") { + callback(new customError("text is not a string", "apierror")); + return; } - - //check if text is less than 100k chars - if(text.length > 100000) - { - callback(new customError("text must be less than 100k chars","apierror")); + + // check if text is less than 100k chars + if (text.length > 100000) { + callback(new customError("text must be less than 100k chars", "apierror")); + return; } } - + var pad = globalPads.get(id); - - //return pad if its already loaded - if(pad != null) - { + + // return pad if it's already loaded + if (pad != null) { callback(null, pad); + return; } - //try to load pad + // try to load pad pad = new Pad(id); - //initalize the pad - pad.init(text, function(err) - { - if(ERR(err, callback)) return; + // initalize the pad + pad.init(text, function(err) { + if (ERR(err, callback)) return; + globalPads.set(id, pad); padList.addPad(id); callback(null, pad); @@ -182,49 +186,48 @@ exports.listAllPads = function(cb) }); } -//checks if a pad exists +// checks if a pad exists exports.doesPadExists = function(padId, callback) { - db.get("pad:"+padId, function(err, value) - { - if(ERR(err, callback)) return; - if(value != null && value.atext){ + db.get("pad:" + padId, function(err, value) { + if (ERR(err, callback)) return; + + if (value != null && value.atext) { callback(null, true); - } - else - { - callback(null, false); + } else { + callback(null, false); } }); } -//returns a sanitized padId, respecting legacy pad id formats +// returns a sanitized padId, respecting legacy pad id formats exports.sanitizePadId = function(padId, callback) { var transform_index = arguments[2] || 0; - //we're out of possible transformations, so just return it - if(transform_index >= padIdTransforms.length) - { + + // we're out of possible transformations, so just return it + if (transform_index >= padIdTransforms.length) { callback(padId); + return; } - //check if padId exists - exports.doesPadExists(padId, function(junk, exists) - { - if(exists) - { + // check if padId exists + exports.doesPadExists(padId, function(junk, exists) { + if (exists) { callback(padId); + return; } - //get the next transformation *that's different* + // get the next transformation *that's different* var transformedPadId = padId; - while(transformedPadId == padId && transform_index < padIdTransforms.length) - { + + while(transformedPadId == padId && transform_index < padIdTransforms.length) { transformedPadId = padId.replace(padIdTransforms[transform_index][0], padIdTransforms[transform_index][1]); transform_index += 1; } - //check the next transform + + // check the next transform exports.sanitizePadId(transformedPadId, callback, transform_index); }); } @@ -237,13 +240,13 @@ exports.isValidPadId = function(padId) /** * Removes the pad from database and unloads it. */ -exports.removePad = function(padId){ - db.remove("pad:"+padId); +exports.removePad = function(padId) { + db.remove("pad:" + padId); exports.unloadPad(padId); padList.removePad(padId); } -//removes a pad from the cache +// removes a pad from the cache exports.unloadPad = function(padId) { globalPads.remove(padId); diff --git a/src/node/db/ReadOnlyManager.js b/src/node/db/ReadOnlyManager.js index f49f71e23..ac7bee045 100644 --- a/src/node/db/ReadOnlyManager.js +++ b/src/node/db/ReadOnlyManager.js @@ -29,43 +29,40 @@ var randomString = require("../utils/randomstring"); * @param {String} padId the id of the pad */ exports.getReadOnlyId = function (padId, callback) -{ +{ var readOnlyId; - + async.waterfall([ - //check if there is a pad2readonly entry - function(callback) - { + // check if there is a pad2readonly entry + function(callback) { db.get("pad2readonly:" + padId, callback); }, - function(dbReadOnlyId, callback) - { - //there is no readOnly Entry in the database, let's create one - if(dbReadOnlyId == null) - { + + function(dbReadOnlyId, callback) { + if (dbReadOnlyId == null) { + // there is no readOnly Entry in the database, let's create one readOnlyId = "r." + randomString(16); - + db.set("pad2readonly:" + padId, readOnlyId); db.set("readonly2pad:" + readOnlyId, padId); - } - //there is a readOnly Entry in the database, let's take this one - else - { + } else { + // there is a readOnly Entry in the database, let's take this one readOnlyId = dbReadOnlyId; } - + callback(); } - ], function(err) - { - if(ERR(err, callback)) return; - //return the results + ], + function(err) { + if (ERR(err, callback)) return; + + // return the results callback(null, readOnlyId); }) } /** - * returns a the padId for a read only id + * returns the padId for a read only id * @param {String} readOnlyId read only id */ exports.getPadId = function(readOnlyId, callback) @@ -74,20 +71,21 @@ exports.getPadId = function(readOnlyId, callback) } /** - * returns a the padId and readonlyPadId in an object for any id + * returns the padId and readonlyPadId in an object for any id * @param {String} padIdOrReadonlyPadId read only id or real pad id */ exports.getIds = function(id, callback) { - if (id.indexOf("r.") == 0) + if (id.indexOf("r.") == 0) { exports.getPadId(id, function (err, value) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; + callback(null, { readOnlyPadId: id, padId: value, // Might be null, if this is an unknown read-only id readonly: true }); }); - else + } else { exports.getReadOnlyId(id, function (err, value) { callback(null, { readOnlyPadId: value, @@ -95,4 +93,5 @@ exports.getIds = function(id, callback) { readonly: false }); }); + } } diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index 2c46ac508..117f2794f 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -18,7 +18,6 @@ * limitations under the License. */ - var ERR = require("async-stacktrace"); var async = require("async"); var authorManager = require("./AuthorManager"); @@ -34,59 +33,57 @@ var authLogger = log4js.getLogger("auth"); * @param padID the pad the user wants to access * @param sessionCookie the session the user has (set via api) * @param token the token of the author (randomly generated at client side, used for public pads) - * @param password the password the user has given to access this pad, can be null + * @param password the password the user has given to access this pad, can be null * @param callback will be called with (err, {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx}) - */ -exports.checkAccess = function (padID, sessionCookie, token, password, callback) -{ + */ +exports.checkAccess = function(padID, sessionCookie, token, password, callback) +{ var statusObject; - - if(!padID) { + + if (!padID) { callback(null, {accessStatus: "deny"}); return; } // allow plugins to deny access var deniedByHook = hooks.callAll("onAccessCheck", {'padID': padID, 'password': password, 'token': token, 'sessionCookie': sessionCookie}).indexOf(false) > -1; - if(deniedByHook) - { + if (deniedByHook) { callback(null, {accessStatus: "deny"}); return; } - // a valid session is required (api-only mode) - if(settings.requireSession) - { - // without sessionCookie, access is denied - if(!sessionCookie) - { + if (settings.requireSession) { + // a valid session is required (api-only mode) + if (!sessionCookie) { + // without sessionCookie, access is denied callback(null, {accessStatus: "deny"}); + return; } - } - // a session is not required, so we'll check if it's a public pad - else - { - // it's not a group pad, means we can grant access - if(padID.indexOf("$") == -1) - { - //get author for this token - authorManager.getAuthor4Token(token, function(err, author) - { - if(ERR(err, callback)) return; - + } else { + // a session is not required, so we'll check if it's a public pad + if (padID.indexOf("$") == -1) { + // it's not a group pad, means we can grant access + + // get author for this token + authorManager.getAuthor4Token(token, function(err, author) { + if (ERR(err, callback)) return; + // assume user has access - statusObject = {accessStatus: "grant", authorID: author}; - // user can't create pads - if(settings.editOnly) - { + statusObject = { accessStatus: "grant", authorID: author }; + + if (settings.editOnly) { + // user can't create pads + // check if pad exists - padManager.doesPadExists(padID, function(err, exists) - { - if(ERR(err, callback)) return; - - // pad doesn't exist - user can't have access - if(!exists) statusObject.accessStatus = "deny"; + padManager.doesPadExists(padID, function(err, exists) { + if (ERR(err, callback)) return; + + if (!exists) { + // pad doesn't exist - user can't have access + statusObject.accessStatus = "deny"; + } + // grant or deny access, with author of token callback(null, statusObject); }); @@ -98,12 +95,12 @@ exports.checkAccess = function (padID, sessionCookie, token, password, callback) // grant access, with author of token callback(null, statusObject); }); - - //don't continue + + // don't continue return; } } - + var groupID = padID.split("$")[0]; var padExists = false; var validSession = false; @@ -114,216 +111,193 @@ exports.checkAccess = function (padID, sessionCookie, token, password, callback) var passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong async.series([ - //get basic informations from the database - function(callback) - { + // get basic informations from the database + function(callback) { async.parallel([ - //does pad exists - function(callback) - { - padManager.doesPadExists(padID, function(err, exists) - { - if(ERR(err, callback)) return; + // does pad exist + function(callback) { + padManager.doesPadExists(padID, function(err, exists) { + if (ERR(err, callback)) return; + padExists = exists; callback(); }); }, - //get information about all sessions contained in this cookie - function(callback) - { - if (!sessionCookie) - { + + // get information about all sessions contained in this cookie + function(callback) { + if (!sessionCookie) { callback(); return; } - + var sessionIDs = sessionCookie.split(','); - async.forEach(sessionIDs, function(sessionID, callback) - { - sessionManager.getSessionInfo(sessionID, function(err, sessionInfo) - { - //skip session if it doesn't exist - if(err && err.message == "sessionID does not exist") - { + + async.forEach(sessionIDs, function(sessionID, callback) { + sessionManager.getSessionInfo(sessionID, function(err, sessionInfo) { + // skip session if it doesn't exist + if (err && err.message == "sessionID does not exist") { authLogger.debug("Auth failed: unknown session"); callback(); + return; } - - if(ERR(err, callback)) return; - + + if (ERR(err, callback)) return; + var now = Math.floor(Date.now()/1000); - - //is it for this group? - if(sessionInfo.groupID != groupID) - { + + // is it for this group? + if (sessionInfo.groupID != groupID) { authLogger.debug("Auth failed: wrong group"); callback(); + return; } - - //is validUntil still ok? - if(sessionInfo.validUntil <= now) - { + + // is validUntil still ok? + if (sessionInfo.validUntil <= now) { authLogger.debug("Auth failed: validUntil"); callback(); + return; } - + // There is a valid session validSession = true; sessionAuthor = sessionInfo.authorID; - + callback(); }); }, callback); }, - //get author for token - function(callback) - { - //get author for this token - authorManager.getAuthor4Token(token, function(err, author) - { - if(ERR(err, callback)) return; + + // get author for token + function(callback) { + // get author for this token + authorManager.getAuthor4Token(token, function(err, author) { + if (ERR(err, callback)) return; + tokenAuthor = author; callback(); }); } ], callback); }, - //get more informations of this pad, if avaiable - function(callback) - { - //skip this if the pad doesn't exists - if(padExists == false) - { + + // get more informations of this pad, if avaiable + function(callback) { + // skip this if the pad doesn't exist + if (padExists == false) { callback(); + return; } - - padManager.getPad(padID, function(err, pad) - { - if(ERR(err, callback)) return; - - //is it a public pad? + + padManager.getPad(padID, function(err, pad) { + if (ERR(err, callback)) return; + + // is it a public pad? isPublic = pad.getPublicStatus(); - - //is it password protected? + + // is it password protected? isPasswordProtected = pad.isPasswordProtected(); - - //is password correct? - if(isPasswordProtected && password && pad.isCorrectPassword(password)) - { + + // is password correct? + if (isPasswordProtected && password && pad.isCorrectPassword(password)) { passwordStatus = "correct"; } - + callback(); }); }, - function(callback) - { - //- a valid session for this group is avaible AND pad exists - if(validSession && padExists) - { - //- the pad is not password protected - if(!isPasswordProtected) - { - //--> grant access - statusObject = {accessStatus: "grant", authorID: sessionAuthor}; - } - //- the setting to bypass password validation is set - else if(settings.sessionNoPassword) - { - //--> grant access - statusObject = {accessStatus: "grant", authorID: sessionAuthor}; - } - //- the pad is password protected and password is correct - else if(isPasswordProtected && passwordStatus == "correct") - { - //--> grant access - statusObject = {accessStatus: "grant", authorID: sessionAuthor}; - } - //- the pad is password protected but wrong password given - else if(isPasswordProtected && passwordStatus == "wrong") - { - //--> deny access, ask for new password and tell them that the password is wrong - statusObject = {accessStatus: "wrongPassword"}; - } - //- the pad is password protected but no password given - else if(isPasswordProtected && passwordStatus == "notGiven") - { - //--> ask for password - statusObject = {accessStatus: "needPassword"}; - } - else - { + + function(callback) { + if (validSession && padExists) { + // - a valid session for this group is avaible AND pad exists + if (!isPasswordProtected) { + // - the pad is not password protected + + // --> grant access + statusObject = { accessStatus: "grant", authorID: sessionAuthor }; + } else if (settings.sessionNoPassword) { + // - the setting to bypass password validation is set + + // --> grant access + statusObject = { accessStatus: "grant", authorID: sessionAuthor }; + } else if (isPasswordProtected && passwordStatus == "correct") { + // - the pad is password protected and password is correct + + // --> grant access + statusObject = { accessStatus: "grant", authorID: sessionAuthor }; + } else if (isPasswordProtected && passwordStatus == "wrong") { + // - the pad is password protected but wrong password given + + // --> deny access, ask for new password and tell them that the password is wrong + statusObject = { accessStatus: "wrongPassword" }; + } else if (isPasswordProtected && passwordStatus == "notGiven") { + // - the pad is password protected but no password given + + // --> ask for password + statusObject = { accessStatus: "needPassword" }; + } else { throw new Error("Ops, something wrong happend"); } - } - //- a valid session for this group avaible but pad doesn't exists - else if(validSession && !padExists) - { - //--> grant access + } else if (validSession && !padExists) { + // - a valid session for this group avaible but pad doesn't exist + + // --> grant access statusObject = {accessStatus: "grant", authorID: sessionAuthor}; - //--> deny access if user isn't allowed to create the pad - if(settings.editOnly) - { + + if (settings.editOnly) { + // --> deny access if user isn't allowed to create the pad authLogger.debug("Auth failed: valid session & pad does not exist"); statusObject.accessStatus = "deny"; } - } - // there is no valid session avaiable AND pad exists - else if(!validSession && padExists) - { - //-- its public and not password protected - if(isPublic && !isPasswordProtected) - { - //--> grant access, with author of token + } else if (!validSession && padExists) { + // there is no valid session avaiable AND pad exists + + // -- it's public and not password protected + if (isPublic && !isPasswordProtected) { + // --> grant access, with author of token statusObject = {accessStatus: "grant", authorID: tokenAuthor}; - } - //- its public and password protected and password is correct - else if(isPublic && isPasswordProtected && passwordStatus == "correct") - { - //--> grant access, with author of token + } else if (isPublic && isPasswordProtected && passwordStatus == "correct") { + // - it's public and password protected and password is correct + + // --> grant access, with author of token statusObject = {accessStatus: "grant", authorID: tokenAuthor}; - } - //- its public and the pad is password protected but wrong password given - else if(isPublic && isPasswordProtected && passwordStatus == "wrong") - { - //--> deny access, ask for new password and tell them that the password is wrong + } else if (isPublic && isPasswordProtected && passwordStatus == "wrong") { + // - it's public and the pad is password protected but wrong password given + + // --> deny access, ask for new password and tell them that the password is wrong statusObject = {accessStatus: "wrongPassword"}; - } - //- its public and the pad is password protected but no password given - else if(isPublic && isPasswordProtected && passwordStatus == "notGiven") - { - //--> ask for password + } else if (isPublic && isPasswordProtected && passwordStatus == "notGiven") { + // - it's public and the pad is password protected but no password given + + // --> ask for password statusObject = {accessStatus: "needPassword"}; - } - //- its not public - else if(!isPublic) - { + } else if (!isPublic) { + // - it's not public + authLogger.debug("Auth failed: invalid session & pad is not public"); - //--> deny access + // --> deny access statusObject = {accessStatus: "deny"}; - } - else - { + } else { throw new Error("Ops, something wrong happend"); } - } - // there is no valid session avaiable AND pad doesn't exists - else - { - authLogger.debug("Auth failed: invalid session & pad does not exist"); - //--> deny access - statusObject = {accessStatus: "deny"}; + } else { + // there is no valid session avaiable AND pad doesn't exist + authLogger.debug("Auth failed: invalid session & pad does not exist"); + // --> deny access + statusObject = {accessStatus: "deny"}; } - + callback(); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; + callback(null, statusObject); }); }; diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index 99803ee71..6b580c033 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -17,7 +17,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); @@ -26,7 +25,7 @@ var db = require("./DB").db; var async = require("async"); var groupMangager = require("./GroupManager"); var authorMangager = require("./AuthorManager"); - + exports.doesSessionExist = function(sessionID, callback) { //check if the database entry of this session exists @@ -36,7 +35,7 @@ exports.doesSessionExist = function(sessionID, callback) callback(null, session != null); }); } - + /** * Creates a new session between an author and a group */ @@ -45,159 +44,148 @@ exports.createSession = function(groupID, authorID, validUntil, callback) var sessionID; async.series([ - //check if group exists + // check if the group exists function(callback) { groupMangager.doesGroupExist(groupID, function(err, exists) { if(ERR(err, callback)) return; - - //group does not exist - if(exists == false) - { - callback(new customError("groupID does not exist","apierror")); - } - //everything is fine, continue - else - { + + // group does not exist + if (exists == false) { + callback(new customError("groupID does not exist", "apierror")); + } else { + // everything is fine, continue callback(); } }); }, - //check if author exists + + // check if the author exists function(callback) { authorMangager.doesAuthorExists(authorID, function(err, exists) { if(ERR(err, callback)) return; - - //author does not exist - if(exists == false) - { - callback(new customError("authorID does not exist","apierror")); - } - //everything is fine, continue - else - { + + if (exists == false) { + // author does not exist + callback(new customError("authorID does not exist", "apierror")); + } else { + // everything is fine, continue callback(); } }); }, - //check validUntil and create the session db entry + + // check validUntil and create the session db entry function(callback) { - //check if rev is a number - if(typeof validUntil != "number") + // check if rev is a number + if (typeof validUntil != "number") { - //try to parse the number - if(isNaN(parseInt(validUntil))) - { - callback(new customError("validUntil is not a number","apierror")); + // try to parse the number + if (isNaN(parseInt(validUntil))) { + callback(new customError("validUntil is not a number", "apierror")); return; } validUntil = parseInt(validUntil); } - - //ensure this is not a negativ number - if(validUntil < 0) - { - callback(new customError("validUntil is a negativ number","apierror")); + + // ensure this is not a negative number + if (validUntil < 0) { + callback(new customError("validUntil is a negativ number", "apierror")); return; } - - //ensure this is not a float value - if(!is_int(validUntil)) - { - callback(new customError("validUntil is a float value","apierror")); + + // ensure this is not a float value + if (!is_int(validUntil)) { + callback(new customError("validUntil is a float value", "apierror")); return; } - - //check if validUntil is in the future - if(Math.floor(Date.now()/1000) > validUntil) - { - callback(new customError("validUntil is in the past","apierror")); + + // check if validUntil is in the future + if (Math.floor(Date.now()/1000) > validUntil) { + callback(new customError("validUntil is in the past", "apierror")); return; } - - //generate sessionID + + // generate sessionID sessionID = "s." + randomString(16); - - //set the session into the database + + // set the session into the database db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil}); - + callback(); }, - //set the group2sessions entry + + // set the group2sessions entry function(callback) { - //get the entry + // get the entry db.get("group2sessions:" + groupID, function(err, group2sessions) { if(ERR(err, callback)) return; - - //the entry doesn't exist so far, let's create it - if(group2sessions == null || group2sessions.sessionIDs == null) - { + + if (group2sessions == null || group2sessions.sessionIDs == null) { + // the entry doesn't exist so far, let's create it group2sessions = {sessionIDs : {}}; } - - //add the entry for this session + + // add the entry for this session group2sessions.sessionIDs[sessionID] = 1; - - //save the new element back + + // save the new element back db.set("group2sessions:" + groupID, group2sessions); - + callback(); }); }, - //set the author2sessions entry + + // set the author2sessions entry function(callback) { - //get the entry + // get the entry db.get("author2sessions:" + authorID, function(err, author2sessions) { if(ERR(err, callback)) return; - - //the entry doesn't exist so far, let's create it - if(author2sessions == null || author2sessions.sessionIDs == null) - { + + if (author2sessions == null || author2sessions.sessionIDs == null) { + // the entry doesn't exist so far, let's create it author2sessions = {sessionIDs : {}}; } - - //add the entry for this session + + // add the entry for this session author2sessions.sessionIDs[sessionID] = 1; - + //save the new element back db.set("author2sessions:" + authorID, author2sessions); - + callback(); }); } ], function(err) { if(ERR(err, callback)) return; - - //return error and sessionID + + // return error and sessionID callback(null, {sessionID: sessionID}); }) } exports.getSessionInfo = function(sessionID, callback) { - //check if the database entry of this session exists + // check if the database entry of this session exists db.get("session:" + sessionID, function (err, session) { if(ERR(err, callback)) return; - - //session does not exists - if(session == null) - { - callback(new customError("sessionID does not exist","apierror")) - } - //everything is fine, return the sessioninfos - else - { + + if (session == null) { + // session does not exist + callback(new customError("sessionID does not exist", "apierror")) + } else { + // everything is fine, return the sessioninfos callback(null, session); } }); @@ -214,27 +202,25 @@ exports.deleteSession = function(sessionID, callback) async.series([ function(callback) { - //get the session entry + // get the session entry db.get("session:" + sessionID, function (err, session) { if(ERR(err, callback)) return; - - //session does not exists - if(session == null) - { - callback(new customError("sessionID does not exist","apierror")) - } - //everything is fine, return the sessioninfos - else - { + + if (session == null) { + // session does not exist + callback(new customError("sessionID does not exist", "apierror")) + } else { + // everything is fine, use the sessioninfos authorID = session.authorID; groupID = session.groupID; - + callback(); } }); }, - //get the group2sessions entry + + // get the group2sessions entry function(callback) { db.get("group2sessions:" + groupID, function (err, _group2sessions) @@ -244,7 +230,8 @@ exports.deleteSession = function(sessionID, callback) callback(); }); }, - //get the author2sessions entry + + // get the author2sessions entry function(callback) { db.get("author2sessions:" + authorID, function (err, _author2sessions) @@ -254,24 +241,25 @@ exports.deleteSession = function(sessionID, callback) callback(); }); }, - //remove the values from the database + + // remove the values from the database function(callback) { //remove the session db.remove("session:" + sessionID); - - //remove session from group2sessions + + // remove session from group2sessions if(group2sessions != null) { // Maybe the group was already deleted delete group2sessions.sessionIDs[sessionID]; db.set("group2sessions:" + groupID, group2sessions); } - //remove session from author2sessions + // remove session from author2sessions if(author2sessions != null) { // Maybe the author was already deleted delete author2sessions.sessionIDs[sessionID]; db.set("author2sessions:" + authorID, author2sessions); } - + callback(); } ], function(err) @@ -286,40 +274,34 @@ exports.listSessionsOfGroup = function(groupID, callback) groupMangager.doesGroupExist(groupID, function(err, exists) { if(ERR(err, callback)) return; - - //group does not exist - if(exists == false) - { - callback(new customError("groupID does not exist","apierror")); - } - //everything is fine, continue - else - { + + if (exists == false) { + // group does not exist + callback(new customError("groupID does not exist", "apierror")); + } else { + // everything is fine, continue listSessionsWithDBKey("group2sessions:" + groupID, callback); } }); } exports.listSessionsOfAuthor = function(authorID, callback) -{ +{ authorMangager.doesAuthorExists(authorID, function(err, exists) { if(ERR(err, callback)) return; - - //group does not exist - if(exists == false) - { - callback(new customError("authorID does not exist","apierror")); - } - //everything is fine, continue - else - { + + if (exists == false) { + // group does not exist + callback(new customError("authorID does not exist", "apierror")); + } else { + // everything is fine, continue listSessionsWithDBKey("author2sessions:" + authorID, callback); } }); } -//this function is basicly the code listSessionsOfAuthor and listSessionsOfGroup has in common +// this function is basically the code listSessionsOfAuthor and listSessionsOfGroup has in common function listSessionsWithDBKey (dbkey, callback) { var sessions; @@ -327,7 +309,7 @@ function listSessionsWithDBKey (dbkey, callback) async.series([ function(callback) { - //get the group2sessions entry + // get the group2sessions entry db.get(dbkey, function(err, sessionObject) { if(ERR(err, callback)) return; @@ -335,26 +317,24 @@ function listSessionsWithDBKey (dbkey, callback) callback(); }); }, + function(callback) - { - //collect all sessionIDs in an arrary + { + // collect all sessionIDs in an arrary var sessionIDs = []; for (var i in sessions) { sessionIDs.push(i); } - - //foreach trough the sessions and get the sessioninfos + + // iterate through the sessions and get the sessioninfos async.forEach(sessionIDs, function(sessionID, callback) { exports.getSessionInfo(sessionID, function(err, sessionInfo) { - if (err == "apierror: sessionID does not exist") - { + if (err == "apierror: sessionID does not exist") { console.warn(`Found bad session ${sessionID} in ${dbkey}`); - } - else if(ERR(err, callback)) - { + } else if(ERR(err, callback)) { return; } @@ -370,8 +350,8 @@ function listSessionsWithDBKey (dbkey, callback) }); } -//checks if a number is an int +// checks if a number is an int function is_int(value) -{ - return (parseFloat(value) == parseInt(value)) && !isNaN(value) +{ + return (parseFloat(value) == parseInt(value)) && !isNaN(value); } diff --git a/src/node/db/SessionStore.js b/src/node/db/SessionStore.js index 974046908..80d35f290 100644 --- a/src/node/db/SessionStore.js +++ b/src/node/db/SessionStore.js @@ -1,4 +1,4 @@ - /* +/* * Stores session data in the database * Source; https://github.com/edy-b/SciFlowWriter/blob/develop/available_plugins/ep_sciflowwriter/db/DirtyStore.js * This is not used for authors that are created via the API at current @@ -13,11 +13,12 @@ var SessionStore = module.exports = function SessionStore() {}; SessionStore.prototype.__proto__ = Store.prototype; -SessionStore.prototype.get = function(sid, fn){ +SessionStore.prototype.get = function(sid, fn) { messageLogger.debug('GET ' + sid); + var self = this; - db.get("sessionstorage:" + sid, function (err, sess) - { + + db.get("sessionstorage:" + sid, function(err, sess) { if (sess) { sess.cookie.expires = 'string' == typeof sess.cookie.expires ? new Date(sess.cookie.expires) : sess.cookie.expires; if (!sess.cookie.expires || new Date() < sess.cookie.expires) { @@ -31,26 +32,30 @@ SessionStore.prototype.get = function(sid, fn){ }); }; -SessionStore.prototype.set = function(sid, sess, fn){ +SessionStore.prototype.set = function(sid, sess, fn) { messageLogger.debug('SET ' + sid); + db.set("sessionstorage:" + sid, sess); - process.nextTick(function(){ - if(fn) fn(); + process.nextTick(function() { + if (fn) fn(); }); }; -SessionStore.prototype.destroy = function(sid, fn){ +SessionStore.prototype.destroy = function(sid, fn) { messageLogger.debug('DESTROY ' + sid); + db.remove("sessionstorage:" + sid); - process.nextTick(function(){ - if(fn) fn(); + process.nextTick(function() { + if (fn) fn(); }); }; -SessionStore.prototype.all = function(fn){ +SessionStore.prototype.all = function(fn) { messageLogger.debug('ALL'); + var sessions = []; - db.forEach(function(key, value){ + + db.forEach(function(key, value) { if (key.substr(0,15) === "sessionstorage:") { sessions.push(value); } @@ -58,20 +63,23 @@ SessionStore.prototype.all = function(fn){ fn(null, sessions); }; -SessionStore.prototype.clear = function(fn){ +SessionStore.prototype.clear = function(fn) { messageLogger.debug('CLEAR'); - db.forEach(function(key, value){ + + db.forEach(function(key, value) { if (key.substr(0,15) === "sessionstorage:") { db.db.remove("session:" + key); } }); - if(fn) fn(); + if (fn) fn(); }; -SessionStore.prototype.length = function(fn){ +SessionStore.prototype.length = function(fn) { messageLogger.debug('LENGTH'); + var i = 0; - db.forEach(function(key, value){ + + db.forEach(function(key, value) { if (key.substr(0,15) === "sessionstorage:") { i++; } diff --git a/src/node/handler/APIHandler.js b/src/node/handler/APIHandler.js index 6ec5907e2..3ed7aa4e1 100644 --- a/src/node/handler/APIHandler.js +++ b/src/node/handler/APIHandler.js @@ -32,19 +32,17 @@ var apiHandlerLogger = log4js.getLogger('APIHandler'); //ensure we have an apikey var apikey = null; var apikeyFilename = absolutePaths.makeAbsolute(argv.apikey || "./APIKEY.txt"); -try -{ + +try { apikey = fs.readFileSync(apikeyFilename,"utf8"); apiHandlerLogger.info(`Api key file read from: "${apikeyFilename}"`); -} -catch(e) -{ +} catch(e) { apiHandlerLogger.info(`Api key file "${apikeyFilename}" not found. Creating with random contents.`); apikey = randomString(32); fs.writeFileSync(apikeyFilename,apikey,"utf8"); } -//a list of all functions +// a list of all functions var version = {}; version["1"] = Object.assign({}, @@ -156,106 +154,92 @@ exports.handle = function(apiVersion, functionName, fields, req, res) { //check if this is a valid apiversion var isKnownApiVersion = false; - for(var knownApiVersion in version) - { - if(knownApiVersion == apiVersion) - { + + for (var knownApiVersion in version) { + if (knownApiVersion == apiVersion) { isKnownApiVersion = true; break; } } - //say goodbye if this is an unknown API version - if(!isKnownApiVersion) - { + // say goodbye if this is an unknown API version + if (!isKnownApiVersion) { res.statusCode = 404; res.send({code: 3, message: "no such api version", data: null}); return; } - //check if this is a valid function name + // check if this is a valid function name var isKnownFunctionname = false; - for(var knownFunctionname in version[apiVersion]) - { - if(knownFunctionname == functionName) - { + + for (var knownFunctionname in version[apiVersion]) { + if (knownFunctionname == functionName) { isKnownFunctionname = true; break; } } - //say goodbye if this is a unknown function - if(!isKnownFunctionname) - { + // say goodbye if this is an unknown function + if (!isKnownFunctionname) { res.send({code: 3, message: "no such function", data: null}); return; } - //check the api key! + // check the api key! fields["apikey"] = fields["apikey"] || fields["api_key"]; - if(fields["apikey"] != apikey.trim()) - { + if (fields["apikey"] != apikey.trim()) { res.statusCode = 401; res.send({code: 4, message: "no or wrong API Key", data: null}); return; } - //sanitize any pad id's before continuing - if(fields["padID"]) - { - padManager.sanitizePadId(fields["padID"], function(padId) - { + // sanitize any padIDs before continuing + if (fields["padID"]) { + padManager.sanitizePadId(fields["padID"], function(padId) { fields["padID"] = padId; callAPI(apiVersion, functionName, fields, req, res); }); - } - else if(fields["padName"]) - { - padManager.sanitizePadId(fields["padName"], function(padId) - { + } else if (fields["padName"]) { + padManager.sanitizePadId(fields["padName"], function(padId) { fields["padName"] = padId; callAPI(apiVersion, functionName, fields, req, res); }); - } - else - { + } else { callAPI(apiVersion, functionName, fields, req, res); } } -//calls the api function +// calls the api function function callAPI(apiVersion, functionName, fields, req, res) { - //put the function parameters in an array + // put the function parameters in an array var functionParams = version[apiVersion][functionName].map(function (field) { return fields[field] - }) + }); - //add a callback function to handle the response - functionParams.push(function(err, data) - { - // no error happend, everything is fine - if(err == null) - { - if(!data) + // add a callback function to handle the response + functionParams.push(function(err, data) { + if (err == null) { + // no error happened, everything is fine + + if (!data) { data = null; + } res.send({code: 0, message: "ok", data: data}); - } - // parameters were wrong and the api stopped execution, pass the error - else if(err.name == "apierror") - { + } else if (err.name == "apierror") { + // parameters were wrong and the api stopped execution, pass the error + res.send({code: 1, message: err.message, data: null}); - } - //an unknown error happend - else - { + } else { + // an unknown error happened + res.send({code: 2, message: "internal error", data: null}); ERR(err); } }); - //call the api function + // call the api function api[functionName].apply(this, functionParams); } diff --git a/src/node/handler/ExportHandler.js b/src/node/handler/ExportHandler.js index db3d2d40d..3bcabf233 100644 --- a/src/node/handler/ExportHandler.js +++ b/src/node/handler/ExportHandler.js @@ -32,13 +32,15 @@ var TidyHtml = require('../utils/TidyHtml'); var convertor = null; -//load abiword only if its enabled -if(settings.abiword != null) +// load abiword only if it is enabled +if (settings.abiword != null) { convertor = require("../utils/Abiword"); +} // Use LibreOffice if an executable has been defined in the settings -if(settings.soffice != null) +if (settings.soffice != null) { convertor = require("../utils/LibreOffice"); +} const tempDirectory = os.tmpdir(); @@ -53,62 +55,55 @@ exports.doExport = function(req, res, padId, type) hooks.aCallFirst("exportFileName", padId, function(err, hookFileName){ // if fileName is set then set it to the padId, note that fileName is returned as an array. - if(hookFileName.length) fileName = hookFileName; + if (hookFileName.length) { + fileName = hookFileName; + } - //tell the browser that this is a downloadable file + // tell the browser that this is a downloadable file res.attachment(fileName + "." + type); - //if this is a plain text export, we can do this directly + // if this is a plain text export, we can do this directly // We have to over engineer this because tabs are stored as attributes and not plain text - if(type == "etherpad"){ - exportEtherpad.getPadRaw(padId, function(err, pad){ - if(!err){ + if (type == "etherpad") { + exportEtherpad.getPadRaw(padId, function(err, pad) { + if (!err) { res.send(pad); // return; } }); - } - else if(type == "txt") - { - exporttxt.getPadTXTDocument(padId, req.params.rev, function(err, txt) - { - if(!err) { + } else if (type == "txt") { + exporttxt.getPadTXTDocument(padId, req.params.rev, function(err, txt) { + if (!err) { res.send(txt); } }); - } - else - { + } else { var html; var randNum; var srcFile, destFile; async.series([ - //render the html document - function(callback) - { - exporthtml.getPadHTMLDocument(padId, req.params.rev, function(err, _html) - { - if(ERR(err, callback)) return; + // render the html document + function(callback) { + exporthtml.getPadHTMLDocument(padId, req.params.rev, function(err, _html) { + if (ERR(err, callback)) return; html = _html; callback(); }); }, - //decide what to do with the html export - function(callback) - { - //if this is a html export, we can send this from here directly - if(type == "html") - { + + // decide what to do with the html export + function(callback) { + // if this is a html export, we can send this from here directly + if (type == "html") { // do any final changes the plugin might want to make - hooks.aCallFirst("exportHTMLSend", html, function(err, newHTML){ - if(newHTML.length) html = newHTML; + hooks.aCallFirst("exportHTMLSend", html, function(err, newHTML) { + if (newHTML.length) html = newHTML; res.send(html); callback("stop"); }); - } - else //write the html export to a file - { + } else { + // write the html export to a file randNum = Math.floor(Math.random()*0xFFFFFFFF); srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html"; fs.writeFile(srcFile, html, callback); @@ -116,64 +111,56 @@ exports.doExport = function(req, res, padId, type) }, // Tidy up the exported HTML - function(callback) - { - //ensure html can be collected by the garbage collector + function(callback) { + // ensure html can be collected by the garbage collector html = null; TidyHtml.tidy(srcFile, callback); }, - //send the convert job to the convertor (abiword, libreoffice, ..) - function(callback) - { + // send the convert job to the convertor (abiword, libreoffice, ..) + function(callback) { destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type; // Allow plugins to overwrite the convert in export process - hooks.aCallAll("exportConvert", {srcFile: srcFile, destFile: destFile, req: req, res: res}, function(err, result){ - if(!err && result.length > 0){ + hooks.aCallAll("exportConvert", { srcFile: srcFile, destFile: destFile, req: req, res: res }, function(err, result) { + if (!err && result.length > 0) { // console.log("export handled by plugin", destFile); handledByPlugin = true; callback(); - }else{ + } else { convertor.convertFile(srcFile, destFile, type, callback); } }); }, - //send the file - function(callback) - { + + // send the file + function(callback) { res.sendFile(destFile, null, callback); }, - //clean up temporary files - function(callback) - { + + // clean up temporary files + function(callback) { async.parallel([ - function(callback) - { + function(callback) { fs.unlink(srcFile, callback); }, - function(callback) - { - //100ms delay to accomidate for slow windows fs - if(os.type().indexOf("Windows") > -1) - { - setTimeout(function() - { + function(callback) { + // 100ms delay to accommodate for slow windows fs + if (os.type().indexOf("Windows") > -1) { + setTimeout(function() { fs.unlink(destFile, callback); }, 100); - } - else - { + } else { fs.unlink(destFile, callback); } } ], callback); } - ], function(err) - { - if(err && err != "stop") ERR(err); + ], + function(err) { + if (err && err != "stop") ERR(err); }) } } diff --git a/src/node/handler/ImportHandler.js b/src/node/handler/ImportHandler.js index 30b773972..dae71bf6d 100644 --- a/src/node/handler/ImportHandler.js +++ b/src/node/handler/ImportHandler.js @@ -37,29 +37,30 @@ var ERR = require("async-stacktrace") var convertor = null; var exportExtension = "htm"; -//load abiword only if its enabled and if soffice is disabled -if(settings.abiword != null && settings.soffice === null) +// load abiword only if it is enabled and if soffice is disabled +if (settings.abiword != null && settings.soffice === null) { convertor = require("../utils/Abiword"); +} -//load soffice only if its enabled -if(settings.soffice != null) { +// load soffice only if it is enabled +if (settings.soffice != null) { convertor = require("../utils/LibreOffice"); exportExtension = "html"; } const tmpDirectory = os.tmpdir(); - + /** * do a requested import - */ + */ exports.doImport = function(req, res, padId) { var apiLogger = log4js.getLogger("ImportHandler"); - //pipe to a file - //convert file to html via abiword or soffice - //set html in the pad - + // pipe to a file + // convert file to html via abiword or soffice + // set html in the pad + var srcFile, destFile , pad , text @@ -68,69 +69,74 @@ exports.doImport = function(req, res, padId) , useConvertor; var randNum = Math.floor(Math.random()*0xFFFFFFFF); - + // setting flag for whether to use convertor or not useConvertor = (convertor != null); async.series([ - //save the uploaded file to /tmp + // save the uploaded file to /tmp function(callback) { var form = new formidable.IncomingForm(); form.keepExtensions = true; form.uploadDir = tmpDirectory; - - form.parse(req, function(err, fields, files) { - //the upload failed, stop at this point - if(err || files.file === undefined) { - if(err) console.warn("Uploading Error: " + err.stack); + + form.parse(req, function(err, fields, files) { + if (err || files.file === undefined) { + // the upload failed, stop at this point + if (err) { + console.warn("Uploading Error: " + err.stack); + } callback("uploadFailed"); return; } - //everything ok, continue - //save the path of the uploaded file + // everything ok, continue + // save the path of the uploaded file srcFile = files.file.path; callback(); }); }, - - //ensure this is a file ending we know, else we change the file ending to .txt - //this allows us to accept source code files like .c or .java + + // ensure this is a file ending we know, else we change the file ending to .txt + // this allows us to accept source code files like .c or .java function(callback) { var fileEnding = path.extname(srcFile).toLowerCase() , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"] , fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1); - - //if the file ending is known, continue as normal - if(fileEndingKnown) { + + // if the file ending is known, continue as normal + if (fileEndingKnown) { callback(); - + return; } - //we need to rename this file with a .txt ending - if(settings.allowUnknownFileEnds === true){ + // we need to rename this file with a .txt ending + if (settings.allowUnknownFileEnds === true) { var oldSrcFile = srcFile; - srcFile = path.join(path.dirname(srcFile),path.basename(srcFile, fileEnding)+".txt"); + srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt"); fs.rename(oldSrcFile, srcFile, callback); - }else{ + } else { console.warn("Not allowing unknown file type to be imported", fileEnding); callback("uploadFailed"); } }, - function(callback){ + + function(callback) { destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension); // Logic for allowing external Import Plugins - hooks.aCallAll("import", {srcFile: srcFile, destFile: destFile}, function(err, result){ - if(ERR(err, callback)) return callback(); - if(result.length > 0){ // This feels hacky and wrong.. + hooks.aCallAll("import", { srcFile: srcFile, destFile: destFile }, function(err, result) { + if (ERR(err, callback)) return callback(); + + if (result.length > 0) { // This feels hacky and wrong.. importHandledByPlugin = true; } callback(); }); }, + function(callback) { var fileEnding = path.extname(srcFile).toLowerCase() var fileIsNotEtherpad = (fileEnding !== ".etherpad"); @@ -141,23 +147,24 @@ exports.doImport = function(req, res, padId) return; } - // we do this here so we can see if the pad has quit ea few edits - padManager.getPad(padId, function(err, _pad){ + // we do this here so we can see if the pad has quite a few edits + padManager.getPad(padId, function(err, _pad) { var headCount = _pad.head; - if(headCount >= 10){ - apiLogger.warn("Direct database Import attempt of a pad that already has content, we wont be doing this") + if (headCount >= 10) { + apiLogger.warn("Direct database Import attempt of a pad that already has content, we wont be doing this"); return callback("padHasData"); } - fs.readFile(srcFile, "utf8", function(err, _text){ + fs.readFile(srcFile, "utf8", function(err, _text) { directDatabaseAccess = true; - importEtherpad.setPadRaw(padId, _text, function(err){ + importEtherpad.setPadRaw(padId, _text, function(err) { callback(); }); }); }); }, - //convert file to html + + // convert file to html if necessary function(callback) { if (importHandledByPlugin || directDatabaseAccess) { callback(); @@ -168,18 +175,20 @@ exports.doImport = function(req, res, padId) var fileEnding = path.extname(srcFile).toLowerCase(); var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm"); var fileIsTXT = (fileEnding === ".txt"); + if (fileIsTXT) useConvertor = false; // Don't use convertor for text files + // See https://github.com/ether/etherpad-lite/issues/2572 if (fileIsHTML || (useConvertor === false)) { // if no convertor only rename fs.rename(srcFile, destFile, callback); - + return; } convertor.convertFile(srcFile, destFile, exportExtension, function(err) { - //catch convert errors - if(err) { + // catch convert errors + if (err) { console.warn("Converting Error:", err); return callback("convertFailed"); } @@ -187,7 +196,7 @@ exports.doImport = function(req, res, padId) callback(); }); }, - + function(callback) { if (useConvertor || directDatabaseAccess) { callback(); @@ -216,17 +225,17 @@ exports.doImport = function(req, res, padId) callback(); }); }, - - //get the pad object + + // get the pad object function(callback) { - padManager.getPad(padId, function(err, _pad){ - if(ERR(err, callback)) return; + padManager.getPad(padId, function(err, _pad) { + if (ERR(err, callback)) return; pad = _pad; callback(); }); }, - - //read the text + + // read the text function(callback) { if (directDatabaseAccess) { callback(); @@ -234,43 +243,46 @@ exports.doImport = function(req, res, padId) return; } - fs.readFile(destFile, "utf8", function(err, _text){ - if(ERR(err, callback)) return; + fs.readFile(destFile, "utf8", function(err, _text) { + if (ERR(err, callback)) return; text = _text; // Title needs to be stripped out else it appends it to the pad.. text = text.replace("", "<!-- <title>"); text = text.replace("","-->"); - //node on windows has a delay on releasing of the file lock. - //We add a 100ms delay to work around this - if(os.type().indexOf("Windows") > -1){ + // node on windows has a delay on releasing of the file lock. + // We add a 100ms delay to work around this + if (os.type().indexOf("Windows") > -1) { setTimeout(function() {callback();}, 100); } else { callback(); } }); }, - - //change text of the pad and broadcast the changeset + + // change text of the pad and broadcast the changeset function(callback) { - if(!directDatabaseAccess){ + if (!directDatabaseAccess) { var fileEnding = path.extname(srcFile).toLowerCase(); if (importHandledByPlugin || useConvertor || fileEnding == ".htm" || fileEnding == ".html") { importHtml.setPadHTML(pad, text, function(e){ - if(e) apiLogger.warn("Error importing, possibly caused by malformed HTML"); + if (e) { + apiLogger.warn("Error importing, possibly caused by malformed HTML"); + } }); } else { pad.setText(text); } } - // Load the Pad into memory then brodcast updates to all clients + // Load the Pad into memory then broadcast updates to all clients padManager.unloadPad(padId); - padManager.getPad(padId, function(err, _pad){ + padManager.getPad(padId, function(err, _pad) { var pad = _pad; padManager.unloadPad(padId); + // direct Database Access means a pad user should perform a switchToPad - // and not attempt to recieve updated pad data.. + // and not attempt to receive updated pad data if (directDatabaseAccess) { callback(); @@ -283,8 +295,8 @@ exports.doImport = function(req, res, padId) }); }, - - //clean up temporary files + + // clean up temporary files function(callback) { if (directDatabaseAccess) { callback(); @@ -308,17 +320,16 @@ exports.doImport = function(req, res, padId) } ], function(err) { var status = "ok"; - - //check for known errors and replace the status - if(err == "uploadFailed" || err == "convertFailed" || err == "padHasData") - { + + // check for known errors and replace the status + if (err == "uploadFailed" || err == "convertFailed" || err == "padHasData") { status = err; err = null; } ERR(err); - //close the connection + // close the connection res.send( " \ \ @@ -331,4 +342,3 @@ exports.doImport = function(req, res, padId) ); }); } - diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 506b3aae4..5c20ed1b6 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -54,8 +54,8 @@ exports.sessioninfos = sessioninfos; // Measure total amount of users stats.gauge('totalUsers', function() { - return Object.keys(socketio.sockets.sockets).length -}) + return Object.keys(socketio.sockets.sockets).length; +}); /** * A changeset queue per pad that is processed by handleUserChanges() @@ -63,7 +63,7 @@ stats.gauge('totalUsers', function() { var padChannels = new channels.channels(handleUserChanges); /** - * Saves the Socket class we need to send and recieve data from the client + * Saves the Socket class we need to send and receive data from the client */ var socketio; @@ -84,7 +84,7 @@ exports.handleConnect = function(client) { stats.meter('connects').mark(); - //Initalize sessioninfos for this new session + // Initalize sessioninfos for this new session sessioninfos[client.id]={}; } @@ -97,11 +97,11 @@ exports.kickSessionsFromPad = function(padID) if(typeof socketio.sockets['clients'] !== 'function') return; - //skip if there is nobody on this pad + // skip if there is nobody on this pad if(_getRoomClients(padID).length == 0) return; - //disconnect everyone from this pad + // disconnect everyone from this pad socketio.sockets.in(padID).json.send({disconnect:"deleted"}); } @@ -113,29 +113,26 @@ exports.handleDisconnect = function(client) { stats.meter('disconnects').mark(); - //save the padname of this session + // save the padname of this session var session = sessioninfos[client.id]; - //if this connection was already etablished with a handshake, send a disconnect message to the others - if(session && session.author) - { - + // if this connection was already etablished with a handshake, send a disconnect message to the others + if (session && session.author) { // Get the IP address from our persistant object var ip = remoteAddress[client.id]; // Anonymize the IP address if IP logging is disabled - if(settings.disableIPlogging) { + if (settings.disableIPlogging) { ip = 'ANONYMOUS'; } - accessLogger.info('[LEAVE] Pad "'+session.padId+'": Author "'+session.author+'" on client '+client.id+' with IP "'+ip+'" left the pad') + accessLogger.info('[LEAVE] Pad "' + session.padId + '": Author "' + session.author + '" on client ' + client.id + ' with IP "' + ip + '" left the pad'); - //get the author color out of the db - authorManager.getAuthorColorId(session.author, function(err, color) - { + // get the author color out of the db + authorManager.getAuthorColorId(session.author, function(err, color) { ERR(err); - //prepare the notification for the other users on the pad, that this user left + // prepare the notification for the other users on the pad, that this user left var messageToTheOtherUsers = { "type": "COLLABROOM", "data": { @@ -149,7 +146,7 @@ exports.handleDisconnect = function(client) } }; - //Go trough all user that are still on the pad, and send them the USER_LEAVE message + // Go through all user that are still on the pad, and send them the USER_LEAVE message client.broadcast.to(session.padId).json.send(messageToTheOtherUsers); // Allow plugins to hook into users leaving the pad @@ -157,7 +154,7 @@ exports.handleDisconnect = function(client) }); } - //Delete the sessioninfos entrys of this session + // Delete the sessioninfos entrys of this session delete sessioninfos[client.id]; } @@ -168,23 +165,24 @@ exports.handleDisconnect = function(client) */ exports.handleMessage = function(client, message) { - if(message == null) - { + if (message == null) { return; } - if(!message.type) - { + + if (!message.type) { return; } - var thisSession = sessioninfos[client.id] - if(!thisSession) { + + var thisSession = sessioninfos[client.id]; + + if (!thisSession) { messageLogger.warn("Dropped message from an unknown connection.") return; } - var handleMessageHook = function(callback){ + var handleMessageHook = function(callback) { // Allow plugins to bypass the readonly message blocker - hooks.aCallAll("handleMessageSecurity", { client: client, message: message }, function ( err, messages ) { + hooks.aCallAll("handleMessageSecurity", { client: client, message: message }, function( err, messages ) { if(ERR(err, callback)) return; _.each(messages, function(newMessage){ if ( newMessage === true ) { @@ -196,7 +194,7 @@ exports.handleMessage = function(client, message) var dropMessage = false; // Call handleMessage hook. If a plugin returns null, the message will be dropped. Note that for all messages // handleMessage will be called, even if the client is not authorized - hooks.aCallAll("handleMessage", { client: client, message: message }, function ( err, messages ) { + hooks.aCallAll("handleMessage", { client: client, message: message }, function( err, messages ) { if(ERR(err, callback)) return; _.each(messages, function(newMessage){ if ( newMessage === null ) { @@ -210,18 +208,18 @@ exports.handleMessage = function(client, message) } - var finalHandler = function () { - //Check what type of message we get and delegate to the other methodes - if(message.type == "CLIENT_READY") { + var finalHandler = function() { + // Check what type of message we get and delegate to the other methods + if (message.type == "CLIENT_READY") { handleClientReady(client, message); - } else if(message.type == "CHANGESET_REQ") { + } else if (message.type == "CHANGESET_REQ") { handleChangesetRequest(client, message); } else if(message.type == "COLLABROOM") { if (thisSession.readonly) { messageLogger.warn("Dropped message, COLLABROOM for readonly pad"); } else if (message.data.type == "USER_CHANGES") { stats.counter('pendingEdits').inc() - padChannels.emit(message.padId, {client: client, message: message});// add to pad queue + padChannels.emit(message.padId, {client: client, message: message}); // add to pad queue } else if (message.data.type == "USERINFO_UPDATE") { handleUserInfoUpdate(client, message); } else if (message.data.type == "CHAT_MESSAGE") { @@ -242,7 +240,7 @@ exports.handleMessage = function(client, message) } else { messageLogger.warn("Dropped message, unknown Message Type " + message.type); } - }; + } /* * In a previous version of this code, an "if (message)" wrapped the @@ -259,11 +257,11 @@ exports.handleMessage = function(client, message) async.series([ handleMessageHook, - //check permissions - function(callback) - { + + // check permissions + function(callback) { // client tried to auth for the first time (first msg from the client) - if(message.type == "CLIENT_READY") { + if (message.type == "CLIENT_READY") { createSessionInfo(client, message); } @@ -274,31 +272,27 @@ exports.handleMessage = function(client, message) // FIXME: Allow to override readwrite access with readonly // Simulate using the load testing tool - if(!sessioninfos[client.id].auth){ + if (!sessioninfos[client.id].auth) { console.error("Auth was never applied to a session. If you are using the stress-test tool then restart Etherpad and the Stress test tool.") return; } var auth = sessioninfos[client.id].auth; - var checkAccessCallback = function(err, statusObject) - { - if(ERR(err, callback)) return; + var checkAccessCallback = function(err, statusObject) { + if (ERR(err, callback)) return; - //access was granted - if(statusObject.accessStatus == "grant") - { + if (statusObject.accessStatus == "grant") { + // access was granted callback(); - } - //no access, send the client a message that tell him why - else - { + } else { + // no access, send the client a message that tells him why client.json.send({accessStatus: statusObject.accessStatus}) } }; - //check if pad is requested via readOnly + // check if pad is requested via readOnly if (auth.padID.indexOf("r.") === 0) { - //Pad is readOnly, first get the real Pad ID + // Pad is readOnly, first get the real Pad ID readOnlyManager.getPadId(auth.padID, function(err, value) { ERR(err); securityManager.checkAccess(value, auth.sessionID, auth.token, auth.password, checkAccessCallback); @@ -321,40 +315,40 @@ function handleSaveRevisionMessage(client, message){ var padId = sessioninfos[client.id].padId; var userId = sessioninfos[client.id].author; - padManager.getPad(padId, function(err, pad) - { - if(ERR(err)) return; + padManager.getPad(padId, function(err, pad) { + if (ERR(err)) return; pad.addSavedRevision(pad.head, userId); }); } /** - * Handles a custom message, different to the function below as it handles objects not strings and you can - * direct the message to specific sessionID + * Handles a custom message, different to the function below as it handles + * objects not strings and you can direct the message to specific sessionID * * @param msg {Object} the message we're sending * @param sessionID {string} the socketIO session to which we're sending this message */ -exports.handleCustomObjectMessage = function (msg, sessionID, cb) { - if(msg.data.type === "CUSTOM"){ - if(sessionID){ // If a sessionID is targeted then send directly to this sessionID - socketio.sockets.socket(sessionID).json.send(msg); // send a targeted message - }else{ - socketio.sockets.in(msg.data.payload.padId).json.send(msg); // broadcast to all clients on this pad +exports.handleCustomObjectMessage = function(msg, sessionID, cb) { + if (msg.data.type === "CUSTOM") { + if (sessionID){ + // a sessionID is targeted: directly to this sessionID + socketio.sockets.socket(sessionID).json.send(msg); + } else { + // broadcast to all clients on this pad + socketio.sockets.in(msg.data.payload.padId).json.send(msg); } } cb(null, {}); } - /** * Handles a custom message (sent via HTTP API request) * * @param padID {Pad} the pad to which we're sending this message * @param msgString {String} the message we're sending */ -exports.handleCustomMessage = function (padID, msgString, cb) { +exports.handleCustomMessage = function(padID, msgString, cb) { var time = Date.now(); var msg = { type: 'COLLABROOM', @@ -390,34 +384,33 @@ function handleChatMessage(client, message) * @param text the text of the chat message * @param padId the padId to send the chat message to */ -exports.sendChatMessageToPadClients = function (time, userId, text, padId) { +exports.sendChatMessageToPadClients = function(time, userId, text, padId) { var pad; var userName; async.series([ - //get the pad - function(callback) - { - padManager.getPad(padId, function(err, _pad) - { - if(ERR(err, callback)) return; + // get the pad + function(callback) { + padManager.getPad(padId, function(err, _pad) { + if (ERR(err, callback)) return; + pad = _pad; callback(); }); }, - function(callback) - { - authorManager.getAuthorName(userId, function(err, _userName) - { - if(ERR(err, callback)) return; + + function(callback) { + authorManager.getAuthorName(userId, function(err, _userName) { + if (ERR(err, callback)) return; + userName = _userName; callback(); }); }, - //save the chat message and broadcast it - function(callback) - { - //save the chat message + + // save the chat message and broadcast it + function(callback) { + // save the chat message pad.appendChatMessage(text, userId, time); var msg = { @@ -431,13 +424,13 @@ exports.sendChatMessageToPadClients = function (time, userId, text, padId) { } }; - //broadcast the chat message to everyone on the pad + // broadcast the chat message to everyone on the pad socketio.sockets.in(padId).json.send(msg); callback(); } - ], function(err) - { + ], + function(err) { ERR(err); }); } @@ -449,13 +442,12 @@ exports.sendChatMessageToPadClients = function (time, userId, text, padId) { */ function handleGetChatMessages(client, message) { - if(message.data.start == null) - { + if (message.data.start == null) { messageLogger.warn("Dropped message, GetChatMessages Message has no start!"); return; } - if(message.data.end == null) - { + + if (message.data.end == null) { messageLogger.warn("Dropped message, GetChatMessages Message has no start!"); return; } @@ -464,8 +456,7 @@ function handleGetChatMessages(client, message) var end = message.data.end; var count = end - start; - if(count < 0 || count > 100) - { + if (count < 0 || count > 100) { messageLogger.warn("Dropped message, GetChatMessages Message, client requested invalid amout of messages!"); return; } @@ -474,21 +465,19 @@ function handleGetChatMessages(client, message) var pad; async.series([ - //get the pad - function(callback) - { - padManager.getPad(padId, function(err, _pad) - { - if(ERR(err, callback)) return; + // get the pad + function(callback) { + padManager.getPad(padId, function(err, _pad) { + if (ERR(err, callback)) return; + pad = _pad; callback(); }); }, - function(callback) - { - pad.getChatMessages(start, end, function(err, chatMessages) - { - if(ERR(err, callback)) return; + + function(callback) { + pad.getChatMessages(start, end, function(err, chatMessages) { + if (ERR(err, callback)) return; var infoMsg = { type: "COLLABROOM", @@ -511,14 +500,13 @@ function handleGetChatMessages(client, message) */ function handleSuggestUserName(client, message) { - //check if all ok - if(message.data.payload.newName == null) - { + // check if all ok + if (message.data.payload.newName == null) { messageLogger.warn("Dropped message, suggestUserName Message has no newName!"); return; } - if(message.data.payload.unnamedId == null) - { + + if (message.data.payload.unnamedId == null) { messageLogger.warn("Dropped message, suggestUserName Message has no unnamedId!"); return; } @@ -526,10 +514,10 @@ function handleSuggestUserName(client, message) var padId = sessioninfos[client.id].padId; var roomClients = _getRoomClients(padId); - //search the author and send him this message + // search the author and send him this message roomClients.forEach(function(client) { var session = sessioninfos[client.id]; - if(session && session.author == message.data.payload.unnamedId) { + if (session && session.author == message.data.payload.unnamedId) { client.json.send(message); } }); @@ -542,37 +530,35 @@ function handleSuggestUserName(client, message) */ function handleUserInfoUpdate(client, message) { - //check if all ok - if(message.data.userInfo == null) - { + // check if all ok + if (message.data.userInfo == null) { messageLogger.warn("Dropped message, USERINFO_UPDATE Message has no userInfo!"); return; } - if(message.data.userInfo.colorId == null) - { + + if (message.data.userInfo.colorId == null) { messageLogger.warn("Dropped message, USERINFO_UPDATE Message has no colorId!"); return; } // Check that we have a valid session and author to update. var session = sessioninfos[client.id]; - if(!session || !session.author || !session.padId) - { + if (!session || !session.author || !session.padId) { messageLogger.warn("Dropped message, USERINFO_UPDATE Session not ready." + message.data); return; } - //Find out the author name of this session + // Find out the author name of this session var author = session.author; // Check colorId is a Hex color var isColor = /(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(message.data.userInfo.colorId) // for #f00 (Thanks Smamatti) - if(!isColor){ + if (!isColor) { messageLogger.warn("Dropped message, USERINFO_UPDATE Color is malformed." + message.data); return; } - //Tell the authorManager about the new attributes + // Tell the authorManager about the new attributes authorManager.setAuthorColorId(author, message.data.userInfo.colorId); authorManager.setAuthorName(author, message.data.userInfo.name); @@ -585,7 +571,7 @@ function handleUserInfoUpdate(client, message) type: "USER_NEWINFO", userInfo: { userId: author, - //set a null name, when there is no name set. cause the client wants it null + // set a null name, when there is no name set. cause the client wants it null name: message.data.userInfo.name || null, colorId: message.data.userInfo.colorId, userAgent: "Anonymous", @@ -594,7 +580,7 @@ function handleUserInfoUpdate(client, message) } }; - //Send the other clients on the pad the update message + // Send the other clients on the pad the update message client.broadcast.to(padId).json.send(infoMsg); } @@ -621,34 +607,34 @@ function handleUserChanges(data, cb) stats.counter('pendingEdits').dec() // Make sure all required fields are present - if(message.data.baseRev == null) - { + if (message.data.baseRev == null) { messageLogger.warn("Dropped message, USER_CHANGES Message has no baseRev!"); return cb(); } - if(message.data.apool == null) - { + + if (message.data.apool == null) { messageLogger.warn("Dropped message, USER_CHANGES Message has no apool!"); return cb(); } - if(message.data.changeset == null) - { + + if (message.data.changeset == null) { messageLogger.warn("Dropped message, USER_CHANGES Message has no changeset!"); return cb(); } - //TODO: this might happen with other messages too => find one place to copy the session - //and always use the copy. atm a message will be ignored if the session is gone even - //if the session was valid when the message arrived in the first place - if(!sessioninfos[client.id]) - { + + // TODO: this might happen with other messages too => find one place to copy the session + // and always use the copy. atm a message will be ignored if the session is gone even + // if the session was valid when the message arrived in the first place + if (!sessioninfos[client.id]) { messageLogger.warn("Dropped message, disconnect happened in the mean time"); return cb(); } - //get all Vars we need + // get all Vars we need var baseRev = message.data.baseRev; var wireApool = (new AttributePool()).fromJsonable(message.data.apool); var changeset = message.data.changeset; + // The client might disconnect between our callbacks. We should still // finish processing the changeset, so keep a reference to the session. var thisSession = sessioninfos[client.id]; @@ -659,31 +645,29 @@ function handleUserChanges(data, cb) var stopWatch = stats.timer('edits').start(); async.series([ - //get the pad - function(callback) - { - padManager.getPad(thisSession.padId, function(err, value) - { - if(ERR(err, callback)) return; + // get the pad + function(callback) { + padManager.getPad(thisSession.padId, function(err, value) { + if (ERR(err, callback)) return; + pad = value; callback(); }); }, - //create the changeset - function(callback) - { - //ex. _checkChangesetAndPool - try - { + // create the changeset + function(callback) { + // ex. _checkChangesetAndPool + + try { // Verify that the changeset has valid syntax and is in canonical form Changeset.checkRep(changeset); // Verify that the attribute indexes used in the changeset are all // defined in the accompanying attribute pool. Changeset.eachAttribNumber(changeset, function(n) { - if (! wireApool.getAttrib(n)) { - throw new Error("Attribute pool is missing attribute "+n+" for changeset "+changeset); + if (!wireApool.getAttrib(n)) { + throw new Error("Attribute pool is missing attribute " + n + " for changeset " + changeset); } }); @@ -693,102 +677,105 @@ function handleUserChanges(data, cb) while(iterator.hasNext()) { op = iterator.next() - //+ can add text with attribs - //= can change or add attribs - //- can have attribs, but they are discarded and don't show up in the attribs - but do show up in the pool + // + can add text with attribs + // = can change or add attribs + // - can have attribs, but they are discarded and don't show up in the attribs - but do show up in the pool op.attribs.split('*').forEach(function(attr) { - if(!attr) return + if (!attr) return; + attr = wireApool.getAttrib(attr) - if(!attr) return - //the empty author is used in the clearAuthorship functionality so this should be the only exception - if('author' == attr[0] && (attr[1] != thisSession.author && attr[1] != '')) throw new Error("Trying to submit changes as another author in changeset "+changeset); - }) + if (!attr) return; + + // the empty author is used in the clearAuthorship functionality so this should be the only exception + if ('author' == attr[0] && (attr[1] != thisSession.author && attr[1] != '')) { + throw new Error("Trying to submit changes as another author in changeset " + changeset); + } + }); } - //ex. adoptChangesetAttribs + // ex. adoptChangesetAttribs - //Afaik, it copies the new attributes from the changeset, to the global Attribute Pool + // Afaik, it copies the new attributes from the changeset, to the global Attribute Pool changeset = Changeset.moveOpsToNewPool(changeset, wireApool, pad.pool); - } - catch(e) - { + } catch(e) { // There is an error in this changeset, so just refuse it client.json.send({disconnect:"badChangeset"}); stats.meter('failedChangesets').mark(); return callback(new Error("Can't apply USER_CHANGES, because "+e.message)); } - //ex. applyUserChanges + // ex. applyUserChanges apool = pad.pool; r = baseRev; // The client's changeset might not be based on the latest revision, // since other clients are sending changes at the same time. // Update the changeset so that it can be applied to the latest revision. - //https://github.com/caolan/async#whilst + // https://github.com/caolan/async#whilst async.whilst( function() { return r < pad.getHeadRevisionNumber(); }, function(callback) { r++; - pad.getRevisionChangeset(r, function(err, c) - { - if(ERR(err, callback)) return; + pad.getRevisionChangeset(r, function(err, c) { + if (ERR(err, callback)) return; // At this point, both "c" (from the pad) and "changeset" (from the // client) are relative to revision r - 1. The follow function // rebases "changeset" so that it is relative to revision r // and can be applied after "c". - try - { + try { // a changeset can be based on an old revision with the same changes in it // prevent eplite from accepting it TODO: better send the client a NEW_CHANGES // of that revision - if(baseRev+1 == r && c == changeset) { + if (baseRev + 1 == r && c == changeset) { client.json.send({disconnect:"badChangeset"}); stats.meter('failedChangesets').mark(); + return callback(new Error("Won't apply USER_CHANGES, because it contains an already accepted changeset")); } + changeset = Changeset.follow(c, changeset, false, apool); - }catch(e){ + } catch(e) { client.json.send({disconnect:"badChangeset"}); stats.meter('failedChangesets').mark(); + return callback(new Error("Can't apply USER_CHANGES, because "+e.message)); } - if ((r - baseRev) % 200 == 0) { // don't let the stack get too deep + if ((r - baseRev) % 200 == 0) { + // don't let the stack get too deep async.nextTick(callback); } else { callback(null); } }); }, - //use the callback of the series function + + // use the callback of the series function callback ); }, - //do correction changesets, and send it to all users - function (callback) - { + + // do correction changesets, and send it to all users + function(callback) { var prevText = pad.text(); - if (Changeset.oldLen(changeset) != prevText.length) - { + if (Changeset.oldLen(changeset) != prevText.length) { client.json.send({disconnect:"badChangeset"}); stats.meter('failedChangesets').mark(); + return callback(new Error("Can't apply USER_CHANGES "+changeset+" with oldLen " + Changeset.oldLen(changeset) + " to document of length " + prevText.length)); } - try - { + try { pad.appendRevision(changeset, thisSession.author); - } - catch(e) - { + } catch(e) { client.json.send({disconnect:"badChangeset"}); stats.meter('failedChangesets').mark(); + return callback(e) } @@ -809,21 +796,25 @@ function handleUserChanges(data, cb) }); callback(); } - ], function(err) - { + ], + function(err) { stopWatch.end() cb(); - if(err) console.warn(err.stack || err) + + if(err) { + console.warn(err.stack || err); + } }); } exports.updatePadClients = function(pad, callback) { - //skip this step if noone is on this pad + // skip this if no-one is on this pad var roomClients = _getRoomClients(pad.id); - if(roomClients.length==0) + if (roomClients.length == 0) { return callback(); + } // since all clients usually get the same set of changesets, store them in local cache // to remove unnecessary roundtrip to the datalayer @@ -833,26 +824,27 @@ exports.updatePadClients = function(pad, callback) // but benefit of reusing cached revision object is HUGE var revCache = {}; - //go trough all sessions on this pad + // go through all sessions on this pad async.forEach(roomClients, function(client, callback){ var sid = client.id; - //https://github.com/caolan/async#whilst - //send them all new changesets + // https://github.com/caolan/async#whilst + // send them all new changesets async.whilst( - function (){ return sessioninfos[sid] && sessioninfos[sid].rev < pad.getHeadRevisionNumber()}, + function() { return sessioninfos[sid] && sessioninfos[sid].rev < pad.getHeadRevisionNumber()}, function(callback) { var r = sessioninfos[sid].rev + 1; async.waterfall([ function(callback) { - if(revCache[r]) + if(revCache[r]) { callback(null, revCache[r]); - else + } else { pad.getRevision(r, callback); + } }, - function(revision, callback) - { + + function(revision, callback) { revCache[r] = revision; var author = revision.meta.author, @@ -860,15 +852,13 @@ exports.updatePadClients = function(pad, callback) currentTime = revision.meta.timestamp; // next if session has not been deleted - if(sessioninfos[sid] == null) + if (sessioninfos[sid] == null) { return callback(null); - - if(author == sessioninfos[sid].author) - { - client.json.send({"type":"COLLABROOM","data":{type:"ACCEPT_COMMIT", newRev:r}}); } - else - { + + if (author == sessioninfos[sid].author) { + client.json.send({"type":"COLLABROOM","data":{type:"ACCEPT_COMMIT", newRev:r}}); + } else { var forWire = Changeset.prepareForWire(revChangeset, pad.pool); var wireMsg = {"type":"COLLABROOM", "data":{type:"NEW_CHANGES", @@ -882,7 +872,8 @@ exports.updatePadClients = function(pad, callback) client.json.send(wireMsg); } - if(sessioninfos[sid]){ + + if (sessioninfos[sid]) { sessioninfos[sid].time = currentTime; sessioninfos[sid].rev = r; } @@ -909,19 +900,18 @@ function _correctMarkersInPad(atext, apool) { while (iter.hasNext()) { var op = iter.next(); - var hasMarker = _.find(AttributeManager.lineAttributes, function(attribute){ + var hasMarker = _.find(AttributeManager.lineAttributes, function(attribute) { return Changeset.opAttributeValue(op, attribute, apool); }) !== undefined; if (hasMarker) { - for(var i=0;i 0 && text.charAt(offset-1) != '\n') { badMarkers.push(offset); } offset++; } - } - else { + } else { offset += op.chars; } } @@ -932,12 +922,15 @@ function _correctMarkersInPad(atext, apool) { // create changeset that removes these bad markers offset = 0; + var builder = Changeset.builder(text.length); + badMarkers.forEach(function(pos) { builder.keepText(text.substring(offset, pos)); builder.remove(1); offset = pos+1; }); + return builder.toString(); } @@ -950,7 +943,7 @@ function handleSwitchToPad(client, message) async.forEach(roomClients, function(client, callback) { var sinfo = sessioninfos[client.id]; - if(sinfo && sinfo.author == currentSession.author) { + if (sinfo && sinfo.author == currentSession.author) { // fix user's counter, works on page refresh or if user closes browser window and then rejoins sessioninfos[client.id] = {}; client.leave(padId); @@ -986,24 +979,23 @@ function createSessionInfo(client, message) */ function handleClientReady(client, message) { - //check if all ok - if(!message.token) - { + // check if all ok + if (!message.token) { messageLogger.warn("Dropped message, CLIENT_READY Message has no token!"); return; } - if(!message.padId) - { + + if (!message.padId) { messageLogger.warn("Dropped message, CLIENT_READY Message has no padId!"); return; } - if(!message.protocolVersion) - { + + if (!message.protocolVersion) { messageLogger.warn("Dropped message, CLIENT_READY Message has no protocolVersion!"); return; } - if(message.protocolVersion != 2) - { + + if (message.protocolVersion != 2) { messageLogger.warn("Dropped message, CLIENT_READY Message has a unknown protocolVersion '" + message.protocolVersion + "'!"); return; } @@ -1019,97 +1011,92 @@ function handleClientReady(client, message) hooks.callAll("clientReady", message); async.series([ - //Get ro/rw id:s - function (callback) - { + // Get ro/rw id:s + function(callback) { readOnlyManager.getIds(message.padId, function(err, value) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; + padIds = value; callback(); }); }, - //check permissions - function(callback) - { + + // check permissions + function(callback) { // Note: message.sessionID is an entierly different kind of - // session from the sessions we use here! Beware! FIXME: Call - // our "sessions" "connections". + // session from the sessions we use here! Beware! + // FIXME: Call our "sessions" "connections". // FIXME: Use a hook instead // FIXME: Allow to override readwrite access with readonly - securityManager.checkAccess (padIds.padId, message.sessionID, message.token, message.password, function(err, statusObject) - { - if(ERR(err, callback)) return; + securityManager.checkAccess(padIds.padId, message.sessionID, message.token, message.password, function(err, statusObject) { + if (ERR(err, callback)) return; - //access was granted - if(statusObject.accessStatus == "grant") - { + if (statusObject.accessStatus == "grant") { + // access was granted author = statusObject.authorID; callback(); - } - //no access, send the client a message that tell him why - else - { + } else { + // no access, send the client a message that tells him why client.json.send({accessStatus: statusObject.accessStatus}) } }); }, - //get all authordata of this new user, and load the pad-object from the database + + // get all authordata of this new user, and load the pad-object from the database function(callback) { async.parallel([ - //get colorId and name - function(callback) - { - authorManager.getAuthor(author, function(err, value) - { - if(ERR(err, callback)) return; + // get colorId and name + function(callback) { + authorManager.getAuthor(author, function(err, value) { + if (ERR(err, callback)) return; + authorColorId = value.colorId; authorName = value.name; callback(); }); }, - //get pad - function(callback) - { - padManager.getPad(padIds.padId, function(err, value) - { - if(ERR(err, callback)) return; + + // get pad + function(callback) { + padManager.getPad(padIds.padId, function(err, value) { + if (ERR(err, callback)) return; + pad = value; callback(); }); } ], callback); }, - //these db requests all need the pad object (timestamp of latest revission, author data) - function(callback) - { + + // these db requests all need the pad object (timestamp of latest revission, author data) + function(callback) { var authors = pad.getAllAuthors(); async.parallel([ - //get timestamp of latest revission needed for timeslider - function(callback) - { - pad.getRevisionDate(pad.getHeadRevisionNumber(), function(err, date) - { - if(ERR(err, callback)) return; + // get timestamp of latest revission needed for timeslider + function(callback) { + pad.getRevisionDate(pad.getHeadRevisionNumber(), function(err, date) { + if (ERR(err, callback)) return; + currentTime = date; callback(); }); }, - //get all author data out of the database - function(callback) - { - async.forEach(authors, function(authorId, callback) - { - authorManager.getAuthor(authorId, function(err, author) - { - if(!author && !err) - { + + // get all author data out of the database + function(callback) { + async.forEach(authors, function(authorId, callback) { + authorManager.getAuthor(authorId, function(err, author) { + if (!author && !err) { messageLogger.error("There is no author for authorId:", authorId); + return callback(); } - if(ERR(err, callback)) return; - historicalAuthorData[authorId] = {name: author.name, colorId: author.colorId}; // Filter author attribs (e.g. don't send author's pads to all clients) + + if (ERR(err, callback)) return; + + historicalAuthorData[authorId] = { name: author.name, colorId: author.colorId }; // Filter author attribs (e.g. don't send author's pads to all clients) callback(); }); }, callback); @@ -1117,56 +1104,57 @@ function handleClientReady(client, message) ], callback); }, - //glue the clientVars together, send them and tell the other clients that a new one is there - function(callback) - { - //Check that the client is still here. It might have disconnected between callbacks. - if(sessioninfos[client.id] === undefined) - return callback(); - //Check if this author is already on the pad, if yes, kick the other sessions! + // glue the clientVars together, send them and tell the other clients that a new one is there + function(callback) { + // Check that the client is still here. It might have disconnected between callbacks. + if(sessioninfos[client.id] === undefined) { + return callback(); + } + + // Check if this author is already on the pad, if yes, kick the other sessions! var roomClients = _getRoomClients(pad.id); async.forEach(roomClients, function(client, callback) { var sinfo = sessioninfos[client.id]; - if(sinfo && sinfo.author == author) { + + if (sinfo && sinfo.author == author) { // fix user's counter, works on page refresh or if user closes browser window and then rejoins sessioninfos[client.id] = {}; client.leave(padIds.padId); - client.json.send({disconnect:"userdup"}); + client.json.send({ disconnect:"userdup" }); } }); - //Save in sessioninfos that this session belonges to this pad + // Save in sessioninfos that this session belonges to this pad sessioninfos[client.id].padId = padIds.padId; sessioninfos[client.id].readOnlyPadId = padIds.readOnlyPadId; sessioninfos[client.id].readonly = padIds.readonly; - //Log creation/(re-)entering of a pad + // Log creation/(re-)entering of a pad var ip = remoteAddress[client.id]; - //Anonymize the IP address if IP logging is disabled - if(settings.disableIPlogging) { + // Anonymize the IP address if IP logging is disabled + if (settings.disableIPlogging) { ip = 'ANONYMOUS'; } - if(pad.head > 0) { - accessLogger.info('[ENTER] Pad "'+padIds.padId+'": Client '+client.id+' with IP "'+ip+'" entered the pad'); - } - else if(pad.head == 0) { - accessLogger.info('[CREATE] Pad "'+padIds.padId+'": Client '+client.id+' with IP "'+ip+'" created the pad'); + if (pad.head > 0) { + accessLogger.info('[ENTER] Pad "' + padIds.padId + '": Client ' + client.id + ' with IP "' + ip + '" entered the pad'); + } else if (pad.head == 0) { + accessLogger.info('[CREATE] Pad "' + padIds.padId + '": Client ' + client.id + ' with IP "' + ip + '" created the pad'); } - //If this is a reconnect, we don't have to send the client the ClientVars again - if(message.reconnect == true) - { - //Join the pad and start receiving updates + if (message.reconnect == true) { + // If this is a reconnect, we don't have to send the client the ClientVars again + // Join the pad and start receiving updates client.join(padIds.padId); - //Save the revision in sessioninfos, we take the revision from the info the client send to us + + // Save the revision in sessioninfos, we take the revision from the info the client send to us sessioninfos[client.id].rev = message.client_rev; - //During the client reconnect, client might miss some revisions from other clients. By using client revision, - //this below code sends all the revisions missed during the client reconnect + // During the client reconnect, client might miss some revisions from other clients. By using client revision, + // this below code sends all the revisions missed during the client reconnect var revisionsNeeded = []; var changesets = {}; @@ -1174,68 +1162,68 @@ function handleClientReady(client, message) var endNum = pad.getHeadRevisionNumber() + 1; async.series([ - //push all the revision numbers needed into revisionsNeeded array - function(callback) - { + // push all the revision numbers needed into revisionsNeeded array + function(callback) { var headNum = pad.getHeadRevisionNumber(); - if (endNum > headNum+1) - endNum = headNum+1; - if (startNum < 0) - startNum = 0; - for(var r=startNum;r headNum+1) { + endNum = headNum+1; + } + + if (startNum < 0) { + startNum = 0; + } + + for (var r = startNum; r < endNum; r++) { revisionsNeeded.push(r); changesets[r] = {}; } + callback(); }, - //get changesets needed for pending revisions - function(callback) - { - async.eachSeries(revisionsNeeded, function(revNum, callback) - { - pad.getRevisionChangeset(revNum, function(err, value) - { - if(ERR(err)) return; + + // get changesets needed for pending revisions + function(callback) { + async.eachSeries(revisionsNeeded, function(revNum, callback) { + pad.getRevisionChangeset(revNum, function(err, value) { + if (ERR(err)) return; + changesets[revNum]['changeset'] = value; callback(); }); }, callback); }, - //get author for each changeset - function(callback) - { - async.eachSeries(revisionsNeeded, function(revNum, callback) - { - pad.getRevisionAuthor(revNum, function(err, value) - { - if(ERR(err)) return; + + // get author for each changeset + function(callback) { + async.eachSeries(revisionsNeeded, function(revNum, callback) { + pad.getRevisionAuthor(revNum, function(err, value) { + if (ERR(err)) return; + changesets[revNum]['author'] = value; callback(); }); }, callback); }, - //get timestamp for each changeset - function(callback) - { - async.eachSeries(revisionsNeeded, function(revNum, callback) - { - pad.getRevisionDate(revNum, function(err, value) - { - if(ERR(err)) return; + + // get timestamp for each changeset + function(callback) { + async.eachSeries(revisionsNeeded, function(revNum, callback) { + pad.getRevisionDate(revNum, function(err, value) { + if (ERR(err)) return; + changesets[revNum]['timestamp'] = value; callback(); }); }, callback); } ], - //return error and pending changesets - function(err) - { - if(ERR(err, callback)) return; - async.eachSeries(revisionsNeeded, function(r, callback) - { + + // return error and pending changesets + function(err) { + if (ERR(err, callback)) return; + + async.eachSeries(revisionsNeeded, function(r, callback) { var forWire = Changeset.prepareForWire(changesets[r]['changeset'], pad.pool); var wireMsg = {"type":"COLLABROOM", "data":{type:"CLIENT_RECONNECT", @@ -1249,8 +1237,8 @@ function handleClientReady(client, message) client.json.send(wireMsg); callback(); }); - if (startNum == endNum) - { + + if (startNum == endNum) { var Msg = {"type":"COLLABROOM", "data":{type:"CLIENT_RECONNECT", noChanges: true, @@ -1259,19 +1247,17 @@ function handleClientReady(client, message) client.json.send(Msg); } }); - } - //This is a normal first connect - else - { - //prepare all values for the wire, there'S a chance that this throws, if the pad is corrupted + } else { + // This is a normal first connect + // prepare all values for the wire, there's a chance that this throws, if the pad is corrupted try { var atext = Changeset.cloneAText(pad.atext); var attribsForWire = Changeset.prepareForWire(atext.attribs, pad.pool); var apool = attribsForWire.pool.toJsonable(); atext.attribs = attribsForWire.translated; - }catch(e) { + } catch(e) { console.error(e.stack || e) - client.json.send({disconnect:"corruptPad"});// pull the breaks + client.json.send({ disconnect:"corruptPad" });// pull the brakes return callback(); } @@ -1335,35 +1321,36 @@ function handleClientReady(client, message) "initialChangesets": [] // FIXME: REMOVE THIS SHIT } - //Add a username to the clientVars if one avaiable - if(authorName != null) - { + // Add a username to the clientVars if one avaiable + if (authorName != null) { clientVars.userName = authorName; } - //call the clientVars-hook so plugins can modify them before they get sent to the client - hooks.aCallAll("clientVars", { clientVars: clientVars, pad: pad }, function ( err, messages ) { - if(ERR(err, callback)) return; + // call the clientVars-hook so plugins can modify them before they get sent to the client + hooks.aCallAll("clientVars", { clientVars: clientVars, pad: pad }, function( err, messages ) { + if (ERR(err, callback)) return; _.each(messages, function(newVars) { - //combine our old object with the new attributes from the hook + // combine our old object with the new attributes from the hook for(var attr in newVars) { clientVars[attr] = newVars[attr]; } }); - //Join the pad and start receiving updates + // Join the pad and start receiving updates client.join(padIds.padId); - //Send the clientVars to the Client - client.json.send({type: "CLIENT_VARS", data: clientVars}); - //Save the current revision in sessioninfos, should be the same as in clientVars + + // Send the clientVars to the Client + client.json.send({ type: "CLIENT_VARS", data: clientVars }); + + // Save the current revision in sessioninfos, should be the same as in clientVars sessioninfos[client.id].rev = pad.getHeadRevisionNumber(); }); } sessioninfos[client.id].author = author; - //prepare the notification for the other users on the pad, that this user joined + // prepare the notification for the other users on the pad, that this user joined var messageToTheOtherUsers = { "type": "COLLABROOM", "data": { @@ -1377,47 +1364,47 @@ function handleClientReady(client, message) } }; - //Add the authorname of this new User, if avaiable - if(authorName != null) - { + // Add the authorname of this new User, if avaiable + if (authorName != null) { messageToTheOtherUsers.data.userInfo.name = authorName; } // notify all existing users about new user client.broadcast.to(padIds.padId).json.send(messageToTheOtherUsers); - //Get sessions for this pad + // Get sessions for this pad var roomClients = _getRoomClients(pad.id); - async.forEach(roomClients, function(roomClient, callback) - { + async.forEach(roomClients, function(roomClient, callback) { var author; - //Jump over, if this session is the connection session - if(roomClient.id == client.id) + // Jump over, if this session is the connection session + if (roomClient.id == client.id) { return callback(); + } - - //Since sessioninfos might change while being enumerated, check if the - //sessionID is still assigned to a valid session - if(sessioninfos[roomClient.id] !== undefined) + // Since sessioninfos might change while being enumerated, check if the + // sessionID is still assigned to a valid session + if (sessioninfos[roomClient.id] !== undefined) { author = sessioninfos[roomClient.id].author; - else // If the client id is not valid, callback(); + } else { + // If the client id is not valid, callback(); return callback(); + } async.waterfall([ - //get the authorname & colorId - function(callback) - { + // get the authorname & colorId + function(callback) { // reuse previously created cache of author's data - if(historicalAuthorData[author]) + if (historicalAuthorData[author]) { callback(null, historicalAuthorData[author]); - else + } else { authorManager.getAuthor(author, callback); + } }, - function (authorInfo, callback) - { - //Send the new User a Notification about this other user + + function(authorInfo, callback) { + // Send the new User a Notification about this other user var msg = { "type": "COLLABROOM", "data": { @@ -1431,13 +1418,14 @@ function handleClientReady(client, message) } } }; + client.json.send(msg); } ], callback); }, callback); } - ],function(err) - { + ], + function(err) { ERR(err); }); } @@ -1447,35 +1435,34 @@ function handleClientReady(client, message) */ function handleChangesetRequest(client, message) { - //check if all ok - if(message.data == null) - { + // check if all ok + if (message.data == null) { messageLogger.warn("Dropped message, changeset request has no data!"); return; } - if(message.padId == null) - { + + if (message.padId == null) { messageLogger.warn("Dropped message, changeset request has no padId!"); return; } - if(message.data.granularity == null) - { + + if (message.data.granularity == null) { messageLogger.warn("Dropped message, changeset request has no granularity!"); return; } - //https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isInteger#Polyfill - if(Math.floor(message.data.granularity) !== message.data.granularity) - { + + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isInteger#Polyfill + if (Math.floor(message.data.granularity) !== message.data.granularity) { messageLogger.warn("Dropped message, changeset request granularity is not an integer!"); return; } - if(message.data.start == null) - { + + if (message.data.start == null) { messageLogger.warn("Dropped message, changeset request has no start!"); return; } - if(message.data.requestID == null) - { + + if (message.data.requestID == null) { messageLogger.warn("Dropped message, changeset request has no requestID!"); return; } @@ -1486,23 +1473,24 @@ function handleChangesetRequest(client, message) var padIds; async.series([ - function (callback) { + function(callback) { readOnlyManager.getIds(message.padId, function(err, value) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; + padIds = value; callback(); }); }, - function (callback) { - //build the requested rough changesets and send them back - getChangesetInfo(padIds.padId, start, end, granularity, function(err, changesetInfo) - { - if(err) return console.error('Error while handling a changeset request for '+padIds.padId, err, message.data); + + function(callback) { + // build the requested rough changesets and send them back + getChangesetInfo(padIds.padId, start, end, granularity, function(err, changesetInfo) { + if (err) return console.error('Error while handling a changeset request for ' + padIds.padId, err, message.data); var data = changesetInfo; data.requestID = message.data.requestID; - client.json.send({type: "CHANGESET_REQ", data: data}); + client.json.send({ type: "CHANGESET_REQ", data: data }); }); } ]); @@ -1526,96 +1514,91 @@ function getChangesetInfo(padId, startNum, endNum, granularity, callback) var head_revision = 0; async.series([ - //get the pad from the database - function(callback) - { - padManager.getPad(padId, function(err, _pad) - { - if(ERR(err, callback)) return; + // get the pad from the database + function(callback) { + padManager.getPad(padId, function(err, _pad) { + if (ERR(err, callback)) return; + pad = _pad; head_revision = pad.getHeadRevisionNumber(); callback(); }); }, - function(callback) - { - //calculate the last full endnum + + function(callback) { + // calculate the last full endnum var lastRev = pad.getHeadRevisionNumber(); - if (endNum > lastRev+1) { - endNum = lastRev+1; + if (endNum > lastRev + 1) { + endNum = lastRev + 1; } - endNum = Math.floor(endNum / granularity)*granularity; + + endNum = Math.floor(endNum / granularity) * granularity; var compositesChangesetNeeded = []; var revTimesNeeded = []; - //figure out which composite Changeset and revTimes we need, to load them in bulk + // figure out which composite Changeset and revTimes we need, to load them in bulk var compositeStart = startNum; - while (compositeStart < endNum) - { + while (compositeStart < endNum) { var compositeEnd = compositeStart + granularity; - //add the composite Changeset we needed - compositesChangesetNeeded.push({start: compositeStart, end: compositeEnd}); + // add the composite Changeset we needed + compositesChangesetNeeded.push({ start: compositeStart, end: compositeEnd }); - //add the t1 time we need + // add the t1 time we need revTimesNeeded.push(compositeStart == 0 ? 0 : compositeStart - 1); - //add the t2 time we need + + // add the t2 time we need revTimesNeeded.push(compositeEnd - 1); compositeStart += granularity; } - //get all needed db values parallel + // get all needed db values parallel async.parallel([ - function(callback) - { - //get all needed composite Changesets - async.forEach(compositesChangesetNeeded, function(item, callback) - { - composePadChangesets(padId, item.start, item.end, function(err, changeset) - { - if(ERR(err, callback)) return; + function(callback) { + // get all needed composite Changesets + async.forEach(compositesChangesetNeeded, function(item, callback) { + composePadChangesets(padId, item.start, item.end, function(err, changeset) { + if (ERR(err, callback)) return; + composedChangesets[item.start + "/" + item.end] = changeset; callback(); }); }, callback); }, - function(callback) - { - //get all needed revision Dates - async.forEach(revTimesNeeded, function(revNum, callback) - { - pad.getRevisionDate(revNum, function(err, revDate) - { - if(ERR(err, callback)) return; + + function(callback) { + // get all needed revision Dates + async.forEach(revTimesNeeded, function(revNum, callback) { + pad.getRevisionDate(revNum, function(err, revDate) { + if (ERR(err, callback)) return; + revisionDate[revNum] = Math.floor(revDate/1000); callback(); }); }, callback); }, - //get the lines - function(callback) - { - getPadLines(padId, startNum-1, function(err, _lines) - { - if(ERR(err, callback)) return; + + // get the lines + function(callback) { + getPadLines(padId, startNum-1, function(err, _lines) { + if (ERR(err, callback)) return; + lines = _lines; callback(); }); } ], callback); }, - //doesn't know what happens here excatly :/ - function(callback) - { + + // don't know what happens here exactly :/ + function(callback) { var compositeStart = startNum; - while (compositeStart < endNum) - { + while (compositeStart < endNum) { var compositeEnd = compositeStart + granularity; - if (compositeEnd > endNum || compositeEnd > head_revision+1) - { + if (compositeEnd > endNum || compositeEnd > head_revision+1) { break; } @@ -1629,12 +1612,9 @@ function getChangesetInfo(padId, startNum, endNum, granularity, callback) var backwards2 = Changeset.moveOpsToNewPool(backwards, pad.apool(), apool); var t1, t2; - if (compositeStart == 0) - { + if (compositeStart == 0) { t1 = revisionDate[0]; - } - else - { + } else { t1 = revisionDate[compositeStart - 1]; } @@ -1649,9 +1629,9 @@ function getChangesetInfo(padId, startNum, endNum, granularity, callback) callback(); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + function(err) { + if (ERR(err, callback)) return; callback(null, {forwardsChangesets: forwardsChangesets, backwardsChangesets: backwardsChangesets, @@ -1674,43 +1654,41 @@ function getPadLines(padId, revNum, callback) var pad; async.series([ - //get the pad from the database - function(callback) - { - padManager.getPad(padId, function(err, _pad) - { - if(ERR(err, callback)) return; + // get the pad from the database + function(callback) { + padManager.getPad(padId, function(err, _pad) { + if (ERR(err, callback)) return; + pad = _pad; callback(); }); }, - //get the atext - function(callback) - { - if(revNum >= 0) - { - pad.getInternalRevisionAText(revNum, function(err, _atext) - { - if(ERR(err, callback)) return; + + // get the atext + function(callback) { + if (revNum >= 0) { + pad.getInternalRevisionAText(revNum, function(err, _atext) { + if (ERR(err, callback)) return; + atext = _atext; callback(); }); - } - else - { + } else { atext = Changeset.makeAText("\n"); callback(null); } }, - function(callback) - { + + function(callback) { result.textlines = Changeset.splitTextLines(atext.text); result.alines = Changeset.splitAttributionLines(atext.attribs, atext.text); callback(null); } - ], function(err) - { - if(ERR(err, callback)) return; + ], + + function(err) { + if (ERR(err, callback)) return; + callback(null, result); }); } @@ -1726,74 +1704,77 @@ function composePadChangesets(padId, startNum, endNum, callback) var changeset; async.series([ - //get the pad from the database - function(callback) - { - padManager.getPad(padId, function(err, _pad) - { - if(ERR(err, callback)) return; + // get the pad from the database + function(callback) { + padManager.getPad(padId, function(err, _pad) { + if (ERR(err, callback)) return; + pad = _pad; callback(); }); }, - //fetch all changesets we need - function(callback) - { + + // fetch all changesets we need + function(callback) { var changesetsNeeded=[]; var headNum = pad.getHeadRevisionNumber(); - if (endNum > headNum+1) - endNum = headNum+1; - if (startNum < 0) + if (endNum > headNum + 1) { + endNum = headNum + 1; + } + + if (startNum < 0) { startNum = 0; - //create a array for all changesets, we will - //replace the values with the changeset later - for(var r=startNum;r b[property]) - return dir? 1 : -1; + if (a[property] < b[property]) { + return dir? -1 : 1; + } + + if (a[property] > b[property]) { + return dir? 1 : -1; + } + // a must be equal to b return 0; - }) + }); } diff --git a/src/node/hooks/express/errorhandling.js b/src/node/hooks/express/errorhandling.js index 104a9c1bb..9a07dc669 100644 --- a/src/node/hooks/express/errorhandling.js +++ b/src/node/hooks/express/errorhandling.js @@ -11,20 +11,23 @@ exports.gracefulShutdown = function(err) { console.error(err); } - //ensure there is only one graceful shutdown running - if(exports.onShutdown) return; + // ensure there is only one graceful shutdown running + if (exports.onShutdown) { + return; + } + exports.onShutdown = true; console.log("graceful shutdown..."); - //do the db shutdown + // do the db shutdown db.db.doShutdown(function() { console.log("db sucessfully closed."); process.exit(0); }); - setTimeout(function(){ + setTimeout(function() { process.exit(1); }, 3000); } @@ -35,14 +38,14 @@ exports.expressCreateServer = function (hook_name, args, cb) { exports.app = args.app; // Handle errors - args.app.use(function(err, req, res, next){ + args.app.use(function(err, req, res, next) { // if an error occurs Connect will pass it down // through these "error-handling" middleware // allowing you to respond however you like res.status(500).send({ error: 'Sorry, something bad happened!' }); console.error(err.stack? err.stack : err.toString()); stats.meter('http500').mark() - }) + }); /* * Connect graceful shutdown with sigint and uncaught exception diff --git a/src/node/hooks/express/importexport.js b/src/node/hooks/express/importexport.js index a62942cc0..fe97aa2f9 100644 --- a/src/node/hooks/express/importexport.js +++ b/src/node/hooks/express/importexport.js @@ -13,7 +13,7 @@ exports.expressCreateServer = function (hook_name, args, cb) { return; } - //if abiword is disabled, and this is a format we only support with abiword, output a message + // if abiword is disabled, and this is a format we only support with abiword, output a message if (settings.exportAvailable() == "no" && ["odt", "pdf", "doc"].indexOf(req.params.type) !== -1) { res.send("This export is not enabled at this Etherpad instance. Set the path to Abiword or SOffice in settings.json to enable this feature"); @@ -24,9 +24,8 @@ exports.expressCreateServer = function (hook_name, args, cb) { hasPadAccess(req, res, function() { console.log('req.params.pad', req.params.pad); - padManager.doesPadExists(req.params.pad, function(err, exists) - { - if(!exists) { + padManager.doesPadExists(req.params.pad, function(err, exists) { + if (!exists) { return next(); } @@ -35,12 +34,11 @@ exports.expressCreateServer = function (hook_name, args, cb) { }); }); - //handle import requests + // handle import requests args.app.post('/p/:pad/import', function(req, res, next) { hasPadAccess(req, res, function() { - padManager.doesPadExists(req.params.pad, function(err, exists) - { - if(!exists) { + padManager.doesPadExists(req.params.pad, function(err, exists) { + if (!exists) { return next(); } diff --git a/src/node/hooks/express/padreadonly.js b/src/node/hooks/express/padreadonly.js index bff8adf7b..77b197c7c 100644 --- a/src/node/hooks/express/padreadonly.js +++ b/src/node/hooks/express/padreadonly.js @@ -5,52 +5,45 @@ var hasPadAccess = require("../../padaccess"); var exporthtml = require("../../utils/ExportHtml"); exports.expressCreateServer = function (hook_name, args, cb) { - //serve read only pad - args.app.get('/ro/:id', function(req, res) - { + // serve read only pad + args.app.get('/ro/:id', function(req, res) { var html; var padId; async.series([ - //translate the read only pad to a padId - function(callback) - { - readOnlyManager.getPadId(req.params.id, function(err, _padId) - { + // translate the read only pad to a padId + function(callback) { + readOnlyManager.getPadId(req.params.id, function(err, _padId) { if(ERR(err, callback)) return; padId = _padId; - //we need that to tell hasPadAcess about the pad + // we need that to tell hasPadAcess about the pad req.params.pad = padId; callback(); }); }, - //render the html document - function(callback) - { - //return if the there is no padId - if(padId == null) - { + // render the html document + function(callback) { + // return if the there is no padId + if(padId == null) { callback("notfound"); return; } - hasPadAccess(req, res, function() - { - //render the html document - exporthtml.getPadHTMLDocument(padId, null, function(err, _html) - { + hasPadAccess(req, res, function() { + // render the html document + exporthtml.getPadHTMLDocument(padId, null, function(err, _html) { if(ERR(err, callback)) return; html = _html; callback(); }); }); } - ], function(err) - { - //throw any unexpected error + ], + function(err) { + // throw any unexpected error if(err && err != "notfound") ERR(err); diff --git a/src/node/hooks/express/padurlsanitize.js b/src/node/hooks/express/padurlsanitize.js index be3ffb1b4..a7fb9f33e 100644 --- a/src/node/hooks/express/padurlsanitize.js +++ b/src/node/hooks/express/padurlsanitize.js @@ -2,29 +2,26 @@ var padManager = require('../../db/PadManager'); var url = require('url'); exports.expressCreateServer = function (hook_name, args, cb) { - //redirects browser to the pad's sanitized url if needed. otherwise, renders the html + + // redirects browser to the pad's sanitized url if needed. otherwise, renders the html args.app.param('pad', function (req, res, next, padId) { - //ensure the padname is valid and the url doesn't end with a / - if(!padManager.isValidPadId(padId) || /\/$/.test(req.url)) - { + // ensure the padname is valid and the url doesn't end with a / + if (!padManager.isValidPadId(padId) || /\/$/.test(req.url)) { res.status(404).send('Such a padname is forbidden'); return; } padManager.sanitizePadId(padId, function(sanitizedPadId) { - //the pad id was sanitized, so we redirect to the sanitized version - if(sanitizedPadId != padId) - { + if (sanitizedPadId != padId) { + // the pad id was sanitized, so we redirect to the sanitized version var real_url = sanitizedPadId; real_url = encodeURIComponent(real_url); var query = url.parse(req.url).query; if ( query ) real_url += '?' + query; res.header('Location', real_url); res.status(302).send('You should be redirected to ' + real_url + ''); - } - //the pad id was fine, so just render it - else - { + } else { + // the pad id was fine, so just render it next(); } }); diff --git a/src/node/hooks/express/tests.js b/src/node/hooks/express/tests.js index d0dcc0cc6..b5c4ca56c 100644 --- a/src/node/hooks/express/tests.js +++ b/src/node/hooks/express/tests.js @@ -4,37 +4,35 @@ var path = require("path") , async = require("async"); exports.expressCreateServer = function (hook_name, args, cb) { - args.app.get('/tests/frontend/specs_list.js', function(req, res){ - + args.app.get('/tests/frontend/specs_list.js', function(req, res) { async.parallel({ - coreSpecs: function(callback){ + coreSpecs: function(callback) { exports.getCoreTests(callback); }, - pluginSpecs: function(callback){ + pluginSpecs: function(callback) { exports.getPluginTests(callback); } }, - function(err, results){ + function(err, results) { var files = results.coreSpecs; // push the core specs to a file object files = files.concat(results.pluginSpecs); // add the plugin Specs to the core specs console.debug("Sent browser the following test specs:", files.sort()); res.send("var specs_list = " + JSON.stringify(files.sort()) + ";\n"); }); - }); - // path.join seems to normalize by default, but we'll just be explicit var rootTestFolder = path.normalize(path.join(npm.root, "../tests/frontend/")); - var url2FilePath = function(url){ + var url2FilePath = function(url) { var subPath = url.substr("/tests/frontend".length); - if (subPath == ""){ + if (subPath == "") { subPath = "index.html" } subPath = subPath.split("?")[0]; var filePath = path.normalize(path.join(rootTestFolder, subPath)); + // make sure we jail the paths to the test folder, otherwise serve index if (filePath.indexOf(rootTestFolder) !== 0) { filePath = path.join(rootTestFolder, "index.html"); @@ -46,13 +44,13 @@ exports.expressCreateServer = function (hook_name, args, cb) { var specFilePath = url2FilePath(req.url); var specFileName = path.basename(specFilePath); - fs.readFile(specFilePath, function(err, content){ - if(err){ return res.send(500); } - + fs.readFile(specFilePath, function(err, content) { + if (err) { return res.send(500); } + content = "describe(" + JSON.stringify(specFileName) + ", function(){ " + content + " });"; res.send(content); - }); + }); }); args.app.get('/tests/frontend/*', function (req, res) { @@ -62,19 +60,21 @@ exports.expressCreateServer = function (hook_name, args, cb) { args.app.get('/tests/frontend', function (req, res) { res.redirect('/tests/frontend/'); - }); + }); } -exports.getPluginTests = function(callback){ +exports.getPluginTests = function(callback) { var pluginSpecs = []; var plugins = fs.readdirSync('node_modules'); - plugins.forEach(function(plugin){ - if(fs.existsSync("node_modules/"+plugin+"/static/tests/frontend/specs")){ // if plugins exists - var specFiles = fs.readdirSync("node_modules/"+plugin+"/static/tests/frontend/specs/"); - async.forEach(specFiles, function(spec){ // for each specFile push it to pluginSpecs - pluginSpecs.push("/static/plugins/"+plugin+"/static/tests/frontend/specs/" + spec); + plugins.forEach(function(plugin) { + if (fs.existsSync("node_modules/" + plugin + "/static/tests/frontend/specs")) { + // if plugins exists + var specFiles = fs.readdirSync("node_modules/" + plugin + "/static/tests/frontend/specs/"); + async.forEach(specFiles, function(spec) { + // for each specFile push it to pluginSpecs + pluginSpecs.push("/static/plugins/" + plugin + "/static/tests/frontend/specs/" + spec); }, - function(err){ + function(err) { // blow up if something bad happens! }); } @@ -82,10 +82,11 @@ exports.getPluginTests = function(callback){ callback(null, pluginSpecs); } -exports.getCoreTests = function(callback){ - fs.readdir('tests/frontend/specs', function(err, coreSpecs){ // get the core test specs - if(err){ return res.send(500); } +exports.getCoreTests = function(callback) { + // get the core test specs + fs.readdir('tests/frontend/specs', function(err, coreSpecs) { + if (err) { return res.send(500); } + callback(null, coreSpecs); }); } - diff --git a/src/node/padaccess.js b/src/node/padaccess.js index 1f2e8834b..c8c888cd0 100644 --- a/src/node/padaccess.js +++ b/src/node/padaccess.js @@ -1,16 +1,16 @@ var ERR = require("async-stacktrace"); var securityManager = require('./db/SecurityManager'); -//checks for padAccess +// checks for padAccess module.exports = function (req, res, callback) { securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password, function(err, accessObj) { - if(ERR(err, callback)) return; + if (ERR(err, callback)) return; - //there is access, continue - if(accessObj.accessStatus == "grant") { + if (accessObj.accessStatus == "grant") { + // there is access, continue callback(); - //no access } else { + // no access res.status(403).send("403 - Can't touch this"); } }); diff --git a/src/node/server.js b/src/node/server.js index 3db54284c..2ba1fa47d 100755 --- a/src/node/server.js +++ b/src/node/server.js @@ -1,7 +1,7 @@ #!/usr/bin/env node /** - * This module is started with bin/run.sh. It sets up a Express HTTP and a Socket.IO Server. - * Static file Requests are answered directly from this module, Socket.IO messages are passed + * This module is started with bin/run.sh. It sets up a Express HTTP and a Socket.IO Server. + * Static file Requests are answered directly from this module, Socket.IO messages are passed * to MessageHandler and minfied requests are passed to minified. */ @@ -46,8 +46,8 @@ NodeVersion.enforceMinNodeVersion('8.9.0'); */ var stats = require('./stats'); stats.gauge('memoryUsage', function() { - return process.memoryUsage().rss -}) + return process.memoryUsage().rss; +}); var settings , db @@ -59,8 +59,8 @@ async.waterfall([ // load npm function(callback) { npm.load({}, function(er) { - callback(er) - }) + callback(er); + }); }, // load everything @@ -73,14 +73,13 @@ async.waterfall([ callback(); }, - //initalize the database - function (callback) - { + // initalize the database + function (callback) { db.init(callback); }, function(callback) { - plugins.update(callback) + plugins.update(callback); }, function (callback) { @@ -93,9 +92,8 @@ async.waterfall([ callback(); }, - //initalize the http server - function (callback) - { + // initalize the http server + function (callback) { hooks.callAll("createServer", {}); callback(null); } diff --git a/src/node/utils/ExportTxt.js b/src/node/utils/ExportTxt.js index 8a40e800d..a42878df6 100644 --- a/src/node/utils/ExportTxt.js +++ b/src/node/utils/ExportTxt.js @@ -30,40 +30,32 @@ function getPadTXT(pad, revNum, callback) var atext = pad.atext; var html; async.waterfall([ + // fetch revision atext + function(callback) { + if (revNum != undefined) { + pad.getInternalRevisionAText(revNum, function(err, revisionAtext) { + if (ERR(err, callback)) return; - - function (callback) - { - if (revNum != undefined) - { - pad.getInternalRevisionAText(revNum, function (err, revisionAtext) - { - if(ERR(err, callback)) return; atext = revisionAtext; callback(); }); - } - else - { + } else { callback(null); } }, // convert atext to html - - - function (callback) - { - html = getTXTFromAtext(pad, atext); // only this line is different to the HTML function + function(callback) { + // only this line is different to the HTML function + html = getTXTFromAtext(pad, atext); callback(null); }], + // run final callback + function(err) { + if (ERR(err, callback)) return; - - function (err) - { - if(ERR(err, callback)) return; callback(null, html); }); } @@ -80,17 +72,14 @@ function getTXTFromAtext(pad, atext, authorColors) var anumMap = {}; var css = ""; - props.forEach(function (propName, i) - { + props.forEach(function(propName, i) { var propTrueNum = apool.putAttrib([propName, true], true); - if (propTrueNum >= 0) - { + if (propTrueNum >= 0) { anumMap[propTrueNum] = i; } }); - function getLineTXT(text, attribs) - { + function getLineTXT(text, attribs) { var propVals = [false, false, false]; var ENTER = 1; var STAY = 2; @@ -106,94 +95,77 @@ function getTXTFromAtext(pad, atext, authorColors) var idx = 0; - function processNextChars(numChars) - { - if (numChars <= 0) - { + function processNextChars(numChars) { + if (numChars <= 0) { return; } var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars)); idx += numChars; - while (iter.hasNext()) - { + while (iter.hasNext()) { var o = iter.next(); var propChanged = false; - Changeset.eachAttribNumber(o.attribs, function (a) - { - if (a in anumMap) - { + + Changeset.eachAttribNumber(o.attribs, function(a) { + if (a in anumMap) { var i = anumMap[a]; // i = 0 => bold, etc. - if (!propVals[i]) - { + + if (!propVals[i]) { propVals[i] = ENTER; propChanged = true; - } - else - { + } else { propVals[i] = STAY; } } }); - for (var i = 0; i < propVals.length; i++) - { - if (propVals[i] === true) - { + + for (var i = 0; i < propVals.length; i++) { + if (propVals[i] === true) { propVals[i] = LEAVE; propChanged = true; - } - else if (propVals[i] === STAY) - { - propVals[i] = true; // set it back + } else if (propVals[i] === STAY) { + // set it back + propVals[i] = true; } } + // now each member of propVal is in {false,LEAVE,ENTER,true} // according to what happens at start of span - if (propChanged) - { + if (propChanged) { // leaving bold (e.g.) also leaves italics, etc. var left = false; - for (var i = 0; i < propVals.length; i++) - { + + for (var i = 0; i < propVals.length; i++) { var v = propVals[i]; - if (!left) - { - if (v === LEAVE) - { + + if (!left) { + if (v === LEAVE) { left = true; } - } - else - { - if (v === true) - { - propVals[i] = STAY; // tag will be closed and re-opened + } else { + if (v === true) { + // tag will be closed and re-opened + propVals[i] = STAY; } } } var tags2close = []; - for (var i = propVals.length - 1; i >= 0; i--) - { - if (propVals[i] === LEAVE) - { + for (var i = propVals.length - 1; i >= 0; i--) { + if (propVals[i] === LEAVE) { //emitCloseTag(i); tags2close.push(i); propVals[i] = false; - } - else if (propVals[i] === STAY) - { + } else if (propVals[i] === STAY) { //emitCloseTag(i); tags2close.push(i); } } - for (var i = 0; i < propVals.length; i++) - { - if (propVals[i] === ENTER || propVals[i] === STAY) - { + for (var i = 0; i < propVals.length; i++) { + if (propVals[i] === ENTER || propVals[i] === STAY) { propVals[i] = true; } } @@ -201,9 +173,9 @@ function getTXTFromAtext(pad, atext, authorColors) } // end if (propChanged) var chars = o.chars; - if (o.lines) - { - chars--; // exclude newline at end of line, if present + if (o.lines) { + // exclude newline at end of line, if present + chars--; } var s = taker.take(chars); @@ -220,19 +192,19 @@ function getTXTFromAtext(pad, atext, authorColors) } // end iteration over spans in line var tags2close = []; - for (var i = propVals.length - 1; i >= 0; i--) - { - if (propVals[i]) - { + for (var i = propVals.length - 1; i >= 0; i--) { + if (propVals[i]) { tags2close.push(i); propVals[i] = false; } } } // end processNextChars + processNextChars(text.length - idx); return(assem.toString()); } // end getLineHTML + var pieces = [css]; // Need to deal with constraints imposed on HTML lists; can @@ -242,41 +214,44 @@ function getTXTFromAtext(pad, atext, authorColors) // so we want to do something reasonable there. We also // want to deal gracefully with blank lines. // => keeps track of the parents level of indentation - for (var i = 0; i < textLines.length; i++) - { + for (var i = 0; i < textLines.length; i++) { var line = _analyzeLine(textLines[i], attribLines[i], apool); var lineContent = getLineTXT(line.text, line.aline); - if(line.listTypeName == "bullet"){ + + if (line.listTypeName == "bullet") { lineContent = "* " + lineContent; // add a bullet } - if(line.listLevel > 0){ - for (var j = line.listLevel - 1; j >= 0; j--){ + + if (line.listLevel > 0) { + for (var j = line.listLevel - 1; j >= 0; j--) { pieces.push('\t'); } - if(line.listTypeName == "number"){ + + if (line.listTypeName == "number") { pieces.push(line.listLevel + ". "); // This is bad because it doesn't truly reflect what the user // sees because browsers do magic on nested
  1. s } + pieces.push(lineContent, '\n'); - }else{ + } else { pieces.push(lineContent, '\n'); } } return pieces.join(''); } + exports.getTXTFromAtext = getTXTFromAtext; -exports.getPadTXTDocument = function (padId, revNum, callback) +exports.getPadTXTDocument = function(padId, revNum, callback) { - padManager.getPad(padId, function (err, pad) - { - if(ERR(err, callback)) return; + padManager.getPad(padId, function(err, pad) { + if (ERR(err, callback)) return; + + getPadTXT(pad, revNum, function(err, html) { + if (ERR(err, callback)) return; - getPadTXT(pad, revNum, function (err, html) - { - if(ERR(err, callback)) return; callback(null, html); }); }); diff --git a/src/node/utils/ImportEtherpad.js b/src/node/utils/ImportEtherpad.js index bf1129cb9..1ff8b9b04 100644 --- a/src/node/utils/ImportEtherpad.js +++ b/src/node/utils/ImportEtherpad.js @@ -18,57 +18,56 @@ var log4js = require('log4js'); var async = require("async"); var db = require("../db/DB").db; -exports.setPadRaw = function(padId, records, callback){ +exports.setPadRaw = function(padId, records, callback) +{ records = JSON.parse(records); - async.eachSeries(Object.keys(records), function(key, cb){ - var value = records[key] + async.eachSeries(Object.keys(records), function(key, cb) { + var value = records[key]; - if(!value){ + if (!value) { return setImmediate(cb); } - // Author data - if(value.padIDs){ - // rewrite author pad ids + if (value.padIDs) { + // Author data - rewrite author pad ids value.padIDs[padId] = 1; var newKey = key; // Does this author already exist? - db.get(key, function(err, author){ - if(author){ - // Yes, add the padID to the author.. - if( Object.prototype.toString.call(author) === '[object Array]'){ + db.get(key, function(err, author) { + if (author) { + // Yes, add the padID to the author + if (Object.prototype.toString.call(author) === '[object Array]') { author.padIDs.push(padId); } value = author; - }else{ + } else { // No, create a new array with the author info in value.padIDs = [padId]; } }); - - // Not author data, probably pad data - }else{ - // we can split it to look to see if its pad data + } else { + // Not author data, probably pad data + // we can split it to look to see if it's pad data var oldPadId = key.split(":"); - // we know its pad data.. - if(oldPadId[0] === "pad"){ - + // we know it's pad data + if (oldPadId[0] === "pad") { // so set the new pad id for the author oldPadId[1] = padId; - + // and create the value var newKey = oldPadId.join(":"); // create the new key } - } + // Write the value to the server db.set(newKey, value); setImmediate(cb); - }, function(){ + }, + function() { callback(null, true); }); } diff --git a/src/node/utils/ImportHtml.js b/src/node/utils/ImportHtml.js index d71e27201..04037eab5 100644 --- a/src/node/utils/ImportHtml.js +++ b/src/node/utils/ImportHtml.js @@ -36,19 +36,22 @@ function setPadHTML(pad, html, callback) // Convert a dom tree into a list of lines and attribute liens // using the content collector object var cc = contentcollector.makeContentCollector(true, null, pad.pool); - try{ // we use a try here because if the HTML is bad it will blow up + try { + // we use a try here because if the HTML is bad it will blow up cc.collectContent(doc); - }catch(e){ + } catch(e) { apiLogger.warn("HTML was not properly formed", e); - return callback(e); // We don't process the HTML because it was bad.. + + // don't process the HTML because it was bad + return callback(e); } var result = cc.finish(); apiLogger.debug('Lines:'); + var i; - for (i = 0; i < result.lines.length; i += 1) - { + for (i = 0; i < result.lines.length; i += 1) { apiLogger.debug('Line ' + (i + 1) + ' text: ' + result.lines[i]); apiLogger.debug('Line ' + (i + 1) + ' attributes: ' + result.lineAttribs[i]); } @@ -59,18 +62,15 @@ function setPadHTML(pad, html, callback) apiLogger.debug(newText); var newAttribs = result.lineAttribs.join('|1+1') + '|1+1'; - function eachAttribRun(attribs, func /*(startInNewText, endInNewText, attribs)*/ ) - { + function eachAttribRun(attribs, func /*(startInNewText, endInNewText, attribs)*/ ) { var attribsIter = Changeset.opIterator(attribs); var textIndex = 0; var newTextStart = 0; var newTextEnd = newText.length; - while (attribsIter.hasNext()) - { + while (attribsIter.hasNext()) { var op = attribsIter.next(); var nextIndex = textIndex + op.chars; - if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) - { + if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) { func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs); } textIndex = nextIndex; @@ -81,13 +81,13 @@ function setPadHTML(pad, html, callback) var builder = Changeset.builder(1); // assemble each line into the builder - eachAttribRun(newAttribs, function(start, end, attribs) - { + eachAttribRun(newAttribs, function(start, end, attribs) { builder.insert(newText.substring(start, end), attribs); }); // the changeset is ready! var theChangeset = builder.toString(); + apiLogger.debug('The changeset: ' + theChangeset); pad.setText("\n"); pad.appendRevision(theChangeset); diff --git a/src/node/utils/padDiff.js b/src/node/utils/padDiff.js index 24d5bb0c2..33938b19c 100644 --- a/src/node/utils/padDiff.js +++ b/src/node/utils/padDiff.js @@ -1,336 +1,348 @@ var Changeset = require("../../static/js/Changeset"); var async = require("async"); var exportHtml = require('./ExportHtml'); - -function PadDiff (pad, fromRev, toRev){ - //check parameters - if(!pad || !pad.id || !pad.atext || !pad.pool) - { + +function PadDiff (pad, fromRev, toRev) { + // check parameters + if (!pad || !pad.id || !pad.atext || !pad.pool) { throw new Error('Invalid pad'); } - + var range = pad.getValidRevisionRange(fromRev, toRev); - if(!range) { throw new Error('Invalid revision range.' + + if (!range) { + throw new Error('Invalid revision range.' + ' startRev: ' + fromRev + - ' endRev: ' + toRev); } - + ' endRev: ' + toRev); + } + this._pad = pad; this._fromRev = range.startRev; this._toRev = range.endRev; this._html = null; this._authors = []; } - -PadDiff.prototype._isClearAuthorship = function(changeset){ - //unpack + +PadDiff.prototype._isClearAuthorship = function(changeset) { + // unpack var unpacked = Changeset.unpack(changeset); - - //check if there is nothing in the charBank - if(unpacked.charBank !== "") + + // check if there is nothing in the charBank + if (unpacked.charBank !== "") { return false; - - //check if oldLength == newLength - if(unpacked.oldLen !== unpacked.newLen) + } + + // check if oldLength == newLength + if (unpacked.oldLen !== unpacked.newLen) { return false; - - //lets iterator over the operators + } + + // lets iterator over the operators var iterator = Changeset.opIterator(unpacked.ops); - - //get the first operator, this should be a clear operator + + // get the first operator, this should be a clear operator var clearOperator = iterator.next(); - - //check if there is only one operator - if(iterator.hasNext() === true) + + // check if there is only one operator + if (iterator.hasNext() === true) { return false; - - //check if this operator doesn't change text - if(clearOperator.opcode !== "=") + } + + // check if this operator doesn't change text + if (clearOperator.opcode !== "=") { return false; - - //check that this operator applys to the complete text - //if the text ends with a new line, its exactly one character less, else it has the same length - if(clearOperator.chars !== unpacked.oldLen-1 && clearOperator.chars !== unpacked.oldLen) + } + + // check that this operator applys to the complete text + // if the text ends with a new line, its exactly one character less, else it has the same length + if (clearOperator.chars !== unpacked.oldLen-1 && clearOperator.chars !== unpacked.oldLen) { return false; - + } + var attributes = []; - Changeset.eachAttribNumber(changeset, function(attrNum){ + Changeset.eachAttribNumber(changeset, function(attrNum) { attributes.push(attrNum); }); - - //check that this changeset uses only one attribute - if(attributes.length !== 1) + + // check that this changeset uses only one attribute + if (attributes.length !== 1) { return false; - + } + var appliedAttribute = this._pad.pool.getAttrib(attributes[0]); - - //check if the applied attribute is an anonymous author attribute - if(appliedAttribute[0] !== "author" || appliedAttribute[1] !== "") + + // check if the applied attribute is an anonymous author attribute + if (appliedAttribute[0] !== "author" || appliedAttribute[1] !== "") { return false; - + } + return true; }; - -PadDiff.prototype._createClearAuthorship = function(rev, callback){ + +PadDiff.prototype._createClearAuthorship = function(rev, callback) { var self = this; - this._pad.getInternalRevisionAText(rev, function(err, atext){ - if(err){ - return callback(err); - } - - //build clearAuthorship changeset + this._pad.getInternalRevisionAText(rev, function(err, atext) { + if (err) { + return callback(err); + } + + // build clearAuthorship changeset var builder = Changeset.builder(atext.text.length); builder.keepText(atext.text, [['author','']], self._pad.pool); var changeset = builder.toString(); - + callback(null, changeset); }); }; - -PadDiff.prototype._createClearStartAtext = function(rev, callback){ + +PadDiff.prototype._createClearStartAtext = function(rev, callback) { var self = this; - - //get the atext of this revision - this._pad.getInternalRevisionAText(rev, function(err, atext){ - if(err){ - return callback(err); - } - - //create the clearAuthorship changeset - self._createClearAuthorship(rev, function(err, changeset){ - if(err){ - return callback(err); - } - - try { - //apply the clearAuthorship changeset - var newAText = Changeset.applyToAText(changeset, atext, self._pad.pool); - } catch(err) { - return callback(err) - } - - callback(null, newAText); - }); - }); -}; - -PadDiff.prototype._getChangesetsInBulk = function(startRev, count, callback) { - var self = this; - - //find out which revisions we need - var revisions = []; - for(var i=startRev;i<(startRev+count) && i<=this._pad.head;i++){ - revisions.push(i); - } - - var changesets = [], authors = []; - - //get all needed revisions - async.forEach(revisions, function(rev, callback){ - self._pad.getRevision(rev, function(err, revision){ - if(err){ + + // get the atext of this revision + this._pad.getInternalRevisionAText(rev, function(err, atext) { + if (err) { + return callback(err); + } + + // create the clearAuthorship changeset + self._createClearAuthorship(rev, function(err, changeset) { + if (err) { return callback(err); } - + + try { + // apply the clearAuthorship changeset + var newAText = Changeset.applyToAText(changeset, atext, self._pad.pool); + } catch(err) { + return callback(err) + } + + callback(null, newAText); + }); + }); +}; + +PadDiff.prototype._getChangesetsInBulk = function(startRev, count, callback) { + var self = this; + + // find out which revisions we need + var revisions = []; + for (var i = startRev; i < (startRev + count) && i <= this._pad.head; i++) { + revisions.push(i); + } + + var changesets = [], authors = []; + + // get all needed revisions + async.forEach(revisions, function(rev, callback) { + self._pad.getRevision(rev, function(err, revision) { + if (err) { + return callback(err); + } + var arrayNum = rev-startRev; - + changesets[arrayNum] = revision.changeset; authors[arrayNum] = revision.meta.author; - + callback(); }); - }, function(err){ + }, + function(err) { callback(err, changesets, authors); }); }; - + PadDiff.prototype._addAuthors = function(authors) { var self = this; - //add to array if not in the array - authors.forEach(function(author){ - if(self._authors.indexOf(author) == -1){ + + // add to array if not in the array + authors.forEach(function(author) { + if (self._authors.indexOf(author) == -1) { self._authors.push(author); } }); }; - + PadDiff.prototype._createDiffAtext = function(callback) { var self = this; var bulkSize = 100; - - //get the cleaned startAText - self._createClearStartAtext(self._fromRev, function(err, atext){ - if(err) { return callback(err); } - + + // get the cleaned startAText + self._createClearStartAtext(self._fromRev, function(err, atext) { + if (err) { return callback(err); } + var superChangeset = null; - + var rev = self._fromRev + 1; - - //async while loop + + // async while loop async.whilst( - //loop condition + // loop condition function () { return rev <= self._toRev; }, - - //loop body + + // loop body function (callback) { - //get the bulk - self._getChangesetsInBulk(rev,bulkSize,function(err, changesets, authors){ + // get the bulk + self._getChangesetsInBulk(rev,bulkSize,function(err, changesets, authors) { var addedAuthors = []; - - //run trough all changesets - for(var i=0;i 0) { if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) { curLine++; @@ -384,22 +396,25 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) { curLineNextOp.chars = 0; curLineOpIter = Changeset.opIterator(alines_get(curLine)); } + if (!curLineNextOp.chars) { curLineOpIter.next(curLineNextOp); } + var charsToUse = Math.min(numChars, curLineNextOp.chars); + func(charsToUse, curLineNextOp.attribs, charsToUse == curLineNextOp.chars && curLineNextOp.lines > 0); numChars -= charsToUse; curLineNextOp.chars -= charsToUse; curChar += charsToUse; } - + if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) { curLine++; curChar = 0; } } - + function skip(N, L) { if (L) { curLine += L; @@ -412,27 +427,29 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) { } } } - + function nextText(numChars) { var len = 0; var assem = Changeset.stringAssembler(); var firstString = lines_get(curLine).substring(curChar); len += firstString.length; assem.append(firstString); - + var lineNum = curLine + 1; + while (len < numChars) { var nextString = lines_get(lineNum); len += nextString.length; assem.append(nextString); lineNum++; } - + return assem.toString().substring(0, numChars); } - + function cachedStrFunc(func) { var cache = {}; + return function (s) { if (!cache[s]) { cache[s] = func(s); @@ -440,57 +457,59 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) { return cache[s]; }; } - + var attribKeys = []; var attribValues = []; - - //iterate over all operators of this changeset + + // iterate over all operators of this changeset while (csIter.hasNext()) { var csOp = csIter.next(); - - if (csOp.opcode == '=') { + + if (csOp.opcode == '=') { var textBank = nextText(csOp.chars); - + // decide if this equal operator is an attribution change or not. We can see this by checkinf if attribs is set. // If the text this operator applies to is only a star, than this is a false positive and should be ignored if (csOp.attribs && textBank != "*") { var deletedAttrib = apool.putAttrib(["removed", true]); var authorAttrib = apool.putAttrib(["author", ""]); - + attribKeys.length = 0; attribValues.length = 0; Changeset.eachAttribNumber(csOp.attribs, function (n) { attribKeys.push(apool.getAttribKey(n)); attribValues.push(apool.getAttribValue(n)); - - if(apool.getAttribKey(n) === "author"){ + + if (apool.getAttribKey(n) === "author") { authorAttrib = n; } }); - + var undoBackToAttribs = cachedStrFunc(function (attribs) { var backAttribs = []; for (var i = 0; i < attribKeys.length; i++) { var appliedKey = attribKeys[i]; var appliedValue = attribValues[i]; var oldValue = Changeset.attribsAttributeValue(attribs, appliedKey, apool); + if (appliedValue != oldValue) { backAttribs.push([appliedKey, oldValue]); } } + return Changeset.makeAttribsString('=', backAttribs, apool); }); - + var oldAttribsAddition = "*" + Changeset.numToString(deletedAttrib) + "*" + Changeset.numToString(authorAttrib); - + var textLeftToProcess = textBank; - - while(textLeftToProcess.length > 0){ - //process till the next line break or process only one line break + + while(textLeftToProcess.length > 0) { + // process till the next line break or process only one line break var lengthToProcess = textLeftToProcess.indexOf("\n"); var lineBreak = false; - switch(lengthToProcess){ - case -1: + switch(lengthToProcess) { + case -1: lengthToProcess=textLeftToProcess.length; break; case 0: @@ -498,27 +517,28 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) { lengthToProcess=1; break; } - - //get the text we want to procceed in this step + + // get the text we want to procceed in this step var processText = textLeftToProcess.substr(0, lengthToProcess); + textLeftToProcess = textLeftToProcess.substr(lengthToProcess); - - if(lineBreak){ - builder.keep(1, 1); //just skip linebreaks, don't do a insert + keep for a linebreak - - //consume the attributes of this linebreak - consumeAttribRuns(1, function(){}); + + if (lineBreak) { + builder.keep(1, 1); // just skip linebreaks, don't do a insert + keep for a linebreak + + // consume the attributes of this linebreak + consumeAttribRuns(1, function() {}); } else { - //add the old text via an insert, but add a deletion attribute + the author attribute of the author who deleted it + // add the old text via an insert, but add a deletion attribute + the author attribute of the author who deleted it var textBankIndex = 0; consumeAttribRuns(lengthToProcess, function (len, attribs, endsLine) { - //get the old attributes back + // get the old attributes back var attribs = (undoBackToAttribs(attribs) || "") + oldAttribsAddition; - + builder.insert(processText.substr(textBankIndex, len), attribs); textBankIndex += len; }); - + builder.keep(lengthToProcess, 0); } } @@ -531,16 +551,16 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) { } else if (csOp.opcode == '-') { var textBank = nextText(csOp.chars); var textBankIndex = 0; - + consumeAttribRuns(csOp.chars, function (len, attribs, endsLine) { builder.insert(textBank.substr(textBankIndex, len), attribs + csOp.attribs); textBankIndex += len; }); } } - + return Changeset.checkRep(builder.toString()); }; - -//export the constructor + +// export the constructor module.exports = PadDiff; diff --git a/src/static/js/pluginfw/installer.js b/src/static/js/pluginfw/installer.js index cd2ed3305..fbd0e1d97 100644 --- a/src/static/js/pluginfw/installer.js +++ b/src/static/js/pluginfw/installer.js @@ -4,12 +4,14 @@ var npm = require("npm"); var request = require("request"); var npmIsLoaded = false; -var withNpm = function (npmfn) { - if(npmIsLoaded) return npmfn(); - npm.load({}, function (er) { +var withNpm = function(npmfn) { + if (npmIsLoaded) return npmfn(); + + npm.load({}, function(er) { if (er) return npmfn(er); + npmIsLoaded = true; - npm.on("log", function (message) { + npm.on("log", function(message) { console.log('npm: ',message) }); npmfn(); @@ -17,26 +19,33 @@ var withNpm = function (npmfn) { } var tasks = 0 + function wrapTaskCb(cb) { - tasks++ + tasks++; + return function() { cb && cb.apply(this, arguments); tasks--; - if(tasks == 0) onAllTasksFinished(); + if (tasks == 0) onAllTasksFinished(); } } + function onAllTasksFinished() { - hooks.aCallAll("restartServer", {}, function () {}); + hooks.aCallAll("restartServer", {}, function() {}); } exports.uninstall = function(plugin_name, cb) { cb = wrapTaskCb(cb); - withNpm(function (er) { + + withNpm(function(er) { if (er) return cb && cb(er); - npm.commands.uninstall([plugin_name], function (er) { + + npm.commands.uninstall([plugin_name], function(er) { if (er) return cb && cb(er); - hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name}, function (er, data) { + + hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name}, function(er, data) { if (er) return cb(er); + plugins.update(cb); }); }); @@ -44,13 +53,17 @@ exports.uninstall = function(plugin_name, cb) { }; exports.install = function(plugin_name, cb) { - cb = wrapTaskCb(cb) - withNpm(function (er) { + cb = wrapTaskCb(cb); + + withNpm(function(er) { if (er) return cb && cb(er); - npm.commands.install([plugin_name], function (er) { + + npm.commands.install([plugin_name], function(er) { if (er) return cb && cb(er); - hooks.aCallAll("pluginInstall", {plugin_name: plugin_name}, function (er, data) { + + hooks.aCallAll("pluginInstall", {plugin_name: plugin_name}, function(er, data) { if (er) return cb(er); + plugins.update(cb); }); }); @@ -63,41 +76,53 @@ var cacheTimestamp = 0; exports.getAvailablePlugins = function(maxCacheAge, cb) { request("https://static.etherpad.org/plugins.json", function(er, response, plugins){ if (er) return cb && cb(er); - if(exports.availablePlugins && maxCacheAge && Math.round(+new Date/1000)-cacheTimestamp <= maxCacheAge) { - return cb && cb(null, exports.availablePlugins) + + if (exports.availablePlugins && maxCacheAge && Math.round(+ new Date / 1000) - cacheTimestamp <= maxCacheAge) { + return cb && cb(null, exports.availablePlugins); } + try { plugins = JSON.parse(plugins); } catch (err) { console.error('error parsing plugins.json:', err); plugins = []; } + exports.availablePlugins = plugins; - cacheTimestamp = Math.round(+new Date/1000); - cb && cb(null, plugins) + cacheTimestamp = Math.round(+ new Date / 1000); + + cb && cb(null, plugins); }); }; exports.search = function(searchTerm, maxCacheAge, cb) { exports.getAvailablePlugins(maxCacheAge, function(er, results) { - if(er) return cb && cb(er); + if (er) return cb && cb(er); + var res = {}; - if (searchTerm) + + if (searchTerm) { searchTerm = searchTerm.toLowerCase(); - for (var pluginName in results) { // for every available plugin + } + + for (var pluginName in results) { + // for every available plugin if (pluginName.indexOf(plugins.prefix) != 0) continue; // TODO: Also search in keywords here! - if(searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm) + if (searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm) && (typeof results[pluginName].description != "undefined" && !~results[pluginName].description.toLowerCase().indexOf(searchTerm) ) - ){ - if(typeof results[pluginName].description === "undefined"){ + ) { + if (typeof results[pluginName].description === "undefined") { console.debug('plugin without Description: %s', results[pluginName].name); } + continue; } + res[pluginName] = results[pluginName]; } - cb && cb(null, res) - }) + + cb && cb(null, res); + }); }; diff --git a/src/static/js/pluginfw/plugins.js b/src/static/js/pluginfw/plugins.js index 7cb4b1bdc..38d6b68c0 100644 --- a/src/static/js/pluginfw/plugins.js +++ b/src/static/js/pluginfw/plugins.js @@ -55,6 +55,7 @@ exports.formatHooks = function (hook_set_name) { exports.callInit = function (cb) { var hooks = require("./hooks"); + async.map( Object.keys(exports.plugins), function (plugin_name, cb) { @@ -83,6 +84,7 @@ exports.update = function (cb) { exports.getPackages(function (er, packages) { var parts = []; var plugins = {}; + // Load plugin metadata ep.json async.forEach( Object.keys(packages), @@ -106,6 +108,7 @@ exports.getPackages = function (cb) { var dir = path.resolve(npm.dir, '..'); readInstalled(dir, function (er, data) { if (er) cb(er, null); + var packages = {}; function flatten(deps) { _.chain(deps).keys().each(function (name) { @@ -116,12 +119,12 @@ exports.getPackages = function (cb) { delete packages[name].dependencies; delete packages[name].parent; } - + // I don't think we need recursion //if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies); }); } - + var tmp = {}; tmp[data.name] = data; flatten(tmp[data.name].dependencies); From e84179831413cd0feee6e1ba0ae80e9826754a5a Mon Sep 17 00:00:00 2001 From: muxator Date: Fri, 15 Feb 2019 22:52:53 +0100 Subject: [PATCH 02/58] prepare to async: typos in error messages This change extracts the grammar correction performed on the async branch, anticipating them in a single commit. It cannot be folded with the previous one, as it is not purely cosmetic. --- src/node/db/API.js | 14 +++++++------- src/node/db/GroupManager.js | 2 +- src/node/db/SessionManager.js | 2 +- src/node/handler/ImportHandler.js | 2 +- src/node/handler/PadMessageHandler.js | 2 +- src/node/handler/SocketIORouter.js | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/node/db/API.js b/src/node/db/API.js index edae1ba32..a2c0d60b7 100644 --- a/src/node/db/API.js +++ b/src/node/db/API.js @@ -204,7 +204,7 @@ exports.getText = function(padID, rev, callback) // ensure this is not a negative number if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negativ number", "apierror")); + callback(new customError("rev is a negative number", "apierror")); return; } @@ -257,7 +257,7 @@ exports.setText = function(padID, text, callback) { // text is required if (typeof text != "string") { - callback(new customError("text is no string", "apierror")); + callback(new customError("text is not a string", "apierror")); return; } @@ -286,7 +286,7 @@ exports.appendText = function(padID, text, callback) { // text is required if (typeof text != "string") { - callback(new customError("text is no string", "apierror")); + callback(new customError("text is not a string", "apierror")); return; } @@ -376,7 +376,7 @@ exports.setHTML = function(padID, html, callback) { // html is required if (typeof html != "string") { - callback(new customError("html is no string", "apierror")); + callback(new customError("html is not a string", "apierror")); return; } @@ -472,7 +472,7 @@ exports.appendChatMessage = function(padID, text, authorID, time, callback) { // text is required if (typeof text != "string") { - callback(new customError("text is no string", "apierror")); + callback(new customError("text is not a string", "apierror")); return; } @@ -569,7 +569,7 @@ exports.saveRevision = function(padID, rev, callback) // ensure this is not a negative number if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negativ number", "apierror")); + callback(new customError("rev is a negative number", "apierror")); return; } @@ -694,7 +694,7 @@ exports.restoreRevision = function(padID, rev, callback) // ensure this is not a negative number if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negativ number", "apierror")); + callback(new customError("rev is a negative number", "apierror")); return; } diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index 5f8183ea9..f384f70aa 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -199,7 +199,7 @@ exports.createGroupIfNotExistsFor = function(groupMapper, callback) { // ensure mapper is optional if (typeof groupMapper != "string") { - callback(new customError("groupMapper is no string", "apierror")); + callback(new customError("groupMapper is not a string", "apierror")); return; } diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index 6b580c033..e039ee459 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -95,7 +95,7 @@ exports.createSession = function(groupID, authorID, validUntil, callback) // ensure this is not a negative number if (validUntil < 0) { - callback(new customError("validUntil is a negativ number", "apierror")); + callback(new customError("validUntil is a negative number", "apierror")); return; } diff --git a/src/node/handler/ImportHandler.js b/src/node/handler/ImportHandler.js index dae71bf6d..ec07bf6e5 100644 --- a/src/node/handler/ImportHandler.js +++ b/src/node/handler/ImportHandler.js @@ -151,7 +151,7 @@ exports.doImport = function(req, res, padId) padManager.getPad(padId, function(err, _pad) { var headCount = _pad.head; if (headCount >= 10) { - apiLogger.warn("Direct database Import attempt of a pad that already has content, we wont be doing this"); + apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this"); return callback("padHasData"); } diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 5c20ed1b6..e32d0e61e 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -457,7 +457,7 @@ function handleGetChatMessages(client, message) var count = end - start; if (count < 0 || count > 100) { - messageLogger.warn("Dropped message, GetChatMessages Message, client requested invalid amout of messages!"); + messageLogger.warn("Dropped message, GetChatMessages Message, client requested invalid amount of messages!"); return; } diff --git a/src/node/handler/SocketIORouter.js b/src/node/handler/SocketIORouter.js index c1e78a910..a94080b9d 100644 --- a/src/node/handler/SocketIORouter.js +++ b/src/node/handler/SocketIORouter.js @@ -116,7 +116,7 @@ exports.setSocketIO = function(_socket) { } } else { // drop message - messageLogger.warn("Dropped message cause of bad permissions:" + stringifyWithoutPassword(message)); + messageLogger.warn("Dropped message because of bad permissions:" + stringifyWithoutPassword(message)); } } }); From 11453d544c2a47b562004e383106599226c51e87 Mon Sep 17 00:00:00 2001 From: muxator Date: Fri, 1 Mar 2019 09:43:41 +0100 Subject: [PATCH 03/58] prepare to async: stricter checks This change is in preparation of the future async refactoring by Ray. It tries to extract as many changes in boolean conditions as possible, in order to make more evident identifying eventual logic bugs in the future work. This proved already useful in at least one case. BEWARE: this commit exposes an incoherency in the DB API, in which, depending on the driver used, some functions can return null or undefined. This condition will be externally fixed by the final commit in this series ("db/DB.js: prevent DB layer from returning undefined"). Until that commit, the code base may have some bugs. --- src/node/db/API.js | 32 +++++++++++++++--------------- src/node/db/AuthorManager.js | 12 +++++------ src/node/db/GroupManager.js | 8 ++++---- src/node/db/SecurityManager.js | 14 ++++++------- src/node/handler/APIHandler.js | 2 +- src/node/handler/SocketIORouter.js | 2 +- src/node/padaccess.js | 2 +- 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/node/db/API.js b/src/node/db/API.js index a2c0d60b7..b6acefcd4 100644 --- a/src/node/db/API.js +++ b/src/node/db/API.js @@ -192,7 +192,7 @@ Example returns: exports.getText = function(padID, rev, callback) { // check if rev is a number - if (rev !== undefined && typeof rev != "number") { + if (rev !== undefined && typeof rev !== "number") { // try to parse the number if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); @@ -256,7 +256,7 @@ Example returns: exports.setText = function(padID, text, callback) { // text is required - if (typeof text != "string") { + if (typeof text !== "string") { callback(new customError("text is not a string", "apierror")); return; } @@ -285,7 +285,7 @@ Example returns: exports.appendText = function(padID, text, callback) { // text is required - if (typeof text != "string") { + if (typeof text !== "string") { callback(new customError("text is not a string", "apierror")); return; } @@ -311,7 +311,7 @@ Example returns: */ exports.getHTML = function(padID, rev, callback) { - if (rev !== undefined && typeof rev != "number") { + if (rev !== undefined && typeof rev !== "number") { if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); return; @@ -375,7 +375,7 @@ Example returns: exports.setHTML = function(padID, html, callback) { // html is required - if (typeof html != "string") { + if (typeof html !== "string") { callback(new customError("html is not a string", "apierror")); return; } @@ -471,7 +471,7 @@ Example returns: exports.appendChatMessage = function(padID, text, authorID, time, callback) { // text is required - if (typeof text != "string") { + if (typeof text !== "string") { callback(new customError("text is not a string", "apierror")); return; } @@ -557,7 +557,7 @@ Example returns: exports.saveRevision = function(padID, rev, callback) { // check if rev is a number - if (rev !== undefined && typeof rev != "number") { + if (rev !== undefined && typeof rev !== "number") { // try to parse the number if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); @@ -636,7 +636,7 @@ exports.createPad = function(padID, text, callback) { if (padID) { // ensure there is no $ in the padID - if (padID.indexOf("$") != -1) { + if (padID.indexOf("$") !== -1) { callback(new customError("createPad can't create group pads", "apierror")); return; } @@ -682,7 +682,7 @@ exports.deletePad = function(padID, callback) exports.restoreRevision = function(padID, rev, callback) { // check if rev is a number - if (rev !== undefined && typeof rev != "number") { + if (rev !== undefined && typeof rev !== "number") { // try to parse the number if (isNaN(parseInt(rev))) { callback(new customError("rev is not a number", "apierror")); @@ -838,7 +838,7 @@ exports.getPadID = function(roID, callback) readOnlyManager.getPadId(roID, function(err, retrievedPadID) { if (ERR(err, callback)) return; - if (retrievedPadID == null) { + if (retrievedPadID === null) { callback(new customError("padID does not exist", "apierror")); return; } @@ -858,7 +858,7 @@ Example returns: exports.setPublicStatus = function(padID, publicStatus, callback) { // ensure this is a group pad - if (padID && padID.indexOf("$") == -1) { + if (padID && padID.indexOf("$") === -1) { callback(new customError("You can only get/set the publicStatus of pads that belong to a group", "apierror")); return; } @@ -868,7 +868,7 @@ exports.setPublicStatus = function(padID, publicStatus, callback) if (ERR(err, callback)) return; // convert string to boolean - if (typeof publicStatus == "string") + if (typeof publicStatus === "string") publicStatus = publicStatus == "true" ? true : false; // set the password @@ -1045,7 +1045,7 @@ Example returns: */ exports.createDiffHTML = function(padID, startRev, endRev, callback) { // check if startRev is a number - if (startRev !== undefined && typeof startRev != "number") { + if (startRev !== undefined && typeof startRev !== "number") { // try to parse the number if (isNaN(parseInt(startRev))) { callback({stop: "startRev is not a number"}); @@ -1056,7 +1056,7 @@ exports.createDiffHTML = function(padID, startRev, endRev, callback) { } // check if endRev is a number - if (endRev !== undefined && typeof endRev != "number") { + if (endRev !== undefined && typeof endRev !== "number") { // try to parse the number if (isNaN(parseInt(endRev))) { callback({stop: "endRev is not a number"}); @@ -1119,13 +1119,13 @@ function is_int(value) // gets a pad safe function getPadSafe(padID, shouldExist, text, callback) { - if (typeof text == "function") { + if (typeof text === "function") { callback = text; text = null; } // check if padID is a string - if (typeof padID != "string") { + if (typeof padID !== "string") { callback(new customError("padID is not a string", "apierror")); return; } diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index 818eaab28..d04c4ca32 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -45,7 +45,7 @@ exports.doesAuthorExists = function(authorID, callback) db.get("globalAuthor:" + authorID, function(err, author) { if (ERR(err, callback)) return; - callback(null, author != null); + callback(null, author !== null); }); } @@ -98,7 +98,7 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) db.get(mapperkey + ":" + mapper, function(err, author) { if (ERR(err, callback)) return; - if (author == null) { + if (author === null) { // there is no author with this mapper, so create one exports.createAuthor(null, function(err, author) { if (ERR(err, callback)) return; @@ -212,7 +212,7 @@ exports.listPadsOfAuthor = function(authorID, callback) db.get("globalAuthor:" + authorID, function(err, author) { if (ERR(err, callback)) return; - if (author == null) { + if (author === null) { // author does not exist callback(new customError("authorID does not exist", "apierror")); @@ -242,7 +242,7 @@ exports.addPad = function(authorID, padID) // get the entry db.get("globalAuthor:" + authorID, function(err, author) { if (ERR(err)) return; - if (author == null) return; + if (author === null) return; if (author.padIDs == null) { // the entry doesn't exist so far, let's create it @@ -266,9 +266,9 @@ exports.removePad = function(authorID, padID) { db.get("globalAuthor:" + authorID, function(err, author) { if (ERR(err)) return; - if (author == null) return; + if (author === null) return; - if (author.padIDs != null) { + if (author.padIDs !== null) { // remove pad from author delete author.padIDs[padID]; db.set("globalAuthor:" + authorID, author); diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index f384f70aa..985883203 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -122,7 +122,7 @@ exports.deleteGroup = function(groupID, callback) if (ERR(err, callback)) return; groups = groups? groups.groupIDs : []; - if (groups.indexOf(groupID) == -1) { + if (groups.indexOf(groupID) === -1) { // it's not listed callback(); @@ -198,7 +198,7 @@ exports.createGroup = function(callback) exports.createGroupIfNotExistsFor = function(groupMapper, callback) { // ensure mapper is optional - if (typeof groupMapper != "string") { + if (typeof groupMapper !== "string") { callback(new customError("groupMapper is not a string", "apierror")); return; } @@ -248,7 +248,7 @@ exports.createGroupPad = function(groupID, padName, text, callback) exports.doesGroupExist(groupID, function(err, exists) { if (ERR(err, callback)) return; - if (exists == false) { + if (!exists) { // group does not exist callback(new customError("groupID does not exist", "apierror")); return; @@ -303,7 +303,7 @@ exports.listPads = function(groupID, callback) if (ERR(err, callback)) return; // ensure the group exists - if (exists == false) { + if (!exists) { callback(new customError("groupID does not exist", "apierror")); return; } diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index 117f2794f..bc8f59426 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -62,7 +62,7 @@ exports.checkAccess = function(padID, sessionCookie, token, password, callback) } } else { // a session is not required, so we'll check if it's a public pad - if (padID.indexOf("$") == -1) { + if (padID.indexOf("$") === -1) { // it's not a group pad, means we can grant access // get author for this token @@ -225,17 +225,17 @@ exports.checkAccess = function(padID, sessionCookie, token, password, callback) // --> grant access statusObject = { accessStatus: "grant", authorID: sessionAuthor }; - } else if (isPasswordProtected && passwordStatus == "correct") { + } else if (isPasswordProtected && passwordStatus === "correct") { // - the pad is password protected and password is correct // --> grant access statusObject = { accessStatus: "grant", authorID: sessionAuthor }; - } else if (isPasswordProtected && passwordStatus == "wrong") { + } else if (isPasswordProtected && passwordStatus === "wrong") { // - the pad is password protected but wrong password given // --> deny access, ask for new password and tell them that the password is wrong statusObject = { accessStatus: "wrongPassword" }; - } else if (isPasswordProtected && passwordStatus == "notGiven") { + } else if (isPasswordProtected && passwordStatus === "notGiven") { // - the pad is password protected but no password given // --> ask for password @@ -261,17 +261,17 @@ exports.checkAccess = function(padID, sessionCookie, token, password, callback) if (isPublic && !isPasswordProtected) { // --> grant access, with author of token statusObject = {accessStatus: "grant", authorID: tokenAuthor}; - } else if (isPublic && isPasswordProtected && passwordStatus == "correct") { + } else if (isPublic && isPasswordProtected && passwordStatus === "correct") { // - it's public and password protected and password is correct // --> grant access, with author of token statusObject = {accessStatus: "grant", authorID: tokenAuthor}; - } else if (isPublic && isPasswordProtected && passwordStatus == "wrong") { + } else if (isPublic && isPasswordProtected && passwordStatus === "wrong") { // - it's public and the pad is password protected but wrong password given // --> deny access, ask for new password and tell them that the password is wrong statusObject = {accessStatus: "wrongPassword"}; - } else if (isPublic && isPasswordProtected && passwordStatus == "notGiven") { + } else if (isPublic && isPasswordProtected && passwordStatus === "notGiven") { // - it's public and the pad is password protected but no password given // --> ask for password diff --git a/src/node/handler/APIHandler.js b/src/node/handler/APIHandler.js index 3ed7aa4e1..5d731ddc7 100644 --- a/src/node/handler/APIHandler.js +++ b/src/node/handler/APIHandler.js @@ -188,7 +188,7 @@ exports.handle = function(apiVersion, functionName, fields, req, res) // check the api key! fields["apikey"] = fields["apikey"] || fields["api_key"]; - if (fields["apikey"] != apikey.trim()) { + if (fields["apikey"] !== apikey.trim()) { res.statusCode = 401; res.send({code: 4, message: "no or wrong API Key", data: null}); return; diff --git a/src/node/handler/SocketIORouter.js b/src/node/handler/SocketIORouter.js index a94080b9d..08f9f47e8 100644 --- a/src/node/handler/SocketIORouter.js +++ b/src/node/handler/SocketIORouter.js @@ -95,7 +95,7 @@ exports.setSocketIO = function(_socket) { var checkAccessCallback = function(err, statusObject) { ERR(err); - if (statusObject.accessStatus == "grant") { + if (statusObject.accessStatus === "grant") { // access was granted, mark the client as authorized and handle the message clientAuthorized = true; handleMessage(client, message); diff --git a/src/node/padaccess.js b/src/node/padaccess.js index c8c888cd0..a25ad642c 100644 --- a/src/node/padaccess.js +++ b/src/node/padaccess.js @@ -6,7 +6,7 @@ module.exports = function (req, res, callback) { securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password, function(err, accessObj) { if (ERR(err, callback)) return; - if (accessObj.accessStatus == "grant") { + if (accessObj.accessStatus === "grant") { // there is access, continue callback(); } else { From 98993fe15620f50d454c85597116dbd1c90fd044 Mon Sep 17 00:00:00 2001 From: muxator Date: Sat, 9 Feb 2019 02:18:36 +0100 Subject: [PATCH 04/58] db/SessionManager.js: "groupMangager" -> "groupManager" Extracted from Ray's work. --- src/node/db/SessionManager.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index e039ee459..5016cf242 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -23,7 +23,7 @@ var customError = require("../utils/customError"); var randomString = require("../utils/randomstring"); var db = require("./DB").db; var async = require("async"); -var groupMangager = require("./GroupManager"); +var groupManager = require("./GroupManager"); var authorMangager = require("./AuthorManager"); exports.doesSessionExist = function(sessionID, callback) @@ -47,7 +47,7 @@ exports.createSession = function(groupID, authorID, validUntil, callback) // check if the group exists function(callback) { - groupMangager.doesGroupExist(groupID, function(err, exists) + groupManager.doesGroupExist(groupID, function(err, exists) { if(ERR(err, callback)) return; @@ -271,7 +271,7 @@ exports.deleteSession = function(sessionID, callback) exports.listSessionsOfGroup = function(groupID, callback) { - groupMangager.doesGroupExist(groupID, function(err, exists) + groupManager.doesGroupExist(groupID, function(err, exists) { if(ERR(err, callback)) return; From b0846ded149a708ddfb5d1d45c4b8c649f9fe626 Mon Sep 17 00:00:00 2001 From: muxator Date: Sat, 9 Feb 2019 02:19:14 +0100 Subject: [PATCH 05/58] db/SessionManager.js: "authorMangager" -> "authorManager" Extracted from Ray's work. --- src/node/db/SessionManager.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index 5016cf242..28ad4e036 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -24,7 +24,7 @@ var randomString = require("../utils/randomstring"); var db = require("./DB").db; var async = require("async"); var groupManager = require("./GroupManager"); -var authorMangager = require("./AuthorManager"); +var authorManager = require("./AuthorManager"); exports.doesSessionExist = function(sessionID, callback) { @@ -64,7 +64,7 @@ exports.createSession = function(groupID, authorID, validUntil, callback) // check if the author exists function(callback) { - authorMangager.doesAuthorExists(authorID, function(err, exists) + authorManager.doesAuthorExists(authorID, function(err, exists) { if(ERR(err, callback)) return; @@ -287,7 +287,7 @@ exports.listSessionsOfGroup = function(groupID, callback) exports.listSessionsOfAuthor = function(authorID, callback) { - authorMangager.doesAuthorExists(authorID, function(err, exists) + authorManager.doesAuthorExists(authorID, function(err, exists) { if(ERR(err, callback)) return; From 3802073695166bc0892e0956e69aa5ab93a21752 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 18 Jan 2019 13:48:46 +0000 Subject: [PATCH 06/58] db/DB.js: allow a Promise return instead of callbacks in init() --- src/node/db/DB.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/node/db/DB.js b/src/node/db/DB.js index 93848b1bd..fa94259b5 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -35,7 +35,7 @@ exports.db = null; * Initalizes the database with the settings provided by the settings module * @param {Function} callback */ -exports.init = function(callback) { +function init(callback) { // initalize the database async db.init(function(err) { if (err) { @@ -50,3 +50,19 @@ exports.init = function(callback) { } }); } + +/** + * Initalizes the database with the settings provided by the settings module + * If the callback is not supplied a Promise is returned instead. + * @param {Function} callback + */ +exports.init = function(callback) +{ + if (callback === undefined) { + return new Promise(resolve => init(resolve)); + } else if (typeof callback === "function") { + init(callback); + } else { + throw new TypeError("DB.init callback parameter"); + } +} From 8d85ae582e83405f36e077ca2deadd7623549043 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 18 Jan 2019 13:49:17 +0000 Subject: [PATCH 07/58] pluginfw/hooks.js: allow returning a Promise in aCallFirst(), aCallAll() Since this code can end up loaded in browsers when using client side plugins, avoid use of ES6 syntax features such as arrow functions until MSIE support is finally dropped. --- src/static/js/pluginfw/hooks.js | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/static/js/pluginfw/hooks.js b/src/static/js/pluginfw/hooks.js index 7f26c3bfb..489709e33 100644 --- a/src/static/js/pluginfw/hooks.js +++ b/src/static/js/pluginfw/hooks.js @@ -78,7 +78,7 @@ exports.callAll = function (hook_name, args) { } } -exports.aCallAll = function (hook_name, args, cb) { +function aCallAll(hook_name, args, cb) { if (!args) args = {}; if (!cb) cb = function () {}; if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []); @@ -93,6 +93,19 @@ exports.aCallAll = function (hook_name, args, cb) { ); } +/* return a Promise if cb is not supplied */ +exports.aCallAll = function (hook_name, args, cb) { + if (cb === undefined) { + return new Promise(function(resolve, reject) { + aCallAll(hook_name, args, function(err, res) { + return err ? reject(err) : resolve(res); + }); + }); + } else { + return aCallAll(hook_name, args, cb); + } +} + exports.callFirst = function (hook_name, args) { if (!args) args = {}; if (exports.plugins.hooks[hook_name] === undefined) return []; @@ -101,7 +114,7 @@ exports.callFirst = function (hook_name, args) { }); } -exports.aCallFirst = function (hook_name, args, cb) { +function aCallFirst(hook_name, args, cb) { if (!args) args = {}; if (!cb) cb = function () {}; if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []); @@ -114,6 +127,19 @@ exports.aCallFirst = function (hook_name, args, cb) { ); } +/* return a Promise if cb is not supplied */ +exports.aCallFirst = function (hook_name, args, cb) { + if (cb === undefined) { + return new Promise(function(resolve, reject) { + aCallFirst(hook_name, args, function(err, res) { + return err ? reject(err) : resolve(res); + }); + }); + } else { + return aCallFirst(hook_name, args, cb); + } +} + exports.callAllStr = function(hook_name, args, sep, pre, post) { if (sep == undefined) sep = ''; if (pre == undefined) pre = ''; From 80b301915480da8281bfa1a388fffef8672e977d Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 18 Jan 2019 13:52:37 +0000 Subject: [PATCH 08/58] pluginfw/plugins.js: converted to Promise API --- src/static/js/pluginfw/plugins.js | 153 ++++++++++++++---------------- 1 file changed, 69 insertions(+), 84 deletions(-) diff --git a/src/static/js/pluginfw/plugins.js b/src/static/js/pluginfw/plugins.js index 38d6b68c0..ed9c66a37 100644 --- a/src/static/js/pluginfw/plugins.js +++ b/src/static/js/pluginfw/plugins.js @@ -1,7 +1,6 @@ var npm = require("npm/lib/npm.js"); var readInstalled = require("./read-installed.js"); var path = require("path"); -var async = require("async"); var fs = require("fs"); var tsort = require("./tsort"); var util = require("util"); @@ -15,6 +14,7 @@ exports.plugins = {}; exports.parts = []; exports.hooks = {}; +// @TODO RPB this appears to be unused exports.ensure = function (cb) { if (!exports.loaded) exports.update(cb); @@ -53,109 +53,94 @@ exports.formatHooks = function (hook_set_name) { return "
    " + res.join("\n") + "
    "; }; -exports.callInit = function (cb) { +exports.callInit = function () { + const fsp_stat = util.promisify(fs.stat); + const fsp_writeFile = util.promisify(fs.writeFile); + var hooks = require("./hooks"); - async.map( - Object.keys(exports.plugins), - function (plugin_name, cb) { - var plugin = exports.plugins[plugin_name]; - fs.stat(path.normalize(path.join(plugin.package.path, ".ep_initialized")), function (err, stats) { - if (err) { - async.waterfall([ - function (cb) { fs.writeFile(path.normalize(path.join(plugin.package.path, ".ep_initialized")), 'done', cb); }, - function (cb) { hooks.aCallAll("init_" + plugin_name, {}, cb); }, - cb, - ]); - } else { - cb(); - } - }); - }, - function () { cb(); } - ); + let p = Object.keys(exports.plugins).map(function (plugin_name) { + let plugin = exports.plugins[plugin_name]; + let ep_init = path.normalize(path.join(plugin.package.path, ".ep_initialized")); + return fsp_stat(ep_init).catch(async function() { + await fsp_writeFile(ep_init, "done"); + await hooks.aCallAll("init_" + plugin_name, {}); + }); + }); + + return Promise.all(p); } exports.pathNormalization = function (part, hook_fn_name) { return path.normalize(path.join(path.dirname(exports.plugins[part.plugin].package.path), hook_fn_name)); } -exports.update = function (cb) { - exports.getPackages(function (er, packages) { - var parts = []; - var plugins = {}; +exports.update = async function () { + let packages = await exports.getPackages(); + var parts = []; + var plugins = {}; - // Load plugin metadata ep.json - async.forEach( - Object.keys(packages), - function (plugin_name, cb) { - loadPlugin(packages, plugin_name, plugins, parts, cb); - }, - function (err) { - if (err) cb(err); - exports.plugins = plugins; - exports.parts = sortParts(parts); - exports.hooks = pluginUtils.extractHooks(exports.parts, "hooks", exports.pathNormalization); - exports.loaded = true; - exports.callInit(cb); - } - ); + // Load plugin metadata ep.json + let p = Object.keys(packages).map(function (plugin_name) { + return loadPlugin(packages, plugin_name, plugins, parts); }); - }; -exports.getPackages = function (cb) { + return Promise.all(p).then(function() { + exports.plugins = plugins; + exports.parts = sortParts(parts); + exports.hooks = pluginUtils.extractHooks(exports.parts, "hooks", exports.pathNormalization); + exports.loaded = true; + }).then(exports.callInit); +} + +exports.getPackages = async function () { // Load list of installed NPM packages, flatten it to a list, and filter out only packages with names that var dir = path.resolve(npm.dir, '..'); - readInstalled(dir, function (er, data) { - if (er) cb(er, null); + let data = await util.promisify(readInstalled)(dir); - var packages = {}; - function flatten(deps) { - _.chain(deps).keys().each(function (name) { - if (name.indexOf(exports.prefix) === 0) { - packages[name] = _.clone(deps[name]); - // Delete anything that creates loops so that the plugin - // list can be sent as JSON to the web client - delete packages[name].dependencies; - delete packages[name].parent; - } + var packages = {}; + function flatten(deps) { + _.chain(deps).keys().each(function (name) { + if (name.indexOf(exports.prefix) === 0) { + packages[name] = _.clone(deps[name]); + // Delete anything that creates loops so that the plugin + // list can be sent as JSON to the web client + delete packages[name].dependencies; + delete packages[name].parent; + } - // I don't think we need recursion - //if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies); - }); - } + // I don't think we need recursion + //if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies); + }); + } - var tmp = {}; - tmp[data.name] = data; - flatten(tmp[data.name].dependencies); - cb(null, packages); - }); + var tmp = {}; + tmp[data.name] = data; + flatten(tmp[data.name].dependencies); + return packages; }; -function loadPlugin(packages, plugin_name, plugins, parts, cb) { +async function loadPlugin(packages, plugin_name, plugins, parts) { + let fsp_readFile = util.promisify(fs.readFile); + var plugin_path = path.resolve(packages[plugin_name].path, "ep.json"); - fs.readFile( - plugin_path, - function (er, data) { - if (er) { - console.error("Unable to load plugin definition file " + plugin_path); - return cb(); - } - try { - var plugin = JSON.parse(data); - plugin['package'] = packages[plugin_name]; - plugins[plugin_name] = plugin; - _.each(plugin.parts, function (part) { - part.plugin = plugin_name; - part.full_name = plugin_name + "/" + part.name; - parts[part.full_name] = part; - }); - } catch (ex) { - console.error("Unable to parse plugin definition file " + plugin_path + ": " + ex.toString()); - } - cb(); + try { + let data = await fsp_readFile(plugin_path); + try { + var plugin = JSON.parse(data); + plugin['package'] = packages[plugin_name]; + plugins[plugin_name] = plugin; + _.each(plugin.parts, function (part) { + part.plugin = plugin_name; + part.full_name = plugin_name + "/" + part.name; + parts[part.full_name] = part; + }); + } catch (ex) { + console.error("Unable to parse plugin definition file " + plugin_path + ": " + ex.toString()); } - ); + } catch (er) { + console.error("Unable to load plugin definition file " + plugin_path); + } } function partsToParentChildList(parts) { From a579dfc2852ca5bfb57b3aa411398b7b7a477489 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 18 Jan 2019 16:10:48 +0000 Subject: [PATCH 09/58] pluginfw/installer.js: use Promise version of hooks.aCallAll() in install(), uninstall() We cannot use arrow functions in this file, because code in /src/static can end up being loaded in browsers, and we still support IE11. --- src/static/js/pluginfw/installer.js | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/static/js/pluginfw/installer.js b/src/static/js/pluginfw/installer.js index fbd0e1d97..dbf8696ed 100644 --- a/src/static/js/pluginfw/installer.js +++ b/src/static/js/pluginfw/installer.js @@ -34,6 +34,10 @@ function onAllTasksFinished() { hooks.aCallAll("restartServer", {}, function() {}); } +/* + * We cannot use arrow functions in this file, because code in /src/static + * can end up being loaded in browsers, and we still support IE11. + */ exports.uninstall = function(plugin_name, cb) { cb = wrapTaskCb(cb); @@ -42,16 +46,18 @@ exports.uninstall = function(plugin_name, cb) { npm.commands.uninstall([plugin_name], function(er) { if (er) return cb && cb(er); - - hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name}, function(er, data) { - if (er) return cb(er); - - plugins.update(cb); - }); + hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name}) + .then(plugins.update) + .then(function() { cb(null) }) + .catch(function(er) { cb(er) }); }); }); }; +/* + * We cannot use arrow functions in this file, because code in /src/static + * can end up being loaded in browsers, and we still support IE11. + */ exports.install = function(plugin_name, cb) { cb = wrapTaskCb(cb); @@ -60,12 +66,10 @@ exports.install = function(plugin_name, cb) { npm.commands.install([plugin_name], function(er) { if (er) return cb && cb(er); - - hooks.aCallAll("pluginInstall", {plugin_name: plugin_name}, function(er, data) { - if (er) return cb(er); - - plugins.update(cb); - }); + hooks.aCallAll("pluginInstall", {plugin_name: plugin_name}) + .then(plugins.update) + .then(function() { cb(null) }) + .catch(function(er) { cb(er) }); }); }); }; From 4877ec319a2a094271ad760246e7936145eea258 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 18 Jan 2019 16:10:25 +0000 Subject: [PATCH 10/58] server.js: rewritten to use Promises --- src/node/server.js | 78 ++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/src/node/server.js b/src/node/server.js index 2ba1fa47d..f683de82c 100755 --- a/src/node/server.js +++ b/src/node/server.js @@ -22,7 +22,6 @@ */ var log4js = require('log4js') - , async = require('async') , NodeVersion = require('./utils/NodeVersion') ; @@ -49,52 +48,37 @@ stats.gauge('memoryUsage', function() { return process.memoryUsage().rss; }); -var settings - , db - , plugins - , hooks; +/* + * no use of let or await here because it would cause startup + * to fail completely on very early versions of NodeJS + */ var npm = require("npm/lib/npm.js"); -async.waterfall([ - // load npm - function(callback) { - npm.load({}, function(er) { - callback(er); +npm.load({}, function() { + var settings = require('./utils/Settings'); + var db = require('./db/DB'); + var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins"); + var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks"); + hooks.plugins = plugins; + + db.init() + .then(plugins.update) + .then(function() { + console.info("Installed plugins: " + plugins.formatPluginsWithVersion()); + console.debug("Installed parts:\n" + plugins.formatParts()); + console.debug("Installed hooks:\n" + plugins.formatHooks()); + + // Call loadSettings hook + hooks.aCallAll("loadSettings", { settings: settings }); + + // initalize the http server + hooks.callAll("createServer", {}); + }) + .catch(function(e) { + console.error("exception thrown: " + e.message); + if (e.stack) { + console.log(e.stack); + } + process.exit(1); }); - }, - - // load everything - function(callback) { - settings = require('./utils/Settings'); - db = require('./db/DB'); - plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins"); - hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks"); - hooks.plugins = plugins; - callback(); - }, - - // initalize the database - function (callback) { - db.init(callback); - }, - - function(callback) { - plugins.update(callback); - }, - - function (callback) { - console.info("Installed plugins: " + plugins.formatPluginsWithVersion()); - console.debug("Installed parts:\n" + plugins.formatParts()); - console.debug("Installed hooks:\n" + plugins.formatHooks()); - - // Call loadSettings hook - hooks.aCallAll("loadSettings", { settings: settings }); - callback(); - }, - - // initalize the http server - function (callback) { - hooks.callAll("createServer", {}); - callback(null); - } -]); +}); From 40c45077ef3a0bfdd3c49eedf47fd18dcb38d0f6 Mon Sep 17 00:00:00 2001 From: muxator Date: Sat, 9 Feb 2019 01:15:50 +0100 Subject: [PATCH 11/58] db/GroupManager.js: factored out a variable Extracted from Ray's work. --- src/node/db/GroupManager.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index 985883203..88aa8aa04 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -122,7 +122,9 @@ exports.deleteGroup = function(groupID, callback) if (ERR(err, callback)) return; groups = groups? groups.groupIDs : []; - if (groups.indexOf(groupID) === -1) { + let index = groups.indexOf(groupID); + + if (index === -1) { // it's not listed callback(); @@ -130,7 +132,7 @@ exports.deleteGroup = function(groupID, callback) } // remove from the list - groups.splice(groups.indexOf(groupID), 1); + groups.splice(index, 1); // store empty group list if (groups.length == 0) { From 17fe32ec0c1078983cb4a35686ea1d5be3b9a8e8 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Mon, 21 Jan 2019 16:28:05 +0000 Subject: [PATCH 12/58] start using "thenify" to support callback and promises PadManager.sanitizePadId() can't use thenify: single arg callback --- src/node/db/API.js | 121 +++++++++++++++++---------------- src/node/db/AuthorManager.js | 41 +++++------ src/node/db/DB.js | 21 +----- src/node/db/GroupManager.js | 29 ++++---- src/node/db/Pad.js | 49 ++++++------- src/node/db/PadManager.js | 38 +++++++---- src/node/db/ReadOnlyManager.js | 13 ++-- src/node/db/SessionManager.js | 25 +++---- src/package.json | 1 + 9 files changed, 170 insertions(+), 168 deletions(-) diff --git a/src/node/db/API.js b/src/node/db/API.js index b6acefcd4..df2e25f10 100644 --- a/src/node/db/API.js +++ b/src/node/db/API.js @@ -33,6 +33,7 @@ var exportTxt = require("../utils/ExportTxt"); var importHtml = require("../utils/ImportHtml"); var cleanText = require("./Pad").cleanText; var PadDiff = require("../utils/padDiff"); +const thenify = require("thenify").withCallback; /**********************/ /**GROUP FUNCTIONS*****/ @@ -103,14 +104,14 @@ Example returns: } */ -exports.getAttributePool = function(padID, callback) +exports.getAttributePool = thenify(function(padID, callback) { getPadSafe(padID, true, function(err, pad) { if (ERR(err, callback)) return; callback(null, {pool: pad.pool}); }); -} +}); /** getRevisionChangeset (padID, [rev]) @@ -125,7 +126,7 @@ Example returns: } */ -exports.getRevisionChangeset = function(padID, rev, callback) +exports.getRevisionChangeset = thenify(function(padID, rev, callback) { // check if rev is a number if (rev !== undefined && typeof rev !== "number") { @@ -179,7 +180,7 @@ exports.getRevisionChangeset = function(padID, rev, callback) callback(null, changeset); }) }); -} +}); /** getText(padID, [rev]) returns the text of a pad @@ -189,7 +190,7 @@ Example returns: {code: 0, message:"ok", data: {text:"Welcome Text"}} {code: 1, message:"padID does not exist", data: null} */ -exports.getText = function(padID, rev, callback) +exports.getText = thenify(function(padID, rev, callback) { // check if rev is a number if (rev !== undefined && typeof rev !== "number") { @@ -242,7 +243,7 @@ exports.getText = function(padID, rev, callback) var padText = exportTxt.getTXTFromAtext(pad, pad.atext); callback(null, {"text": padText}); }); -} +}); /** setText(padID, text) sets the text of a pad @@ -253,7 +254,7 @@ Example returns: {code: 1, message:"padID does not exist", data: null} {code: 1, message:"text too long", data: null} */ -exports.setText = function(padID, text, callback) +exports.setText = thenify(function(padID, text, callback) { // text is required if (typeof text !== "string") { @@ -271,7 +272,7 @@ exports.setText = function(padID, text, callback) // update the clients on the pad padMessageHandler.updatePadClients(pad, callback); }); -} +}); /** appendText(padID, text) appends text to a pad @@ -282,7 +283,7 @@ Example returns: {code: 1, message:"padID does not exist", data: null} {code: 1, message:"text too long", data: null} */ -exports.appendText = function(padID, text, callback) +exports.appendText = thenify(function(padID, text, callback) { // text is required if (typeof text !== "string") { @@ -299,7 +300,7 @@ exports.appendText = function(padID, text, callback) // update the clients on the pad padMessageHandler.updatePadClients(pad, callback); }); -}; +}); /** getHTML(padID, [rev]) returns the html of a pad @@ -309,7 +310,7 @@ Example returns: {code: 0, message:"ok", data: {text:"Welcome Text"}} {code: 1, message:"padID does not exist", data: null} */ -exports.getHTML = function(padID, rev, callback) +exports.getHTML = thenify(function(padID, rev, callback) { if (rev !== undefined && typeof rev !== "number") { if (isNaN(parseInt(rev))) { @@ -362,7 +363,7 @@ exports.getHTML = function(padID, rev, callback) callback(null, data); }); }); -} +}); /** setHTML(padID, html) sets the text of a pad based on HTML @@ -372,7 +373,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.setHTML = function(padID, html, callback) +exports.setHTML = thenify(function(padID, html, callback) { // html is required if (typeof html !== "string") { @@ -395,7 +396,7 @@ exports.setHTML = function(padID, html, callback) padMessageHandler.updatePadClients(pad, callback); }); }); -} +}); /******************/ /**CHAT FUNCTIONS */ @@ -413,7 +414,7 @@ Example returns: {code: 1, message:"padID does not exist", data: null} */ -exports.getChatHistory = function(padID, start, end, callback) +exports.getChatHistory = thenify(function(padID, start, end, callback) { if (start && end) { if (start < 0) { @@ -458,7 +459,7 @@ exports.getChatHistory = function(padID, start, end, callback) callback(null, {messages: msgs}); }); }); -} +}); /** appendChatMessage(padID, text, authorID, time), creates a chat message for the pad id, time is a timestamp @@ -468,7 +469,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.appendChatMessage = function(padID, text, authorID, time, callback) +exports.appendChatMessage = thenify(function(padID, text, authorID, time, callback) { // text is required if (typeof text !== "string") { @@ -486,7 +487,7 @@ exports.appendChatMessage = function(padID, text, authorID, time, callback) padMessageHandler.sendChatMessageToPadClients(time, authorID, text, padID); callback(); -} +}); /*****************/ /**PAD FUNCTIONS */ @@ -500,7 +501,7 @@ Example returns: {code: 0, message:"ok", data: {revisions: 56}} {code: 1, message:"padID does not exist", data: null} */ -exports.getRevisionsCount = function(padID, callback) +exports.getRevisionsCount = thenify(function(padID, callback) { // get the pad getPadSafe(padID, true, function(err, pad) { @@ -508,7 +509,7 @@ exports.getRevisionsCount = function(padID, callback) callback(null, {revisions: pad.getHeadRevisionNumber()}); }); -} +}); /** getSavedRevisionsCount(padID) returns the number of saved revisions of this pad @@ -518,7 +519,7 @@ Example returns: {code: 0, message:"ok", data: {savedRevisions: 42}} {code: 1, message:"padID does not exist", data: null} */ -exports.getSavedRevisionsCount = function(padID, callback) +exports.getSavedRevisionsCount = thenify(function(padID, callback) { // get the pad getPadSafe(padID, true, function(err, pad) { @@ -526,7 +527,7 @@ exports.getSavedRevisionsCount = function(padID, callback) callback(null, {savedRevisions: pad.getSavedRevisionsNumber()}); }); -} +}); /** listSavedRevisions(padID) returns the list of saved revisions of this pad @@ -536,7 +537,7 @@ Example returns: {code: 0, message:"ok", data: {savedRevisions: [2, 42, 1337]}} {code: 1, message:"padID does not exist", data: null} */ -exports.listSavedRevisions = function(padID, callback) +exports.listSavedRevisions = thenify(function(padID, callback) { // get the pad getPadSafe(padID, true, function(err, pad) { @@ -544,7 +545,7 @@ exports.listSavedRevisions = function(padID, callback) callback(null, {savedRevisions: pad.getSavedRevisionsList()}); }); -} +}); /** saveRevision(padID) returns the list of saved revisions of this pad @@ -554,7 +555,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.saveRevision = function(padID, rev, callback) +exports.saveRevision = thenify(function(padID, rev, callback) { // check if rev is a number if (rev !== undefined && typeof rev !== "number") { @@ -601,7 +602,7 @@ exports.saveRevision = function(padID, rev, callback) callback(); }); }); -} +}); /** getLastEdited(padID) returns the timestamp of the last revision of the pad @@ -611,7 +612,7 @@ Example returns: {code: 0, message:"ok", data: {lastEdited: 1340815946602}} {code: 1, message:"padID does not exist", data: null} */ -exports.getLastEdited = function(padID, callback) +exports.getLastEdited = thenify(function(padID, callback) { // get the pad getPadSafe(padID, true, function(err, pad) { @@ -622,7 +623,7 @@ exports.getLastEdited = function(padID, callback) callback(null, {lastEdited: value}); }); }); -} +}); /** createPad(padName [, text]) creates a new pad in this group @@ -632,7 +633,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"pad does already exist", data: null} */ -exports.createPad = function(padID, text, callback) +exports.createPad = thenify(function(padID, text, callback) { if (padID) { // ensure there is no $ in the padID @@ -653,7 +654,7 @@ exports.createPad = function(padID, text, callback) if (ERR(err, callback)) return; callback(); }); -} +}); /** deletePad(padID) deletes a pad @@ -663,14 +664,14 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.deletePad = function(padID, callback) +exports.deletePad = thenify(function(padID, callback) { getPadSafe(padID, true, function(err, pad) { if (ERR(err, callback)) return; pad.remove(callback); }); -} +}); /** restoreRevision(padID, [rev]) Restores revision from past as new changeset @@ -679,7 +680,7 @@ exports.deletePad = function(padID, callback) {code:0, message:"ok", data:null} {code: 1, message:"padID does not exist", data: null} */ -exports.restoreRevision = function(padID, rev, callback) +exports.restoreRevision = thenify(function(padID, rev, callback) { // check if rev is a number if (rev !== undefined && typeof rev !== "number") { @@ -762,7 +763,7 @@ exports.restoreRevision = function(padID, rev, callback) }); }); -}; +}); /** copyPad(sourceID, destinationID[, force=false]) copies a pad. If force is true, @@ -773,14 +774,14 @@ Example returns: {code: 0, message:"ok", data: {padID: destinationID}} {code: 1, message:"padID does not exist", data: null} */ -exports.copyPad = function(sourceID, destinationID, force, callback) +exports.copyPad = thenify(function(sourceID, destinationID, force, callback) { getPadSafe(sourceID, true, function(err, pad) { if (ERR(err, callback)) return; pad.copy(destinationID, force, callback); }); -} +}); /** movePad(sourceID, destinationID[, force=false]) moves a pad. If force is true, @@ -791,7 +792,7 @@ Example returns: {code: 0, message:"ok", data: {padID: destinationID}} {code: 1, message:"padID does not exist", data: null} */ -exports.movePad = function(sourceID, destinationID, force, callback) +exports.movePad = thenify(function(sourceID, destinationID, force, callback) { getPadSafe(sourceID, true, function(err, pad) { if (ERR(err, callback)) return; @@ -801,7 +802,7 @@ exports.movePad = function(sourceID, destinationID, force, callback) pad.remove(callback); }); }); -} +}); /** getReadOnlyLink(padID) returns the read only link of a pad @@ -810,7 +811,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.getReadOnlyID = function(padID, callback) +exports.getReadOnlyID = thenify(function(padID, callback) { // we don't need the pad object, but this function does all the security stuff for us getPadSafe(padID, true, function(err) { @@ -822,7 +823,7 @@ exports.getReadOnlyID = function(padID, callback) callback(null, {readOnlyID: readOnlyId}); }); }); -} +}); /** getPadID(roID) returns the padID of a pad based on the readonlyID(roID) @@ -832,7 +833,7 @@ Example returns: {code: 0, message:"ok", data: {padID: padID}} {code: 1, message:"padID does not exist", data: null} */ -exports.getPadID = function(roID, callback) +exports.getPadID = thenify(function(roID, callback) { // get the PadId readOnlyManager.getPadId(roID, function(err, retrievedPadID) { @@ -845,7 +846,7 @@ exports.getPadID = function(roID, callback) callback(null, {padID: retrievedPadID}); }); -} +}); /** setPublicStatus(padID, publicStatus) sets a boolean for the public status of a pad @@ -855,7 +856,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.setPublicStatus = function(padID, publicStatus, callback) +exports.setPublicStatus = thenify(function(padID, publicStatus, callback) { // ensure this is a group pad if (padID && padID.indexOf("$") === -1) { @@ -876,7 +877,7 @@ exports.setPublicStatus = function(padID, publicStatus, callback) callback(); }); -} +}); /** getPublicStatus(padID) return true of false @@ -886,7 +887,7 @@ Example returns: {code: 0, message:"ok", data: {publicStatus: true}} {code: 1, message:"padID does not exist", data: null} */ -exports.getPublicStatus = function(padID, callback) +exports.getPublicStatus = thenify(function(padID, callback) { // ensure this is a group pad if (padID && padID.indexOf("$") == -1) { @@ -900,7 +901,7 @@ exports.getPublicStatus = function(padID, callback) callback(null, {publicStatus: pad.getPublicStatus()}); }); -} +}); /** setPassword(padID, password) returns ok or a error message @@ -910,7 +911,7 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.setPassword = function(padID, password, callback) +exports.setPassword = thenify(function(padID, password, callback) { // ensure this is a group pad if (padID && padID.indexOf("$") == -1) { @@ -927,7 +928,7 @@ exports.setPassword = function(padID, password, callback) callback(); }); -} +}); /** isPasswordProtected(padID) returns true or false @@ -937,7 +938,7 @@ Example returns: {code: 0, message:"ok", data: {passwordProtection: true}} {code: 1, message:"padID does not exist", data: null} */ -exports.isPasswordProtected = function(padID, callback) +exports.isPasswordProtected = thenify(function(padID, callback) { // ensure this is a group pad if (padID && padID.indexOf("$") == -1) { @@ -951,7 +952,7 @@ exports.isPasswordProtected = function(padID, callback) callback(null, {isPasswordProtected: pad.isPasswordProtected()}); }); -} +}); /** listAuthorsOfPad(padID) returns an array of authors who contributed to this pad @@ -961,7 +962,7 @@ Example returns: {code: 0, message:"ok", data: {authorIDs : ["a.s8oes9dhwrvt0zif", "a.akf8finncvomlqva"]} {code: 1, message:"padID does not exist", data: null} */ -exports.listAuthorsOfPad = function(padID, callback) +exports.listAuthorsOfPad = thenify(function(padID, callback) { // get the pad getPadSafe(padID, true, function(err, pad) { @@ -969,7 +970,7 @@ exports.listAuthorsOfPad = function(padID, callback) callback(null, {authorIDs: pad.getAllAuthors()}); }); -} +}); /** sendClientsMessage(padID, msg) sends a message to all clients connected to the @@ -994,7 +995,7 @@ Example returns: {code: 1, message:"padID does not exist"} */ -exports.sendClientsMessage = function(padID, msg, callback) { +exports.sendClientsMessage = thenify(function(padID, msg, callback) { getPadSafe(padID, true, function(err, pad) { if (ERR(err, callback)) { return; @@ -1002,7 +1003,7 @@ exports.sendClientsMessage = function(padID, msg, callback) { padMessageHandler.handleCustomMessage(padID, msg, callback); } ); -} +}); /** checkToken() returns ok when the current api token is valid @@ -1012,10 +1013,10 @@ Example returns: {"code":0,"message":"ok","data":null} {"code":4,"message":"no or wrong API Key","data":null} */ -exports.checkToken = function(callback) +exports.checkToken = thenify(function(callback) { callback(); -} +}); /** getChatHead(padID) returns the chatHead (last number of the last chat-message) of the pad @@ -1025,7 +1026,7 @@ Example returns: {code: 0, message:"ok", data: {chatHead: 42}} {code: 1, message:"padID does not exist", data: null} */ -exports.getChatHead = function(padID, callback) +exports.getChatHead = thenify(function(padID, callback) { // get the pad getPadSafe(padID, true, function(err, pad) { @@ -1033,7 +1034,7 @@ exports.getChatHead = function(padID, callback) callback(null, {chatHead: pad.chatHead}); }); -} +}); /** createDiffHTML(padID, startRev, endRev) returns an object of diffs from 2 points in a pad @@ -1043,7 +1044,7 @@ Example returns: {"code":0,"message":"ok","data":{"html":"Welcome to Etherpad!

    This pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!

    Get involved with Etherpad at http://etherpad.org
    aw

    ","authors":["a.HKIv23mEbachFYfH",""]}} {"code":4,"message":"no or wrong API Key","data":null} */ -exports.createDiffHTML = function(padID, startRev, endRev, callback) { +exports.createDiffHTML = thenify(function(padID, startRev, endRev, callback) { // check if startRev is a number if (startRev !== undefined && typeof startRev !== "number") { // try to parse the number @@ -1104,7 +1105,7 @@ exports.createDiffHTML = function(padID, startRev, endRev, callback) { callback(err, {html: html, authors: authors}) }); }); -} +}); /******************************/ /** INTERNAL HELPER FUNCTIONS */ diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index d04c4ca32..0db187706 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -22,6 +22,7 @@ var ERR = require("async-stacktrace"); var db = require("./DB").db; var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; +const thenify = require("thenify").withCallback; exports.getColorPalette = function() { return [ @@ -39,7 +40,7 @@ exports.getColorPalette = function() { /** * Checks if the author exists */ -exports.doesAuthorExists = function(authorID, callback) +exports.doesAuthorExists = thenify(function(authorID, callback) { // check if the database entry of this author exists db.get("globalAuthor:" + authorID, function(err, author) { @@ -47,14 +48,14 @@ exports.doesAuthorExists = function(authorID, callback) callback(null, author !== null); }); -} +}); /** * Returns the AuthorID for a token. * @param {String} token The token * @param {Function} callback callback (err, author) */ -exports.getAuthor4Token = function(token, callback) +exports.getAuthor4Token = thenify(function(token, callback) { mapAuthorWithDBKey("token2author", token, function(err, author) { if (ERR(err, callback)) return; @@ -62,7 +63,7 @@ exports.getAuthor4Token = function(token, callback) // return only the sub value authorID callback(null, author ? author.authorID : author); }); -} +}); /** * Returns the AuthorID for a mapper. @@ -70,7 +71,7 @@ exports.getAuthor4Token = function(token, callback) * @param {String} name The name of the author (optional) * @param {Function} callback callback (err, author) */ -exports.createAuthorIfNotExistsFor = function(authorMapper, name, callback) +exports.createAuthorIfNotExistsFor = thenify(function(authorMapper, name, callback) { mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author) { if (ERR(err, callback)) return; @@ -83,7 +84,7 @@ exports.createAuthorIfNotExistsFor = function(authorMapper, name, callback) // return the authorID callback(null, author); }); -} +}); /** * Returns the AuthorID for a mapper. We can map using a mapperkey, @@ -126,7 +127,7 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) * Internal function that creates the database entry for an author * @param {String} name The name of the author */ -exports.createAuthor = function(name, callback) +exports.createAuthor = thenify(function(name, callback) { // create the new author name var author = "a." + randomString(16); @@ -142,27 +143,27 @@ exports.createAuthor = function(name, callback) db.set("globalAuthor:" + author, authorObj); callback(null, {authorID: author}); -} +}); /** * Returns the Author Obj of the author * @param {String} author The id of the author * @param {Function} callback callback(err, authorObj) */ -exports.getAuthor = function(author, callback) +exports.getAuthor = thenify(function(author, callback) { db.get("globalAuthor:" + author, callback); -} +}); /** * Returns the color Id of the author * @param {String} author The id of the author * @param {Function} callback callback(err, colorId) */ -exports.getAuthorColorId = function(author, callback) +exports.getAuthorColorId = thenify(function(author, callback) { db.getSub("globalAuthor:" + author, ["colorId"], callback); -} +}); /** * Sets the color Id of the author @@ -170,20 +171,20 @@ exports.getAuthorColorId = function(author, callback) * @param {String} colorId The color id of the author * @param {Function} callback (optional) */ -exports.setAuthorColorId = function(author, colorId, callback) +exports.setAuthorColorId = thenify(function(author, colorId, callback) { db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback); -} +}); /** * Returns the name of the author * @param {String} author The id of the author * @param {Function} callback callback(err, name) */ -exports.getAuthorName = function(author, callback) +exports.getAuthorName = thenify(function(author, callback) { db.getSub("globalAuthor:" + author, ["name"], callback); -} +}); /** * Sets the name of the author @@ -191,17 +192,17 @@ exports.getAuthorName = function(author, callback) * @param {String} name The name of the author * @param {Function} callback (optional) */ -exports.setAuthorName = function(author, name, callback) +exports.setAuthorName = thenify(function(author, name, callback) { db.setSub("globalAuthor:" + author, ["name"], name, callback); -} +}); /** * Returns an array of all pads this author contributed to * @param {String} author The id of the author * @param {Function} callback (optional) */ -exports.listPadsOfAuthor = function(authorID, callback) +exports.listPadsOfAuthor = thenify(function(authorID, callback) { /* There are two other places where this array is manipulated: * (1) When the author is added to a pad, the author object is also updated @@ -230,7 +231,7 @@ exports.listPadsOfAuthor = function(authorID, callback) callback(null, {padIDs: pads}); }); -} +}); /** * Adds a new pad to the list of contributions diff --git a/src/node/db/DB.js b/src/node/db/DB.js index fa94259b5..e6ddd5fb3 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -22,6 +22,7 @@ var ueberDB = require("ueberdb2"); var settings = require("../utils/Settings"); var log4js = require('log4js'); +const thenify = require("thenify").withCallback; // set database settings var db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB")); @@ -35,7 +36,7 @@ exports.db = null; * Initalizes the database with the settings provided by the settings module * @param {Function} callback */ -function init(callback) { +exports.init = thenify(function (callback) { // initalize the database async db.init(function(err) { if (err) { @@ -49,20 +50,4 @@ function init(callback) { callback(null); } }); -} - -/** - * Initalizes the database with the settings provided by the settings module - * If the callback is not supplied a Promise is returned instead. - * @param {Function} callback - */ -exports.init = function(callback) -{ - if (callback === undefined) { - return new Promise(resolve => init(resolve)); - } else if (typeof callback === "function") { - init(callback); - } else { - throw new TypeError("DB.init callback parameter"); - } -} +}); diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index 88aa8aa04..d72efdff1 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -25,8 +25,9 @@ var db = require("./DB").db; var async = require("async"); var padManager = require("./PadManager"); var sessionManager = require("./SessionManager"); +const thenify = require("thenify").withCallback; -exports.listAllGroups = function(callback) { +exports.listAllGroups = thenify(function(callback) { db.get("groups", function (err, groups) { if (ERR(err, callback)) return; @@ -43,9 +44,9 @@ exports.listAllGroups = function(callback) { } callback(null, {groupIDs: groupIDs}); }); -} +}); -exports.deleteGroup = function(groupID, callback) +exports.deleteGroup = thenify(function(groupID, callback) { var group; @@ -158,18 +159,18 @@ exports.deleteGroup = function(groupID, callback) if (ERR(err, callback)) return; callback(); }); -} +}); -exports.doesGroupExist = function(groupID, callback) +exports.doesGroupExist = thenify(function(groupID, callback) { // try to get the group entry db.get("group:" + groupID, function (err, group) { if (ERR(err, callback)) return; callback(null, group != null); }); -} +}); -exports.createGroup = function(callback) +exports.createGroup = thenify(function(callback) { // search for non existing groupID var groupID = "g." + randomString(16); @@ -195,9 +196,9 @@ exports.createGroup = function(callback) callback(null, {groupID: groupID}); }); }); -} +}); -exports.createGroupIfNotExistsFor = function(groupMapper, callback) +exports.createGroupIfNotExistsFor = thenify(function(groupMapper, callback) { // ensure mapper is optional if (typeof groupMapper !== "string") { @@ -237,9 +238,9 @@ exports.createGroupIfNotExistsFor = function(groupMapper, callback) // there is no group for this mapper, let's create a group createGroupForMapper(callback) }); -} +}); -exports.createGroupPad = function(groupID, padName, text, callback) +exports.createGroupPad = thenify(function(groupID, padName, text, callback) { // create the padID var padID = groupID + "$" + padName; @@ -297,9 +298,9 @@ exports.createGroupPad = function(groupID, padName, text, callback) callback(null, {padID: padID}); }); -} +}); -exports.listPads = function(groupID, callback) +exports.listPads = thenify(function(groupID, callback) { exports.doesGroupExist(groupID, function(err, exists) { if (ERR(err, callback)) return; @@ -321,4 +322,4 @@ exports.listPads = function(groupID, callback) callback(null, {padIDs: pads}); }); }); -} +}); diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index c5a75a89f..34d000631 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -18,6 +18,7 @@ var readOnlyManager = require("./ReadOnlyManager"); var crypto = require("crypto"); var randomString = require("../utils/randomstring"); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); +const thenify = require("thenify").withCallback; // serialization/deserialization attributes var attributeBlackList = ["id"]; @@ -131,22 +132,22 @@ Pad.prototype.saveToDatabase = function saveToDatabase() { } // get time of last edit (changeset application) -Pad.prototype.getLastEdit = function getLastEdit(callback) { +Pad.prototype.getLastEdit = thenify(function getLastEdit(callback) { var revNum = this.getHeadRevisionNumber(); db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); -} +}); -Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum, callback) { +Pad.prototype.getRevisionChangeset = thenify(function getRevisionChangeset(revNum, callback) { db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"], callback); -}; +}); -Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum, callback) { +Pad.prototype.getRevisionAuthor = thenify(function getRevisionAuthor(revNum, callback) { db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"], callback); -}; +}); -Pad.prototype.getRevisionDate = function getRevisionDate(revNum, callback) { +Pad.prototype.getRevisionDate = thenify(function getRevisionDate(revNum, callback) { db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); -}; +}); Pad.prototype.getAllAuthors = function getAllAuthors() { var authors = []; @@ -160,7 +161,7 @@ Pad.prototype.getAllAuthors = function getAllAuthors() { return authors; }; -Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targetRev, callback) { +Pad.prototype.getInternalRevisionAText = thenify(function getInternalRevisionAText(targetRev, callback) { var _this = this; var keyRev = this.getKeyRevisionNumber(targetRev); @@ -228,13 +229,13 @@ Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targe if (ERR(err, callback)) return; callback(null, atext); }); -}; +}); -Pad.prototype.getRevision = function getRevisionChangeset(revNum, callback) { +Pad.prototype.getRevision = thenify(function getRevisionChangeset(revNum, callback) { db.get("pad:" + this.id + ":revs:" + revNum, callback); -}; +}); -Pad.prototype.getAllAuthorColors = function getAllAuthorColors(callback) { +Pad.prototype.getAllAuthorColors = thenify(function getAllAuthorColors(callback) { var authors = this.getAllAuthors(); var returnTable = {}; var colorPalette = authorManager.getColorPalette(); @@ -254,7 +255,7 @@ Pad.prototype.getAllAuthorColors = function getAllAuthorColors(callback) { function(err) { callback(err, returnTable); }); -}; +}); Pad.prototype.getValidRevisionRange = function getValidRevisionRange(startRev, endRev) { startRev = parseInt(startRev, 10); @@ -325,7 +326,7 @@ Pad.prototype.appendChatMessage = function appendChatMessage(text, userId, time) this.saveToDatabase(); }; -Pad.prototype.getChatMessage = function getChatMessage(entryNum, callback) { +Pad.prototype.getChatMessage = thenify(function getChatMessage(entryNum, callback) { var _this = this; var entry; @@ -359,9 +360,9 @@ Pad.prototype.getChatMessage = function getChatMessage(entryNum, callback) { if (ERR(err, callback)) return; callback(null, entry); }); -}; +}); -Pad.prototype.getChatMessages = function getChatMessages(start, end, callback) { +Pad.prototype.getChatMessages = thenify(function getChatMessages(start, end, callback) { // collect the numbers of chat entries and in which order we need them var neededEntries = []; var order = 0; @@ -398,9 +399,9 @@ Pad.prototype.getChatMessages = function getChatMessages(start, end, callback) { callback(null, cleanedEntries); }); -}; +}); -Pad.prototype.init = function init(text, callback) { +Pad.prototype.init = thenify(function init(text, callback) { var _this = this; // replace text with default text if text isn't set @@ -432,9 +433,9 @@ Pad.prototype.init = function init(text, callback) { hooks.callAll("padLoad", { 'pad': _this }); callback(null); }); -}; +}); -Pad.prototype.copy = function copy(destinationID, force, callback) { +Pad.prototype.copy = thenify(function copy(destinationID, force, callback) { var sourceID = this.id; var _this = this; var destGroupID; @@ -581,9 +582,9 @@ Pad.prototype.copy = function copy(destinationID, force, callback) { if (ERR(err, callback)) return; callback(null, { padID: destinationID }); }); -}; +}); -Pad.prototype.remove = function remove(callback) { +Pad.prototype.remove = thenify(function remove(callback) { var padID = this.id; var _this = this; @@ -676,7 +677,7 @@ Pad.prototype.remove = function remove(callback) { if (ERR(err, callback)) return; callback(); }); -}; +}); // set in db Pad.prototype.setPublicStatus = function setPublicStatus(publicStatus) { diff --git a/src/node/db/PadManager.js b/src/node/db/PadManager.js index b198de81c..469a7f496 100644 --- a/src/node/db/PadManager.js +++ b/src/node/db/PadManager.js @@ -22,6 +22,7 @@ var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); var Pad = require("../db/Pad").Pad; var db = require("./DB").db; +const thenify = require("thenify").withCallback; /** * A cache of all loaded Pads. @@ -53,7 +54,7 @@ var padList = { list: [], sorted : false, initiated: false, - init: function(cb) { + init: thenify(function(cb) { db.findKeys("pad:*", "*:*:*", function(err, dbData) { if (ERR(err, cb)) return; @@ -68,18 +69,18 @@ var padList = { }); return this; - }, - load: function(cb) { + }), + load: thenify(function(cb) { if (this.initiated) { cb && cb(); } else { this.init(cb); } - }, + }), /** * Returns all pads in alphabetical order as array. */ - getPads: function(cb) { + getPads: thenify(function(cb) { this.load(function() { if (!padList.sorted) { padList.list = padList.list.sort(); @@ -88,7 +89,7 @@ var padList = { cb && cb(padList.list); }) - }, + }), addPad: function(name) { if (!this.initiated) return; @@ -125,7 +126,7 @@ var padIdTransforms = [ * @param id A String with the id of the pad * @param {Function} callback */ -exports.getPad = function(id, text, callback) +exports.getPad = thenify(function(id, text, callback) { // check if this is a valid padId if (!exports.isValidPadId(id)) { @@ -177,17 +178,17 @@ exports.getPad = function(id, text, callback) padList.addPad(id); callback(null, pad); }); -} +}); -exports.listAllPads = function(cb) +exports.listAllPads = thenify(function(cb) { padList.getPads(function(list) { cb && cb(null, {padIDs: list}); }); -} +}); // checks if a pad exists -exports.doesPadExists = function(padId, callback) +exports.doesPadExists = thenify(function(padId, callback) { db.get("pad:" + padId, function(err, value) { if (ERR(err, callback)) return; @@ -198,10 +199,10 @@ exports.doesPadExists = function(padId, callback) callback(null, false); } }); -} +}); // returns a sanitized padId, respecting legacy pad id formats -exports.sanitizePadId = function(padId, callback) { +function sanitizePadId(padId, callback) { var transform_index = arguments[2] || 0; // we're out of possible transformations, so just return it @@ -228,10 +229,19 @@ exports.sanitizePadId = function(padId, callback) { } // check the next transform - exports.sanitizePadId(transformedPadId, callback, transform_index); + sanitizePadId(transformedPadId, callback, transform_index); }); } +// sanitizePadId can't use thenify: single arg callback +exports.sanitizePadId = function(padId, callback) { + if (callback) { + return sanitizePadId(padId, callback); + } else { + return new Promise(resolve => sanitizePadId(padId, resolve)); + } +} + exports.isValidPadId = function(padId) { return /^(g.[a-zA-Z0-9]{16}\$)?[^$]{1,50}$/.test(padId); diff --git a/src/node/db/ReadOnlyManager.js b/src/node/db/ReadOnlyManager.js index ac7bee045..14c6f75c7 100644 --- a/src/node/db/ReadOnlyManager.js +++ b/src/node/db/ReadOnlyManager.js @@ -23,12 +23,13 @@ var ERR = require("async-stacktrace"); var db = require("./DB").db; var async = require("async"); var randomString = require("../utils/randomstring"); +const thenify = require("thenify").withCallback; /** * returns a read only id for a pad * @param {String} padId the id of the pad */ -exports.getReadOnlyId = function (padId, callback) +exports.getReadOnlyId = thenify(function (padId, callback) { var readOnlyId; @@ -59,22 +60,22 @@ exports.getReadOnlyId = function (padId, callback) // return the results callback(null, readOnlyId); }) -} +}); /** * returns the padId for a read only id * @param {String} readOnlyId read only id */ -exports.getPadId = function(readOnlyId, callback) +exports.getPadId = thenify(function(readOnlyId, callback) { db.get("readonly2pad:" + readOnlyId, callback); -} +}); /** * returns the padId and readonlyPadId in an object for any id * @param {String} padIdOrReadonlyPadId read only id or real pad id */ -exports.getIds = function(id, callback) { +exports.getIds = thenify(function(id, callback) { if (id.indexOf("r.") == 0) { exports.getPadId(id, function (err, value) { if (ERR(err, callback)) return; @@ -94,4 +95,4 @@ exports.getIds = function(id, callback) { }); }); } -} +}); diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index 28ad4e036..f1f759da6 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -25,8 +25,9 @@ var db = require("./DB").db; var async = require("async"); var groupManager = require("./GroupManager"); var authorManager = require("./AuthorManager"); +const thenify = require("thenify").withCallback; -exports.doesSessionExist = function(sessionID, callback) +exports.doesSessionExist = thenify(function(sessionID, callback) { //check if the database entry of this session exists db.get("session:" + sessionID, function (err, session) @@ -34,12 +35,12 @@ exports.doesSessionExist = function(sessionID, callback) if(ERR(err, callback)) return; callback(null, session != null); }); -} +}); /** * Creates a new session between an author and a group */ -exports.createSession = function(groupID, authorID, validUntil, callback) +exports.createSession = thenify(function(groupID, authorID, validUntil, callback) { var sessionID; @@ -172,9 +173,9 @@ exports.createSession = function(groupID, authorID, validUntil, callback) // return error and sessionID callback(null, {sessionID: sessionID}); }) -} +}); -exports.getSessionInfo = function(sessionID, callback) +exports.getSessionInfo = thenify(function(sessionID, callback) { // check if the database entry of this session exists db.get("session:" + sessionID, function (err, session) @@ -189,12 +190,12 @@ exports.getSessionInfo = function(sessionID, callback) callback(null, session); } }); -} +}); /** * Deletes a session */ -exports.deleteSession = function(sessionID, callback) +exports.deleteSession = thenify(function(sessionID, callback) { var authorID, groupID; var group2sessions, author2sessions; @@ -267,9 +268,9 @@ exports.deleteSession = function(sessionID, callback) if(ERR(err, callback)) return; callback(); }) -} +}); -exports.listSessionsOfGroup = function(groupID, callback) +exports.listSessionsOfGroup = thenify(function(groupID, callback) { groupManager.doesGroupExist(groupID, function(err, exists) { @@ -283,9 +284,9 @@ exports.listSessionsOfGroup = function(groupID, callback) listSessionsWithDBKey("group2sessions:" + groupID, callback); } }); -} +}); -exports.listSessionsOfAuthor = function(authorID, callback) +exports.listSessionsOfAuthor = thenify(function(authorID, callback) { authorManager.doesAuthorExists(authorID, function(err, exists) { @@ -299,7 +300,7 @@ exports.listSessionsOfAuthor = function(authorID, callback) listSessionsWithDBKey("author2sessions:" + authorID, callback); } }); -} +}); // this function is basically the code listSessionsOfAuthor and listSessionsOfGroup has in common function listSessionsWithDBKey (dbkey, callback) diff --git a/src/package.json b/src/package.json index e3cb756a6..fa63b0710 100644 --- a/src/package.json +++ b/src/package.json @@ -57,6 +57,7 @@ "slide": "1.1.6", "socket.io": "2.1.1", "swagger-node-express": "2.1.3", + "thenify": "^3.3.0", "tinycon": "0.0.1", "ueberdb2": "0.4.0", "uglify-js": "2.6.2", From ec5baa2ab337b2bd46375514e114cde141d7e2f3 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Tue, 22 Jan 2019 12:58:26 +0000 Subject: [PATCH 13/58] PadMessageHandler.js: convert two remaining API calls to thenify --- src/node/handler/PadMessageHandler.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index e32d0e61e..2edd9390e 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -38,6 +38,7 @@ var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); var channels = require("channels"); var stats = require('../stats'); var remoteAddress = require("../utils/RemoteAddress").remoteAddress; +const thenify = require("thenify").withCallback; /** * A associative array that saves informations about a session @@ -1788,16 +1789,16 @@ function _getRoomClients(padID) { /** * Get the number of users in a pad */ -exports.padUsersCount = function(padID, callback) { +exports.padUsersCount = thenify(function(padID, callback) { callback(null, { padUsersCount: _getRoomClients(padID).length }); -} +}); /** * Get the list of users in a pad */ -exports.padUsers = function(padID, callback) { +exports.padUsers = thenify(function(padID, callback) { var result = []; var roomClients = _getRoomClients(padID); @@ -1821,6 +1822,6 @@ exports.padUsers = function(padID, callback) { callback(null, {padUsers: result}); }); -} +}); exports.sessioninfos = sessioninfos; From c4f1f837478ab72899c5ead035d0d43069f064ba Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Tue, 22 Jan 2019 13:30:28 +0000 Subject: [PATCH 14/58] APIHandler.js: use promises --- src/node/handler/APIHandler.js | 50 +++++++++++++++------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/node/handler/APIHandler.js b/src/node/handler/APIHandler.js index 5d731ddc7..07b9b55c5 100644 --- a/src/node/handler/APIHandler.js +++ b/src/node/handler/APIHandler.js @@ -150,7 +150,7 @@ exports.version = version; * @req express request object * @res express response object */ -exports.handle = function(apiVersion, functionName, fields, req, res) +exports.handle = async function(apiVersion, functionName, fields, req, res) { //check if this is a valid apiversion var isKnownApiVersion = false; @@ -194,41 +194,38 @@ exports.handle = function(apiVersion, functionName, fields, req, res) return; } - // sanitize any padIDs before continuing - if (fields["padID"]) { - padManager.sanitizePadId(fields["padID"], function(padId) { - fields["padID"] = padId; - callAPI(apiVersion, functionName, fields, req, res); - }); - } else if (fields["padName"]) { - padManager.sanitizePadId(fields["padName"], function(padId) { - fields["padName"] = padId; - callAPI(apiVersion, functionName, fields, req, res); - }); - } else { - callAPI(apiVersion, functionName, fields, req, res); + try { + // sanitize any padIDs before continuing + if (fields["padID"]) { + fields["padID"] = await padManager.sanitizePadId(fields["padID"]); + } else if (fields["padName"]) { + fields["padName"] = await padManager.sanitizePadId(fields["padName"]); + } + await callAPI(apiVersion, functionName, fields, req, res); + } catch (e) { + ERR(e); } } // calls the api function -function callAPI(apiVersion, functionName, fields, req, res) +async function callAPI(apiVersion, functionName, fields, req, res) { // put the function parameters in an array var functionParams = version[apiVersion][functionName].map(function (field) { return fields[field] }); - // add a callback function to handle the response - functionParams.push(function(err, data) { - if (err == null) { - // no error happened, everything is fine + try { + // call the api function + let data = await api[functionName].apply(this, functionParams); - if (!data) { + if (!data) { data = null; - } + } - res.send({code: 0, message: "ok", data: data}); - } else if (err.name == "apierror") { + res.send({code: 0, message: "ok", data: data}); + } catch (err) { + if (err.name == "apierror") { // parameters were wrong and the api stopped execution, pass the error res.send({code: 1, message: err.message, data: null}); @@ -236,10 +233,7 @@ function callAPI(apiVersion, functionName, fields, req, res) // an unknown error happened res.send({code: 2, message: "internal error", data: null}); - ERR(err); + throw err; } - }); - - // call the api function - api[functionName].apply(this, functionParams); + } } From 5d7162ac9ae5f97ab4822c45557dc1226699ab25 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Tue, 22 Jan 2019 14:58:25 +0000 Subject: [PATCH 15/58] utils/ImportHtml.js: migrate to thenify --- src/node/utils/ImportHtml.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/node/utils/ImportHtml.js b/src/node/utils/ImportHtml.js index 04037eab5..d46b715d3 100644 --- a/src/node/utils/ImportHtml.js +++ b/src/node/utils/ImportHtml.js @@ -18,6 +18,7 @@ var log4js = require('log4js'); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var contentcollector = require("ep_etherpad-lite/static/js/contentcollector"); var cheerio = require("cheerio"); +const thenify = require("thenify").withCallback; function setPadHTML(pad, html, callback) { @@ -94,4 +95,4 @@ function setPadHTML(pad, html, callback) callback(null); } -exports.setPadHTML = setPadHTML; +exports.setPadHTML = thenify(setPadHTML); From 584e48143014faaa6ef38ed715a5763000c0c2a6 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Tue, 22 Jan 2019 15:48:29 +0000 Subject: [PATCH 16/58] PadMessageHandler.js: migrate to thenify --- src/node/handler/PadMessageHandler.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 2edd9390e..8fc43b156 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -61,7 +61,7 @@ stats.gauge('totalUsers', function() { /** * A changeset queue per pad that is processed by handleUserChanges() */ -var padChannels = new channels.channels(handleUserChanges); +var padChannels = new channels.channels(thenify(handleUserChanges)); /** * Saves the Socket class we need to send and receive data from the client @@ -330,7 +330,7 @@ function handleSaveRevisionMessage(client, message){ * @param msg {Object} the message we're sending * @param sessionID {string} the socketIO session to which we're sending this message */ -exports.handleCustomObjectMessage = function(msg, sessionID, cb) { +exports.handleCustomObjectMessage = thenify(function(msg, sessionID, cb) { if (msg.data.type === "CUSTOM") { if (sessionID){ // a sessionID is targeted: directly to this sessionID @@ -341,7 +341,8 @@ exports.handleCustomObjectMessage = function(msg, sessionID, cb) { } } cb(null, {}); -} +}); + /** * Handles a custom message (sent via HTTP API request) @@ -349,7 +350,7 @@ exports.handleCustomObjectMessage = function(msg, sessionID, cb) { * @param padID {Pad} the pad to which we're sending this message * @param msgString {String} the message we're sending */ -exports.handleCustomMessage = function(padID, msgString, cb) { +exports.handleCustomMessage = thenify(function(padID, msgString, cb) { var time = Date.now(); var msg = { type: 'COLLABROOM', @@ -361,7 +362,7 @@ exports.handleCustomMessage = function(padID, msgString, cb) { socketio.sockets.in(padID).json.send(msg); cb(null, {}); -} +}); /** * Handles a Chat Message From 5ef4a2d1d5eb66186b2c1766d3058ed8d537b624 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Tue, 22 Jan 2019 17:30:33 +0000 Subject: [PATCH 17/58] more thenify in node/utils/* --- src/node/utils/ExportEtherpad.js | 5 +++-- src/node/utils/ExportHtml.js | 7 ++++--- src/node/utils/ImportEtherpad.js | 5 +++-- src/node/utils/TidyHtml.js | 5 +++-- src/node/utils/padDiff.js | 25 +++++++++++++------------ 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/node/utils/ExportEtherpad.js b/src/node/utils/ExportEtherpad.js index a68ab0b2a..84fe80333 100644 --- a/src/node/utils/ExportEtherpad.js +++ b/src/node/utils/ExportEtherpad.js @@ -18,8 +18,9 @@ var async = require("async"); var db = require("../db/DB").db; var ERR = require("async-stacktrace"); +const thenify = require("thenify").withCallback; -exports.getPadRaw = function(padId, callback){ +exports.getPadRaw = thenify(function(padId, callback){ async.waterfall([ function(cb){ db.get("pad:"+padId, cb); @@ -69,4 +70,4 @@ exports.getPadRaw = function(padId, callback){ ], function(err, data){ callback(null, data); }); -} +}); diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index f001fe452..fb15d867e 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -25,6 +25,7 @@ var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); var eejs = require('ep_etherpad-lite/node/eejs'); var _analyzeLine = require('./ExportHelper')._analyzeLine; var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace; +const thenify = require("thenify").withCallback; function getPadHTML(pad, revNum, callback) { @@ -67,7 +68,7 @@ function getPadHTML(pad, revNum, callback) }); } -exports.getPadHTML = getPadHTML; +exports.getPadHTML = thenify(getPadHTML); exports.getHTMLFromAtext = getHTMLFromAtext; function getHTMLFromAtext(pad, atext, authorColors) @@ -459,7 +460,7 @@ function getHTMLFromAtext(pad, atext, authorColors) return pieces.join(''); } -exports.getPadHTMLDocument = function (padId, revNum, callback) +exports.getPadHTMLDocument = thenify(function (padId, revNum, callback) { padManager.getPad(padId, function (err, pad) { @@ -484,7 +485,7 @@ exports.getPadHTMLDocument = function (padId, revNum, callback) }); }); }); -}; +}); // copied from ACE var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; diff --git a/src/node/utils/ImportEtherpad.js b/src/node/utils/ImportEtherpad.js index 1ff8b9b04..138c42936 100644 --- a/src/node/utils/ImportEtherpad.js +++ b/src/node/utils/ImportEtherpad.js @@ -17,8 +17,9 @@ var log4js = require('log4js'); var async = require("async"); var db = require("../db/DB").db; +const thenify = require("thenify").withCallback; -exports.setPadRaw = function(padId, records, callback) +exports.setPadRaw = thenify(function(padId, records, callback) { records = JSON.parse(records); @@ -70,4 +71,4 @@ exports.setPadRaw = function(padId, records, callback) function() { callback(null, true); }); -} +}); diff --git a/src/node/utils/TidyHtml.js b/src/node/utils/TidyHtml.js index 5d4e6ed75..0f7119894 100644 --- a/src/node/utils/TidyHtml.js +++ b/src/node/utils/TidyHtml.js @@ -5,8 +5,9 @@ var log4js = require('log4js'); var settings = require('./Settings'); var spawn = require('child_process').spawn; +const thenify = require("thenify").withCallback; -exports.tidy = function(srcFile, callback) { +exports.tidy = thenify(function(srcFile, callback) { var logger = log4js.getLogger('TidyHtml'); // Don't do anything if Tidy hasn't been enabled @@ -38,4 +39,4 @@ exports.tidy = function(srcFile, callback) { return callback('Tidy died with exit code ' + code); } }); -}; +}); diff --git a/src/node/utils/padDiff.js b/src/node/utils/padDiff.js index 33938b19c..a801fdc84 100644 --- a/src/node/utils/padDiff.js +++ b/src/node/utils/padDiff.js @@ -1,6 +1,7 @@ var Changeset = require("../../static/js/Changeset"); var async = require("async"); var exportHtml = require('./ExportHtml'); +const thenify = require("thenify").withCallback; function PadDiff (pad, fromRev, toRev) { // check parameters @@ -78,7 +79,7 @@ PadDiff.prototype._isClearAuthorship = function(changeset) { return true; }; -PadDiff.prototype._createClearAuthorship = function(rev, callback) { +PadDiff.prototype._createClearAuthorship = thenify(function(rev, callback) { var self = this; this._pad.getInternalRevisionAText(rev, function(err, atext) { if (err) { @@ -92,9 +93,9 @@ PadDiff.prototype._createClearAuthorship = function(rev, callback) { callback(null, changeset); }); -}; +}); -PadDiff.prototype._createClearStartAtext = function(rev, callback) { +PadDiff.prototype._createClearStartAtext = thenify(function(rev, callback) { var self = this; // get the atext of this revision @@ -119,9 +120,9 @@ PadDiff.prototype._createClearStartAtext = function(rev, callback) { callback(null, newAText); }); }); -}; +}); -PadDiff.prototype._getChangesetsInBulk = function(startRev, count, callback) { +PadDiff.prototype._getChangesetsInBulk = thenify(function(startRev, count, callback) { var self = this; // find out which revisions we need @@ -150,7 +151,7 @@ PadDiff.prototype._getChangesetsInBulk = function(startRev, count, callback) { function(err) { callback(err, changesets, authors); }); -}; +}); PadDiff.prototype._addAuthors = function(authors) { var self = this; @@ -163,7 +164,7 @@ PadDiff.prototype._addAuthors = function(authors) { }); }; -PadDiff.prototype._createDiffAtext = function(callback) { +PadDiff.prototype._createDiffAtext = thenify(function(callback) { var self = this; var bulkSize = 100; @@ -237,9 +238,9 @@ PadDiff.prototype._createDiffAtext = function(callback) { } ); }); -}; +}); -PadDiff.prototype.getHtml = function(callback) { +PadDiff.prototype.getHtml = thenify(function(callback) { // cache the html if (this._html != null) { return callback(null, this._html); @@ -283,9 +284,9 @@ PadDiff.prototype.getHtml = function(callback) { function(err) { callback(err, html); }); -}; +}); -PadDiff.prototype.getAuthors = function(callback) { +PadDiff.prototype.getAuthors = thenify(function(callback) { var self = this; // check if html was already produced, if not produce it, this generates the author array at the same time @@ -300,7 +301,7 @@ PadDiff.prototype.getAuthors = function(callback) { } else { callback(null, self._authors); } -}; +}); PadDiff.prototype._extendChangesetWithAuthor = function(changeset, author, apool) { // unpack From 0c2d6625410b81dbd299ed92c5e97735580166de Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 12:24:53 +0000 Subject: [PATCH 18/58] plugins download and search: converted to Promises Also fixed a bug where the system would make a request to the central server for the plugin list for every search even if the list was already cached. --- src/node/hooks/express/adminplugins.js | 49 +++++++++++++------------- src/static/js/pluginfw/installer.js | 40 +++++++++++---------- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/node/hooks/express/adminplugins.js b/src/node/hooks/express/adminplugins.js index 4c94c5780..7cfb160b9 100644 --- a/src/node/hooks/express/adminplugins.js +++ b/src/node/hooks/express/adminplugins.js @@ -42,14 +42,10 @@ exports.socketio = function(hook_name, args, cb) { socket.emit("results:installed", {installed: installed}); }); - socket.on("checkUpdates", function() { + socket.on("checkUpdates", async function() { // Check plugins for updates - installer.getAvailablePlugins(/*maxCacheAge:*/ 60 * 10, function(er, results) { - if (er) { - console.warn(er); - socket.emit("results:updatable", {updatable: {}}); - return; - } + try { + let results = await installer.getAvailablePlugins(/*maxCacheAge:*/ 60 * 10); var updatable = _(plugins.plugins).keys().filter(function(plugin) { if (!results[plugin]) return false; @@ -61,27 +57,26 @@ exports.socketio = function(hook_name, args, cb) { }); socket.emit("results:updatable", {updatable: updatable}); - }); + } catch (er) { + console.warn(er); + + socket.emit("results:updatable", {updatable: {}}); + } }); - socket.on("getAvailable", function(query) { - installer.getAvailablePlugins(/*maxCacheAge:*/ false, function(er, results) { - if (er) { - console.error(er); - results = {}; - } - - socket.emit("results:available", results); - }); + socket.on("getAvailable", async function(query) { + try { + let results = await installer.getAvailablePlugins(/*maxCacheAge:*/ false); + socket.emit("results:available", results); + } catch (er) { + console.error(er); + socket.emit("results:available", {}); + } }); - socket.on("search", function(query) { - installer.search(query.searchTerm, /*maxCacheAge:*/ 60 * 10, function(er, results) { - if (er) { - console.error(er); - results = {}; - } - + socket.on("search", async function(query) { + try { + let results = await installer.search(query.searchTerm, /*maxCacheAge:*/ 60 * 10); var res = Object.keys(results) .map(function(pluginName) { return results[pluginName]; @@ -92,7 +87,11 @@ exports.socketio = function(hook_name, args, cb) { res = sortPluginList(res, query.sortBy, query.sortDir) .slice(query.offset, query.offset+query.limit); socket.emit("results:search", {results: res, query: query}); - }); + } catch (er) { + console.error(er); + + socket.emit("results:search", {results: {}, query: query}); + } }); socket.on("install", function(plugin_name) { diff --git a/src/static/js/pluginfw/installer.js b/src/static/js/pluginfw/installer.js index dbf8696ed..934d5f036 100644 --- a/src/static/js/pluginfw/installer.js +++ b/src/static/js/pluginfw/installer.js @@ -77,33 +77,35 @@ exports.install = function(plugin_name, cb) { exports.availablePlugins = null; var cacheTimestamp = 0; -exports.getAvailablePlugins = function(maxCacheAge, cb) { - request("https://static.etherpad.org/plugins.json", function(er, response, plugins){ - if (er) return cb && cb(er); +exports.getAvailablePlugins = function(maxCacheAge) { + var nowTimestamp = Math.round(Date.now() / 1000); - if (exports.availablePlugins && maxCacheAge && Math.round(+ new Date / 1000) - cacheTimestamp <= maxCacheAge) { - return cb && cb(null, exports.availablePlugins); + return new Promise(function(resolve, reject) { + // check cache age before making any request + if (exports.availablePlugins && maxCacheAge && (nowTimestamp - cacheTimestamp) <= maxCacheAge) { + return resolve(exports.availablePlugins); } - try { - plugins = JSON.parse(plugins); - } catch (err) { - console.error('error parsing plugins.json:', err); - plugins = []; - } + request("https://static.etherpad.org/plugins.json", function(er, response, plugins) { + if (er) return reject(er); - exports.availablePlugins = plugins; - cacheTimestamp = Math.round(+ new Date / 1000); + try { + plugins = JSON.parse(plugins); + } catch (err) { + console.error('error parsing plugins.json:', err); + plugins = []; + } - cb && cb(null, plugins); + exports.availablePlugins = plugins; + cacheTimestamp = nowTimestamp; + resolve(plugins); + }); }); }; -exports.search = function(searchTerm, maxCacheAge, cb) { - exports.getAvailablePlugins(maxCacheAge, function(er, results) { - if (er) return cb && cb(er); - +exports.search = function(searchTerm, maxCacheAge) { + return exports.getAvailablePlugins(maxCacheAge).then(function(results) { var res = {}; if (searchTerm) { @@ -127,6 +129,6 @@ exports.search = function(searchTerm, maxCacheAge, cb) { res[pluginName] = results[pluginName]; } - cb && cb(null, res); + return res; }); }; From 23a3a079a601ba8d8565420d5268ade11661bc7c Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 16:21:40 +0000 Subject: [PATCH 19/58] tests.js: remove use of async.js Use real `async` instead of async.js where applicable. The `getPluginTests()` function was never truly async anyway because it only contains calls to synchronous `fs` modules. --- src/node/hooks/express/tests.js | 73 +++++++++++++++------------------ 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/src/node/hooks/express/tests.js b/src/node/hooks/express/tests.js index b5c4ca56c..443e9f685 100644 --- a/src/node/hooks/express/tests.js +++ b/src/node/hooks/express/tests.js @@ -1,24 +1,19 @@ var path = require("path") , npm = require("npm") , fs = require("fs") - , async = require("async"); + , util = require("util"); exports.expressCreateServer = function (hook_name, args, cb) { - args.app.get('/tests/frontend/specs_list.js', function(req, res) { - async.parallel({ - coreSpecs: function(callback) { - exports.getCoreTests(callback); - }, - pluginSpecs: function(callback) { - exports.getPluginTests(callback); - } - }, - function(err, results) { - var files = results.coreSpecs; // push the core specs to a file object - files = files.concat(results.pluginSpecs); // add the plugin Specs to the core specs - console.debug("Sent browser the following test specs:", files.sort()); - res.send("var specs_list = " + JSON.stringify(files.sort()) + ";\n"); - }); + args.app.get('/tests/frontend/specs_list.js', async function(req, res) { + let [coreTests, pluginTests] = await Promise.all([ + exports.getCoreTests(), + exports.getPluginTests() + ]); + + // merge the two sets of results + let files = [].concat(coreTests, pluginTests).sort(); + console.debug("Sent browser the following test specs:", files); + res.send("var specs_list = " + JSON.stringify(files) + ";\n"); }); // path.join seems to normalize by default, but we'll just be explicit @@ -63,30 +58,30 @@ exports.expressCreateServer = function (hook_name, args, cb) { }); } -exports.getPluginTests = function(callback) { - var pluginSpecs = []; - var plugins = fs.readdirSync('node_modules'); - plugins.forEach(function(plugin) { - if (fs.existsSync("node_modules/" + plugin + "/static/tests/frontend/specs")) { - // if plugins exists - var specFiles = fs.readdirSync("node_modules/" + plugin + "/static/tests/frontend/specs/"); - async.forEach(specFiles, function(spec) { - // for each specFile push it to pluginSpecs - pluginSpecs.push("/static/plugins/" + plugin + "/static/tests/frontend/specs/" + spec); - }, - function(err) { - // blow up if something bad happens! - }); - } - }); - callback(null, pluginSpecs); +const readdir = util.promisify(fs.readdir); + +exports.getPluginTests = async function(callback) { + const moduleDir = "node_modules/"; + const specPath = "/static/tests/frontend/specs/"; + const staticDir = "/static/plugins/"; + + let pluginSpecs = []; + + let plugins = await readdir(moduleDir); + let promises = plugins + .map(plugin => [ plugin, moduleDir + plugin + specPath] ) + .filter(([plugin, specDir]) => fs.existsSync(specDir)) // check plugin exists + .map(([plugin, specDir]) => { + return readdir(specDir) + .then(specFiles => specFiles.map(spec => { + pluginSpecs.push(staticDir + plugin + specPath + spec); + })); + }); + + return Promise.all(promises).then(() => pluginSpecs); } -exports.getCoreTests = function(callback) { +exports.getCoreTests = function() { // get the core test specs - fs.readdir('tests/frontend/specs', function(err, coreSpecs) { - if (err) { return res.send(500); } - - callback(null, coreSpecs); - }); + return readdir('tests/frontend/specs'); } From 34fdaa4e8c656d13bd2cc33d5396cbe4aa7e8367 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 16:25:29 +0000 Subject: [PATCH 20/58] db/SecurityManager.js: convert checkAccess() to thenify --- src/node/db/SecurityManager.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index bc8f59426..e9aefcd61 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -27,6 +27,7 @@ var sessionManager = require("./SessionManager"); var settings = require("../utils/Settings"); var log4js = require('log4js'); var authLogger = log4js.getLogger("auth"); +const thenify = require("thenify").withCallback; /** * This function controlls the access to a pad, it checks if the user can access a pad. @@ -36,7 +37,7 @@ var authLogger = log4js.getLogger("auth"); * @param password the password the user has given to access this pad, can be null * @param callback will be called with (err, {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx}) */ -exports.checkAccess = function(padID, sessionCookie, token, password, callback) +exports.checkAccess = thenify(function(padID, sessionCookie, token, password, callback) { var statusObject; @@ -300,4 +301,4 @@ exports.checkAccess = function(padID, sessionCookie, token, password, callback) callback(null, statusObject); }); -}; +}); From d5d28717c47c40a4e91e74d3d8363fa543f08f63 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 16:29:36 +0000 Subject: [PATCH 21/58] access controls: promisification `getPadAccess()` (src/node/padaccess.js) is now "promise only", resolving to `true` or `false` as appropriate, and throwing an exception if there's an error. The two call sites (padreadonly.js and importexport.js) updated to match. --- src/node/hooks/express/importexport.js | 37 ++++++++--------- src/node/hooks/express/padreadonly.js | 57 ++++++-------------------- src/node/padaccess.js | 15 ++++--- 3 files changed, 39 insertions(+), 70 deletions(-) diff --git a/src/node/hooks/express/importexport.js b/src/node/hooks/express/importexport.js index fe97aa2f9..ef10e0145 100644 --- a/src/node/hooks/express/importexport.js +++ b/src/node/hooks/express/importexport.js @@ -5,12 +5,11 @@ var importHandler = require('../../handler/ImportHandler'); var padManager = require("../../db/PadManager"); exports.expressCreateServer = function (hook_name, args, cb) { - args.app.get('/p/:pad/:rev?/export/:type', function(req, res, next) { + args.app.get('/p/:pad/:rev?/export/:type', async function(req, res, next) { var types = ["pdf", "doc", "txt", "html", "odt", "etherpad"]; //send a 404 if we don't support this filetype if (types.indexOf(req.params.type) == -1) { - next(); - return; + return next(); } // if abiword is disabled, and this is a format we only support with abiword, output a message @@ -22,28 +21,26 @@ exports.expressCreateServer = function (hook_name, args, cb) { res.header("Access-Control-Allow-Origin", "*"); - hasPadAccess(req, res, function() { + if (await hasPadAccess(req, res)) { console.log('req.params.pad', req.params.pad); - padManager.doesPadExists(req.params.pad, function(err, exists) { - if (!exists) { - return next(); - } + let exists = await padManager.doesPadExists(req.params.pad); + if (!exists) { + return next(); + } - exportHandler.doExport(req, res, req.params.pad, req.params.type); - }); - }); + exportHandler.doExport(req, res, req.params.pad, req.params.type); + } }); // handle import requests - args.app.post('/p/:pad/import', function(req, res, next) { - hasPadAccess(req, res, function() { - padManager.doesPadExists(req.params.pad, function(err, exists) { - if (!exists) { - return next(); - } + args.app.post('/p/:pad/import', async function(req, res, next) { + if (await hasPadAccess(req, res)) { + let exists = await padManager.doesPadExists(req.params.pad); + if (!exists) { + return next(); + } - importHandler.doImport(req, res, req.params.pad); - }); - }); + importHandler.doImport(req, res, req.params.pad); + } }); } diff --git a/src/node/hooks/express/padreadonly.js b/src/node/hooks/express/padreadonly.js index 77b197c7c..f699e27e9 100644 --- a/src/node/hooks/express/padreadonly.js +++ b/src/node/hooks/express/padreadonly.js @@ -1,57 +1,26 @@ -var async = require('async'); -var ERR = require("async-stacktrace"); var readOnlyManager = require("../../db/ReadOnlyManager"); var hasPadAccess = require("../../padaccess"); var exporthtml = require("../../utils/ExportHtml"); exports.expressCreateServer = function (hook_name, args, cb) { // serve read only pad - args.app.get('/ro/:id', function(req, res) { - var html; - var padId; + args.app.get('/ro/:id', async function(req, res) { - async.series([ - // translate the read only pad to a padId - function(callback) { - readOnlyManager.getPadId(req.params.id, function(err, _padId) { - if(ERR(err, callback)) return; + // translate the read only pad to a padId + let padId = await readOnlyManager.getPadId(req.params.id); + if (padId == null) { + res.status(404).send('404 - Not Found'); + return; + } - padId = _padId; + // we need that to tell hasPadAcess about the pad + req.params.pad = padId; - // we need that to tell hasPadAcess about the pad - req.params.pad = padId; - - callback(); - }); - }, + if (await hasPadAccess(req, res)) { // render the html document - function(callback) { - // return if the there is no padId - if(padId == null) { - callback("notfound"); - return; - } - - hasPadAccess(req, res, function() { - // render the html document - exporthtml.getPadHTMLDocument(padId, null, function(err, _html) { - if(ERR(err, callback)) return; - html = _html; - callback(); - }); - }); - } - ], - function(err) { - // throw any unexpected error - if(err && err != "notfound") - ERR(err); - - if(err == "notfound") - res.status(404).send('404 - Not Found'); - else - res.send(html); - }); + html = await exporthtml.getPadHTMLDocument(padId, null); + res.send(html); + } }); } diff --git a/src/node/padaccess.js b/src/node/padaccess.js index a25ad642c..3449f7d16 100644 --- a/src/node/padaccess.js +++ b/src/node/padaccess.js @@ -1,17 +1,20 @@ -var ERR = require("async-stacktrace"); var securityManager = require('./db/SecurityManager'); // checks for padAccess -module.exports = function (req, res, callback) { - securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password, function(err, accessObj) { - if (ERR(err, callback)) return; +module.exports = async function (req, res) { + try { + let accessObj = await securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password); if (accessObj.accessStatus === "grant") { // there is access, continue - callback(); + return true; } else { // no access res.status(403).send("403 - Can't touch this"); + return false; } - }); + } catch (err) { + // @TODO - send internal server error here? + throw err; + } } From b699621e5a239292dc576fb2f928075daf76f243 Mon Sep 17 00:00:00 2001 From: muxator Date: Sat, 9 Feb 2019 00:05:21 +0100 Subject: [PATCH 22/58] padurlsanitize.js: invert a condition prior to refactoring Extracted from Ray's work. --- src/node/hooks/express/padurlsanitize.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/node/hooks/express/padurlsanitize.js b/src/node/hooks/express/padurlsanitize.js index a7fb9f33e..b5ce926ae 100644 --- a/src/node/hooks/express/padurlsanitize.js +++ b/src/node/hooks/express/padurlsanitize.js @@ -12,7 +12,10 @@ exports.expressCreateServer = function (hook_name, args, cb) { } padManager.sanitizePadId(padId, function(sanitizedPadId) { - if (sanitizedPadId != padId) { + if (sanitizedPadId === padId) { + // the pad id was fine, so just render it + next(); + } else { // the pad id was sanitized, so we redirect to the sanitized version var real_url = sanitizedPadId; real_url = encodeURIComponent(real_url); @@ -20,9 +23,6 @@ exports.expressCreateServer = function (hook_name, args, cb) { if ( query ) real_url += '?' + query; res.header('Location', real_url); res.status(302).send('You should be redirected to ' + real_url + ''); - } else { - // the pad id was fine, so just render it - next(); } }); }); From 96d875b4d15b01e1a48c2de8bae08ffacb802c0e Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 16:36:28 +0000 Subject: [PATCH 23/58] padurlsanitize.js: rewritten to consume promises --- src/node/hooks/express/padurlsanitize.js | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/node/hooks/express/padurlsanitize.js b/src/node/hooks/express/padurlsanitize.js index b5ce926ae..ad8d3c431 100644 --- a/src/node/hooks/express/padurlsanitize.js +++ b/src/node/hooks/express/padurlsanitize.js @@ -4,26 +4,26 @@ var url = require('url'); exports.expressCreateServer = function (hook_name, args, cb) { // redirects browser to the pad's sanitized url if needed. otherwise, renders the html - args.app.param('pad', function (req, res, next, padId) { + args.app.param('pad', async function (req, res, next, padId) { // ensure the padname is valid and the url doesn't end with a / if (!padManager.isValidPadId(padId) || /\/$/.test(req.url)) { res.status(404).send('Such a padname is forbidden'); return; } - padManager.sanitizePadId(padId, function(sanitizedPadId) { - if (sanitizedPadId === padId) { - // the pad id was fine, so just render it - next(); - } else { - // the pad id was sanitized, so we redirect to the sanitized version - var real_url = sanitizedPadId; - real_url = encodeURIComponent(real_url); - var query = url.parse(req.url).query; - if ( query ) real_url += '?' + query; - res.header('Location', real_url); - res.status(302).send('You should be redirected to ' + real_url + ''); - } - }); + let sanitizedPadId = await padManager.sanitizePadId(padId); + + if (sanitizedPadId === padId) { + // the pad id was fine, so just render it + next(); + } else { + // the pad id was sanitized, so we redirect to the sanitized version + var real_url = sanitizedPadId; + real_url = encodeURIComponent(real_url); + var query = url.parse(req.url).query; + if ( query ) real_url += '?' + query; + res.header('Location', real_url); + res.status(302).send('You should be redirected to ' + real_url + ''); + } }); } From 630af9af7d673b8547e72bc8e3296dc7207b6c28 Mon Sep 17 00:00:00 2001 From: muxator Date: Sat, 9 Feb 2019 00:14:53 +0100 Subject: [PATCH 24/58] db/SessionStore.js: call nextTick() only if there is something to do Changed two occurrences of: process.nextTick(function() { if (fn) fn(); }); with if (fn) { process.nextTick(fn); } i.e. such that no function even gets added to the `nextTick` queue unless there's actually a function to be called. Extracted from Ray's work. --- src/node/db/SessionStore.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/node/db/SessionStore.js b/src/node/db/SessionStore.js index 80d35f290..b9686b19b 100644 --- a/src/node/db/SessionStore.js +++ b/src/node/db/SessionStore.js @@ -36,18 +36,18 @@ SessionStore.prototype.set = function(sid, sess, fn) { messageLogger.debug('SET ' + sid); db.set("sessionstorage:" + sid, sess); - process.nextTick(function() { - if (fn) fn(); - }); + if (fn) { + process.nextTick(fn); + } }; SessionStore.prototype.destroy = function(sid, fn) { messageLogger.debug('DESTROY ' + sid); db.remove("sessionstorage:" + sid); - process.nextTick(function() { - if (fn) fn(); - }); + if (fn) { + process.nextTick(fn); + } }; SessionStore.prototype.all = function(fn) { From 583ea92aaf8bd9b450572f4feb306f6f411b10da Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 16:58:43 +0000 Subject: [PATCH 25/58] db/SessionStore.js: do not migrate to Promises. Make optional all(), clear() and length() 1. This module was not migrated to Promises, because it is only used via express-session, which can't actually use promises anyway. 2. all(), clear() and length() depend on the presence of the `db.forEach()` function, which in ueberdb2 doesn't even exist. Fortunately those three methods are optional, so I made their existence conditional on the presence of `db.forEach`. 3. in SessionStore.clear(), replaced a call to db.db.remove() with db.remove() --- src/node/db/SessionStore.js | 76 +++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/src/node/db/SessionStore.js b/src/node/db/SessionStore.js index b9686b19b..647cbbc8d 100644 --- a/src/node/db/SessionStore.js +++ b/src/node/db/SessionStore.js @@ -2,6 +2,9 @@ * Stores session data in the database * Source; https://github.com/edy-b/SciFlowWriter/blob/develop/available_plugins/ep_sciflowwriter/db/DirtyStore.js * This is not used for authors that are created via the API at current + * + * RPB: this module was not migrated to Promises, because it is only used via + * express-session, which can't actually use promises anyway. */ var Store = require('ep_etherpad-lite/node_modules/express-session').Store, @@ -50,39 +53,46 @@ SessionStore.prototype.destroy = function(sid, fn) { } }; -SessionStore.prototype.all = function(fn) { - messageLogger.debug('ALL'); +/* + * RPB: the following methods are optional requirements for a compatible session + * store for express-session, but in any case appear to depend on a + * non-existent feature of ueberdb2 + */ +if (db.forEach) { + SessionStore.prototype.all = function(fn) { + messageLogger.debug('ALL'); - var sessions = []; + var sessions = []; - db.forEach(function(key, value) { - if (key.substr(0,15) === "sessionstorage:") { - sessions.push(value); - } - }); - fn(null, sessions); -}; - -SessionStore.prototype.clear = function(fn) { - messageLogger.debug('CLEAR'); - - db.forEach(function(key, value) { - if (key.substr(0,15) === "sessionstorage:") { - db.db.remove("session:" + key); - } - }); - if (fn) fn(); -}; - -SessionStore.prototype.length = function(fn) { - messageLogger.debug('LENGTH'); - - var i = 0; - - db.forEach(function(key, value) { - if (key.substr(0,15) === "sessionstorage:") { - i++; - } - }); - fn(null, i); + db.forEach(function(key, value) { + if (key.substr(0,15) === "sessionstorage:") { + sessions.push(value); + } + }); + fn(null, sessions); + }; + + SessionStore.prototype.clear = function(fn) { + messageLogger.debug('CLEAR'); + + db.forEach(function(key, value) { + if (key.substr(0,15) === "sessionstorage:") { + db.remove("session:" + key); + } + }); + if (fn) fn(); + }; + + SessionStore.prototype.length = function(fn) { + messageLogger.debug('LENGTH'); + + var i = 0; + + db.forEach(function(key, value) { + if (key.substr(0,15) === "sessionstorage:") { + i++; + } + }); + fn(null, i); + } }; From 29e9f86cad2763b215f2111290b4e7da4ecdf1e8 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 23 Jan 2019 18:08:47 +0000 Subject: [PATCH 26/58] db/DB.js: add Promise-only API methods Promisified methods: - get() - set() - findKeys() - getSub() - setSub() - remove() - doShutdown() --- src/node/db/DB.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/node/db/DB.js b/src/node/db/DB.js index e6ddd5fb3..c462ac303 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -23,6 +23,7 @@ var ueberDB = require("ueberdb2"); var settings = require("../utils/Settings"); var log4js = require('log4js'); const thenify = require("thenify").withCallback; +const util = require("util"); // set database settings var db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB")); @@ -47,6 +48,12 @@ exports.init = thenify(function (callback) { } else { // everything ok exports.db = db; + + // set up Promise-based methods + ['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove', 'doShutdown'].forEach(fn => { + exports[fn] = util.promisify(db[fn].bind(db)); + }); + callback(null); } }); From 70a045ad3cad5acd7617ffd46089d1cc9f75e18a Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 12:56:57 +0000 Subject: [PATCH 27/58] db/GroupManager.js: mostly converted to Promises / async --- src/node/db/GroupManager.js | 341 +++++++++++------------------------- 1 file changed, 102 insertions(+), 239 deletions(-) diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index d72efdff1..29a7ff063 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -21,305 +21,168 @@ var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; -var db = require("./DB").db; -var async = require("async"); +var db = require("./DB"); var padManager = require("./PadManager"); var sessionManager = require("./SessionManager"); const thenify = require("thenify").withCallback; -exports.listAllGroups = thenify(function(callback) { - db.get("groups", function (err, groups) { - if (ERR(err, callback)) return; - - if (groups == null) { - // there are no groups - callback(null, {groupIDs: []}); - - return; - } - - var groupIDs = []; - for (var groupID in groups) { - groupIDs.push(groupID); - } - callback(null, {groupIDs: groupIDs}); - }); -}); - -exports.deleteGroup = thenify(function(groupID, callback) +exports.listAllGroups = async function() { - var group; + let groups = await db.get("groups"); + groups = groups || {}; - async.series([ - // ensure group exists - function (callback) { - // try to get the group entry - db.get("group:" + groupID, function (err, _group) { - if (ERR(err, callback)) return; + let groupIDs = Object.keys(groups); + return { groupIDs }; +} - if (_group == null) { - // group does not exist - callback(new customError("groupID does not exist", "apierror")); - return; - } +exports.deleteGroup = async function(groupID) +{ + let group = await db.get("group:" + groupID); - // group exists, everything is fine - group = _group; - callback(); - }); - }, + // ensure group exists + if (group == null) { + // group does not exist + throw new customError("groupID does not exist", "apierror"); + } - // iterate through all pads of this group and delete them - function(callback) { - // collect all padIDs in an array, that allows us to use async.forEach - var padIDs = []; - for(var i in group.pads) { - padIDs.push(i); - } + // iterate through all pads of this group and delete them + for (let padID in group.pads) { + let pad = await padManager.getPad(padID); + await pad.remove(); + } - // loop through all pads and delete them - async.forEach(padIDs, function(padID, callback) { - padManager.getPad(padID, function(err, pad) { - if (ERR(err, callback)) return; + // iterate through group2sessions and delete all sessions + let group2sessions = await db.get("group2sessions:" + groupID); + let sessions = group2sessions ? group2sessions.sessionsIDs : []; - pad.remove(callback); - }); - }, callback); - }, + // loop through all sessions and delete them + for (let session in sessions) { + await sessionManager.deleteSession(session); + } - // iterate through group2sessions and delete all sessions - function(callback) - { - // try to get the group entry - db.get("group2sessions:" + groupID, function (err, group2sessions) { - if (ERR(err, callback)) return; + // remove group and group2sessions entry + await db.remove("group2sessions:" + groupID); + await db.remove("group:" + groupID); - // skip if there is no group2sessions entry - if (group2sessions == null) { callback(); return } + // unlist the group + let groups = await exports.listAllGroups(); + groups = groups ? groups.groupIDs : []; - // collect all sessions in an array, that allows us to use async.forEach - var sessions = []; - for (var i in group2sessions.sessionsIDs) { - sessions.push(i); - } + let index = groups.indexOf(groupID); - // loop through all sessions and delete them - async.forEach(sessions, function(session, callback) { - sessionManager.deleteSession(session, callback); - }, callback); - }); - }, + if (index === -1) { + // it's not listed - // remove group and group2sessions entry - function(callback) { - db.remove("group2sessions:" + groupID); - db.remove("group:" + groupID); - callback(); - }, + return; + } - // unlist the group - function(callback) { - exports.listAllGroups(function(err, groups) { - if (ERR(err, callback)) return; - groups = groups? groups.groupIDs : []; + // remove from the list + groups.splice(index, 1); - let index = groups.indexOf(groupID); - - if (index === -1) { - // it's not listed - callback(); - - return; - } - - // remove from the list - groups.splice(index, 1); - - // store empty group list - if (groups.length == 0) { - db.set("groups", {}); - callback(); - return; - } - - // regenerate group list - var newGroups = {}; - async.forEach(groups, function(group, cb) { - newGroups[group] = 1; - cb(); - }, - function() { - db.set("groups", newGroups); - callback(); - }); - }); - } - ], - function(err) { - if (ERR(err, callback)) return; - callback(); - }); -}); + // regenerate group list + var newGroups = {}; + groups.forEach(group => newGroups[group] = 1); + await db.set("groups", newGroups); +} +// @TODO: this is the only function still called with a callback exports.doesGroupExist = thenify(function(groupID, callback) { // try to get the group entry - db.get("group:" + groupID, function (err, group) { + db.db.get("group:" + groupID, function (err, group) { if (ERR(err, callback)) return; callback(null, group != null); }); }); -exports.createGroup = thenify(function(callback) +exports.createGroup = async function() { // search for non existing groupID var groupID = "g." + randomString(16); // create the group - db.set("group:" + groupID, {pads: {}}); + await db.set("group:" + groupID, {pads: {}}); // list the group - exports.listAllGroups(function(err, groups) { - if (ERR(err, callback)) return; + let groups = await exports.listAllGroups(); + groups = groups? groups.groupIDs : []; + groups.push(groupID); - groups = groups? groups.groupIDs : []; - groups.push(groupID); + // regenerate group list + var newGroups = {}; + groups.forEach(group => newGroups[group] = 1); + await db.set("groups", newGroups); - // regenerate group list - var newGroups = {}; - async.forEach(groups, function(group, cb) { - newGroups[group] = 1; - cb(); - }, - function() { - db.set("groups", newGroups); - callback(null, {groupID: groupID}); - }); - }); -}); + return { groupID }; +} -exports.createGroupIfNotExistsFor = thenify(function(groupMapper, callback) +exports.createGroupIfNotExistsFor = async function(groupMapper) { // ensure mapper is optional if (typeof groupMapper !== "string") { - callback(new customError("groupMapper is not a string", "apierror")); - return; + throw new customError("groupMapper is not a string", "apierror"); } // try to get a group for this mapper - db.get("mapper2group:" + groupMapper, function(err, groupID) { - function createGroupForMapper(cb) { - exports.createGroup(function(err, responseObj) { - if (ERR(err, cb)) return; + let groupID = await db.get("mapper2group:" + groupMapper); - // create the mapper entry for this group - db.set("mapper2group:" + groupMapper, responseObj.groupID); + if (groupID) { + // there is a group for this mapper + let exists = await exports.doesGroupExist(groupID); - cb(null, responseObj); - }); - } + if (exists) return { groupID }; + } - if (ERR(err, callback)) return; + // hah, the returned group doesn't exist, let's create one + let result = await exports.createGroup(); - if (groupID) { - // there is a group for this mapper - exports.doesGroupExist(groupID, function(err, exists) { - if (ERR(err, callback)) return; + // create the mapper entry for this group + await db.set("mapper2group:" + groupMapper, result.groupID); - if (exists) return callback(null, {groupID: groupID}); + return result; +} - // hah, the returned group doesn't exist, let's create one - createGroupForMapper(callback) - }); - - return; - } - - // there is no group for this mapper, let's create a group - createGroupForMapper(callback) - }); -}); - -exports.createGroupPad = thenify(function(groupID, padName, text, callback) +exports.createGroupPad = async function(groupID, padName, text) { // create the padID - var padID = groupID + "$" + padName; + let padID = groupID + "$" + padName; - async.series([ - // ensure group exists - function (callback) { - exports.doesGroupExist(groupID, function(err, exists) { - if (ERR(err, callback)) return; + // ensure group exists + let groupExists = await exports.doesGroupExist(groupID); - if (!exists) { - // group does not exist - callback(new customError("groupID does not exist", "apierror")); - return; - } + if (!groupExists) { + throw new customError("groupID does not exist", "apierror"); + } - // group exists, everything is fine - callback(); - }); - }, + // ensure pad doesn't exist already + let padExists = await padManager.doesPadExists(padID); - // ensure pad doesn't exist already - function (callback) { - padManager.doesPadExists(padID, function(err, exists) { - if (ERR(err, callback)) return; + if (padExists) { + // pad exists already + throw new customError("padName does already exist", "apierror"); + } - if (exists == true) { - // pad exists already - callback(new customError("padName does already exist", "apierror")); - return; - } + // create the pad + await padManager.getPad(padID, text); - // pad does not exist, everything is fine - callback(); - }); - }, + //create an entry in the group for this pad + await db.setSub("group:" + groupID, ["pads", padID], 1); - // create the pad - function (callback) { - padManager.getPad(padID, text, function(err) { - if (ERR(err, callback)) return; + return { padID }; +} - callback(); - }); - }, - - // create an entry in the group for this pad - function (callback) { - db.setSub("group:" + groupID, ["pads", padID], 1); - callback(); - } - ], - function(err) { - if (ERR(err, callback)) return; - - callback(null, {padID: padID}); - }); -}); - -exports.listPads = thenify(function(groupID, callback) +exports.listPads = async function(groupID) { - exports.doesGroupExist(groupID, function(err, exists) { - if (ERR(err, callback)) return; + let exists = await exports.doesGroupExist(groupID); - // ensure the group exists - if (!exists) { - callback(new customError("groupID does not exist", "apierror")); - return; - } + // ensure the group exists + if (!exists) { + throw new customError("groupID does not exist", "apierror"); + } - // group exists, let's get the pads - db.getSub("group:" + groupID, ["pads"], function(err, result) { - if (ERR(err, callback)) return; + // group exists, let's get the pads + let result = await db.getSub("group:" + groupID, ["pads"]); + let padIDs = Object.keys(result); - var pads = []; - for ( var padId in result ) { - pads.push(padId); - } - callback(null, {padIDs: pads}); - }); - }); -}); + return { padIDs }; +} From 16c4c33f49d5fb3e4fa0124e4d59c0ccce42dfde Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 13:37:24 +0000 Subject: [PATCH 28/58] db/AuthorManager.js: renamed doesAuthorExists() -> doesAuthorExist() Removed the 's' for consistency with the other `doesFooExist()` manager calls. Retained an alias for plugins that might be using it. --- src/node/db/AuthorManager.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index 0db187706..0676b1b8a 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -40,7 +40,7 @@ exports.getColorPalette = function() { /** * Checks if the author exists */ -exports.doesAuthorExists = thenify(function(authorID, callback) +exports.doesAuthorExist = thenify(function(authorID, callback) { // check if the database entry of this author exists db.get("globalAuthor:" + authorID, function(err, author) { @@ -50,6 +50,9 @@ exports.doesAuthorExists = thenify(function(authorID, callback) }); }); +/* exported for backwards compatibility */ +exports.doesAuthorExists = exports.doesAuthorExist; + /** * Returns the AuthorID for a token. * @param {String} token The token From a875ca6c3089a6a3f02f3fa0403aba302f42daa5 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 14:53:24 +0000 Subject: [PATCH 29/58] db/SessionManager.js: mostly converted to Promises --- src/node/db/SessionManager.js | 394 ++++++++++++---------------------- 1 file changed, 133 insertions(+), 261 deletions(-) diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index f1f759da6..67d3e3816 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -21,160 +21,106 @@ var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); var randomString = require("../utils/randomstring"); -var db = require("./DB").db; -var async = require("async"); +var db = require("./DB"); var groupManager = require("./GroupManager"); var authorManager = require("./AuthorManager"); const thenify = require("thenify").withCallback; -exports.doesSessionExist = thenify(function(sessionID, callback) +exports.doesSessionExist = async function(sessionID) { //check if the database entry of this session exists - db.get("session:" + sessionID, function (err, session) - { - if(ERR(err, callback)) return; - callback(null, session != null); - }); -}); + let session = await db.get("session:" + sessionID); + return (session !== null); +} /** * Creates a new session between an author and a group */ -exports.createSession = thenify(function(groupID, authorID, validUntil, callback) +exports.createSession = async function(groupID, authorID, validUntil) { - var sessionID; + // check if the group exists + let groupExists = await groupManager.doesGroupExist(groupID); + if (!groupExists) { + throw new customError("groupID does not exist", "apierror"); + } - async.series([ - // check if the group exists - function(callback) - { - groupManager.doesGroupExist(groupID, function(err, exists) - { - if(ERR(err, callback)) return; + // check if the author exists + let authorExists = await authorManager.doesAuthorExist(authorID); + if (!authorExists) { + throw new customError("authorID does not exist", "apierror"); + } - // group does not exist - if (exists == false) { - callback(new customError("groupID does not exist", "apierror")); - } else { - // everything is fine, continue - callback(); - } - }); - }, + // try to parse validUntil if it's not a number + if (typeof validUntil !== "number") { + validUntil = parseInt(validUntil); + } - // check if the author exists - function(callback) - { - authorManager.doesAuthorExists(authorID, function(err, exists) - { - if(ERR(err, callback)) return; + // check it's a valid number + if (isNaN(validUntil)) { + throw new customError("validUntil is not a number", "apierror"); + } - if (exists == false) { - // author does not exist - callback(new customError("authorID does not exist", "apierror")); - } else { - // everything is fine, continue - callback(); - } - }); - }, + // ensure this is not a negative number + if (validUntil < 0) { + throw new customError("validUntil is a negative number", "apierror"); + } - // check validUntil and create the session db entry - function(callback) - { - // check if rev is a number - if (typeof validUntil != "number") - { - // try to parse the number - if (isNaN(parseInt(validUntil))) { - callback(new customError("validUntil is not a number", "apierror")); - return; - } + // ensure this is not a float value + if (!is_int(validUntil)) { + throw new customError("validUntil is a float value", "apierror"); + } - validUntil = parseInt(validUntil); - } + // check if validUntil is in the future + if (validUntil < Math.floor(Date.now() / 1000)) { + throw new customError("validUntil is in the past", "apierror"); + } - // ensure this is not a negative number - if (validUntil < 0) { - callback(new customError("validUntil is a negative number", "apierror")); - return; - } + // generate sessionID + let sessionID = "s." + randomString(16); - // ensure this is not a float value - if (!is_int(validUntil)) { - callback(new customError("validUntil is a float value", "apierror")); - return; - } + // set the session into the database + await db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil}); - // check if validUntil is in the future - if (Math.floor(Date.now()/1000) > validUntil) { - callback(new customError("validUntil is in the past", "apierror")); - return; - } + // get the entry + let group2sessions = await db.get("group2sessions:" + groupID); - // generate sessionID - sessionID = "s." + randomString(16); + /* + * In some cases, the db layer could return "undefined" as well as "null". + * Thus, it is not possible to perform strict null checks on group2sessions. + * In a previous version of this code, a strict check broke session + * management. + * + * See: https://github.com/ether/etherpad-lite/issues/3567#issuecomment-468613960 + */ + if (!group2sessions || !group2sessions.sessionIDs) { + // the entry doesn't exist so far, let's create it + group2sessions = {sessionIDs : {}}; + } - // set the session into the database - db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil}); + // add the entry for this session + group2sessions.sessionIDs[sessionID] = 1; - callback(); - }, + // save the new element back + await db.set("group2sessions:" + groupID, group2sessions); - // set the group2sessions entry - function(callback) - { - // get the entry - db.get("group2sessions:" + groupID, function(err, group2sessions) - { - if(ERR(err, callback)) return; + // get the author2sessions entry + let author2sessions = await db.get("author2sessions:" + authorID); - if (group2sessions == null || group2sessions.sessionIDs == null) { - // the entry doesn't exist so far, let's create it - group2sessions = {sessionIDs : {}}; - } + if (author2sessions == null || author2sessions.sessionIDs == null) { + // the entry doesn't exist so far, let's create it + author2sessions = {sessionIDs : {}}; + } - // add the entry for this session - group2sessions.sessionIDs[sessionID] = 1; + // add the entry for this session + author2sessions.sessionIDs[sessionID] = 1; - // save the new element back - db.set("group2sessions:" + groupID, group2sessions); + //save the new element back + await db.set("author2sessions:" + authorID, author2sessions); - callback(); - }); - }, - - // set the author2sessions entry - function(callback) - { - // get the entry - db.get("author2sessions:" + authorID, function(err, author2sessions) - { - if(ERR(err, callback)) return; - - if (author2sessions == null || author2sessions.sessionIDs == null) { - // the entry doesn't exist so far, let's create it - author2sessions = {sessionIDs : {}}; - } - - // add the entry for this session - author2sessions.sessionIDs[sessionID] = 1; - - //save the new element back - db.set("author2sessions:" + authorID, author2sessions); - - callback(); - }); - } - ], function(err) - { - if(ERR(err, callback)) return; - - // return error and sessionID - callback(null, {sessionID: sessionID}); - }) -}); + return { sessionID }; +} +// @TODO once external dependencies are using Promises exports.getSessionInfo = thenify(function(sessionID, callback) { // check if the database entry of this session exists @@ -195,160 +141,86 @@ exports.getSessionInfo = thenify(function(sessionID, callback) /** * Deletes a session */ -exports.deleteSession = thenify(function(sessionID, callback) +exports.deleteSession = async function(sessionID) { - var authorID, groupID; - var group2sessions, author2sessions; + // ensure that the session exists + let session = await db.get("session:" + sessionID); + if (session == null) { + throw new customError("sessionID does not exist", "apierror"); + } - async.series([ - function(callback) - { - // get the session entry - db.get("session:" + sessionID, function (err, session) - { - if(ERR(err, callback)) return; + // everything is fine, use the sessioninfos + let groupID = session.groupID; + let authorID = session.authorID; - if (session == null) { - // session does not exist - callback(new customError("sessionID does not exist", "apierror")) - } else { - // everything is fine, use the sessioninfos - authorID = session.authorID; - groupID = session.groupID; + // get the group2sessions and author2sessions entries + let group2sessions = await db.get("group2sessions:" + groupID); + let author2sessions = await db.get("author2sessions:" + authorID); - callback(); - } - }); - }, + // remove the session + await db.remove("session:" + sessionID); - // get the group2sessions entry - function(callback) - { - db.get("group2sessions:" + groupID, function (err, _group2sessions) - { - if(ERR(err, callback)) return; - group2sessions = _group2sessions; - callback(); - }); - }, + // remove session from group2sessions + if (group2sessions != null) { // Maybe the group was already deleted + delete group2sessions.sessionIDs[sessionID]; + await db.set("group2sessions:" + groupID, group2sessions); + } - // get the author2sessions entry - function(callback) - { - db.get("author2sessions:" + authorID, function (err, _author2sessions) - { - if(ERR(err, callback)) return; - author2sessions = _author2sessions; - callback(); - }); - }, + // remove session from author2sessions + if (author2sessions != null) { // Maybe the author was already deleted + delete author2sessions.sessionIDs[sessionID]; + await db.set("author2sessions:" + authorID, author2sessions); + } +} - // remove the values from the database - function(callback) - { - //remove the session - db.remove("session:" + sessionID); - - // remove session from group2sessions - if(group2sessions != null) { // Maybe the group was already deleted - delete group2sessions.sessionIDs[sessionID]; - db.set("group2sessions:" + groupID, group2sessions); - } - - // remove session from author2sessions - if(author2sessions != null) { // Maybe the author was already deleted - delete author2sessions.sessionIDs[sessionID]; - db.set("author2sessions:" + authorID, author2sessions); - } - - callback(); - } - ], function(err) - { - if(ERR(err, callback)) return; - callback(); - }) -}); - -exports.listSessionsOfGroup = thenify(function(groupID, callback) +exports.listSessionsOfGroup = async function(groupID) { - groupManager.doesGroupExist(groupID, function(err, exists) - { - if(ERR(err, callback)) return; + // check that the group exists + let exists = await groupManager.doesGroupExist(groupID); + if (!exists) { + throw new customError("groupID does not exist", "apierror"); + } - if (exists == false) { - // group does not exist - callback(new customError("groupID does not exist", "apierror")); - } else { - // everything is fine, continue - listSessionsWithDBKey("group2sessions:" + groupID, callback); - } - }); -}); + let sessions = await listSessionsWithDBKey("group2sessions:" + groupID); + return sessions; +} -exports.listSessionsOfAuthor = thenify(function(authorID, callback) +exports.listSessionsOfAuthor = async function(authorID) { - authorManager.doesAuthorExists(authorID, function(err, exists) - { - if(ERR(err, callback)) return; + // check that the author exists + let exists = await authorManager.doesAuthorExist(authorID) + if (!exists) { + throw new customError("authorID does not exist", "apierror"); + } - if (exists == false) { - // group does not exist - callback(new customError("authorID does not exist", "apierror")); - } else { - // everything is fine, continue - listSessionsWithDBKey("author2sessions:" + authorID, callback); - } - }); -}); + let sessions = await listSessionsWithDBKey("author2sessions:" + authorID); + return sessions; +} // this function is basically the code listSessionsOfAuthor and listSessionsOfGroup has in common -function listSessionsWithDBKey (dbkey, callback) +// required to return null rather than an empty object if there are none +async function listSessionsWithDBKey(dbkey, callback) { - var sessions; + // get the group2sessions entry + let sessionObject = await db.get(dbkey); + let sessions = sessionObject ? sessionObject.sessionIDs : null; - async.series([ - function(callback) - { - // get the group2sessions entry - db.get(dbkey, function(err, sessionObject) - { - if(ERR(err, callback)) return; - sessions = sessionObject ? sessionObject.sessionIDs : null; - callback(); - }); - }, - - function(callback) - { - // collect all sessionIDs in an arrary - var sessionIDs = []; - for (var i in sessions) - { - sessionIDs.push(i); + // iterate through the sessions and get the sessioninfos + for (let sessionID in sessions) { + try { + let sessionInfo = await exports.getSessionInfo(sessionID); + sessions[sessionID] = sessionInfo; + } catch (err) { + if (err == "apierror: sessionID does not exist") { + console.warn(`Found bad session ${sessionID} in ${dbkey}`); + sessions[sessionID] = null; + } else { + throw err; } - - // iterate through the sessions and get the sessioninfos - async.forEach(sessionIDs, function(sessionID, callback) - { - exports.getSessionInfo(sessionID, function(err, sessionInfo) - { - if (err == "apierror: sessionID does not exist") { - console.warn(`Found bad session ${sessionID} in ${dbkey}`); - } else if(ERR(err, callback)) { - return; - } - - sessions[sessionID] = sessionInfo; - callback(); - }); - }, callback); } - ], function(err) - { - if(ERR(err, callback)) return; - callback(null, sessions); - }); + } + + return sessions; } // checks if a number is an int From bf9e3f92b5d3597758cba0f7461b5dec4bfbbc56 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 15:05:12 +0000 Subject: [PATCH 30/58] db/PadManager.js: renamed doesPadExists() -> doesPadExist() Removed the 's' for consistency with the other `doesFooExist()` manager calls. Retained an alias for plugins that might be using it. --- src/node/db/PadManager.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/node/db/PadManager.js b/src/node/db/PadManager.js index 469a7f496..d2747d313 100644 --- a/src/node/db/PadManager.js +++ b/src/node/db/PadManager.js @@ -188,7 +188,7 @@ exports.listAllPads = thenify(function(cb) }); // checks if a pad exists -exports.doesPadExists = thenify(function(padId, callback) +exports.doesPadExist = thenify(function(padId, callback) { db.get("pad:" + padId, function(err, value) { if (ERR(err, callback)) return; @@ -201,6 +201,9 @@ exports.doesPadExists = thenify(function(padId, callback) }); }); +// alias for backwards compatibility +exports.doesPadExists = exports.doesPadExist; + // returns a sanitized padId, respecting legacy pad id formats function sanitizePadId(padId, callback) { var transform_index = arguments[2] || 0; From eedae98e2fe133adf783c0d297c84c9787375649 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 15:15:16 +0000 Subject: [PATCH 31/58] db/PadManager.js: convert sanitizePadId() to Promises The function is now iterative rather than recursive. --- src/node/db/PadManager.js | 67 +++++++++++++-------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/src/node/db/PadManager.js b/src/node/db/PadManager.js index d2747d313..e478636b0 100644 --- a/src/node/db/PadManager.js +++ b/src/node/db/PadManager.js @@ -112,15 +112,6 @@ var padList = { // initialises the all-knowing data structure -/** - * An array of padId transformations. These represent changes in pad name policy over - * time, and allow us to "play back" these changes so legacy padIds can be found. - */ -var padIdTransforms = [ - [/\s+/g, '_'], - [/:+/g, '_'] -]; - /** * Returns a Pad Object with the callback * @param id A String with the id of the pad @@ -204,45 +195,31 @@ exports.doesPadExist = thenify(function(padId, callback) // alias for backwards compatibility exports.doesPadExists = exports.doesPadExist; +/** + * An array of padId transformations. These represent changes in pad name policy over + * time, and allow us to "play back" these changes so legacy padIds can be found. + */ +const padIdTransforms = [ + [/\s+/g, '_'], + [/:+/g, '_'] +]; + // returns a sanitized padId, respecting legacy pad id formats -function sanitizePadId(padId, callback) { - var transform_index = arguments[2] || 0; +exports.sanitizePadId = async function sanitizePadId(padId) { + for (let i = 0, n = padIdTransforms.length; i < n; ++i) { + let exists = await exports.doesPadExist(padId); + + if (exists) { + return padId; + } + + let [from, to] = padIdTransforms[i]; + + padId = padId.replace(from, to); + } // we're out of possible transformations, so just return it - if (transform_index >= padIdTransforms.length) { - callback(padId); - - return; - } - - // check if padId exists - exports.doesPadExists(padId, function(junk, exists) { - if (exists) { - callback(padId); - - return; - } - - // get the next transformation *that's different* - var transformedPadId = padId; - - while(transformedPadId == padId && transform_index < padIdTransforms.length) { - transformedPadId = padId.replace(padIdTransforms[transform_index][0], padIdTransforms[transform_index][1]); - transform_index += 1; - } - - // check the next transform - sanitizePadId(transformedPadId, callback, transform_index); - }); -} - -// sanitizePadId can't use thenify: single arg callback -exports.sanitizePadId = function(padId, callback) { - if (callback) { - return sanitizePadId(padId, callback); - } else { - return new Promise(resolve => sanitizePadId(padId, resolve)); - } + return padId; } exports.isValidPadId = function(padId) From 8f53e4407ed326ed96154b547e786c16a5c31d3f Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 15:47:25 +0000 Subject: [PATCH 32/58] db/AuthorManager.js: partial conversion to Promises --- src/node/db/AuthorManager.js | 134 +++++++++++++++-------------------- 1 file changed, 59 insertions(+), 75 deletions(-) diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index 0676b1b8a..d204b64ab 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -19,7 +19,7 @@ */ var ERR = require("async-stacktrace"); -var db = require("./DB").db; +var db = require("./DB"); var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; const thenify = require("thenify").withCallback; @@ -40,15 +40,12 @@ exports.getColorPalette = function() { /** * Checks if the author exists */ -exports.doesAuthorExist = thenify(function(authorID, callback) +exports.doesAuthorExist = async function(authorID) { - // check if the database entry of this author exists - db.get("globalAuthor:" + authorID, function(err, author) { - if (ERR(err, callback)) return; + let author = await db.get("globalAuthor:" + authorID); - callback(null, author !== null); - }); -}); + return author !== null; +} /* exported for backwards compatibility */ exports.doesAuthorExists = exports.doesAuthorExist; @@ -72,22 +69,18 @@ exports.getAuthor4Token = thenify(function(token, callback) * Returns the AuthorID for a mapper. * @param {String} token The mapper * @param {String} name The name of the author (optional) - * @param {Function} callback callback (err, author) */ -exports.createAuthorIfNotExistsFor = thenify(function(authorMapper, name, callback) +exports.createAuthorIfNotExistsFor = async function(authorMapper, name) { - mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author) { - if (ERR(err, callback)) return; + let author = await mapAuthorWithDBKey("mapper2author", authorMapper); - if (name) { - // set the name of this author - exports.setAuthorName(author.authorID, name); - } + if (name) { + // set the name of this author + await exports.setAuthorName(author.authorID, name); + } - // return the authorID - callback(null, author); - }); -}); + return author; +}; /** * Returns the AuthorID for a mapper. We can map using a mapperkey, @@ -96,10 +89,10 @@ exports.createAuthorIfNotExistsFor = thenify(function(authorMapper, name, callba * @param {String} mapper The mapper * @param {Function} callback callback (err, author) */ -function mapAuthorWithDBKey (mapperkey, mapper, callback) +let mapAuthorWithDBKey = thenify(function mapAuthorWithDBKey (mapperkey, mapper, callback) { // try to map to an author - db.get(mapperkey + ":" + mapper, function(err, author) { + db.db.get(mapperkey + ":" + mapper, function(err, author) { if (ERR(err, callback)) return; if (author === null) { @@ -108,7 +101,7 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) if (ERR(err, callback)) return; // create the token2author relation - db.set(mapperkey + ":" + mapper, author.authorID); + db.db.set(mapperkey + ":" + mapper, author.authorID); // return the author callback(null, author); @@ -119,12 +112,12 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) // there is an author with this mapper // update the timestamp of this author - db.setSub("globalAuthor:" + author, ["timestamp"], Date.now()); + db.db.setSub("globalAuthor:" + author, ["timestamp"], Date.now()); // return the author callback(null, {authorID: author}); }); -} +}); /** * Internal function that creates the database entry for an author @@ -143,7 +136,7 @@ exports.createAuthor = thenify(function(name, callback) }; // set the global author db entry - db.set("globalAuthor:" + author, authorObj); + db.db.set("globalAuthor:" + author, authorObj); callback(null, {authorID: author}); }); @@ -155,7 +148,7 @@ exports.createAuthor = thenify(function(name, callback) */ exports.getAuthor = thenify(function(author, callback) { - db.get("globalAuthor:" + author, callback); + db.db.get("globalAuthor:" + author, callback); }); /** @@ -165,7 +158,7 @@ exports.getAuthor = thenify(function(author, callback) */ exports.getAuthorColorId = thenify(function(author, callback) { - db.getSub("globalAuthor:" + author, ["colorId"], callback); + db.db.getSub("globalAuthor:" + author, ["colorId"], callback); }); /** @@ -176,7 +169,7 @@ exports.getAuthorColorId = thenify(function(author, callback) */ exports.setAuthorColorId = thenify(function(author, colorId, callback) { - db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback); + db.db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback); }); /** @@ -186,7 +179,7 @@ exports.setAuthorColorId = thenify(function(author, colorId, callback) */ exports.getAuthorName = thenify(function(author, callback) { - db.getSub("globalAuthor:" + author, ["name"], callback); + db.db.getSub("globalAuthor:" + author, ["name"], callback); }); /** @@ -197,15 +190,14 @@ exports.getAuthorName = thenify(function(author, callback) */ exports.setAuthorName = thenify(function(author, name, callback) { - db.setSub("globalAuthor:" + author, ["name"], name, callback); + db.db.setSub("globalAuthor:" + author, ["name"], name, callback); }); /** * Returns an array of all pads this author contributed to * @param {String} author The id of the author - * @param {Function} callback (optional) */ -exports.listPadsOfAuthor = thenify(function(authorID, callback) +exports.listPadsOfAuthor = async function(authorID) { /* There are two other places where this array is manipulated: * (1) When the author is added to a pad, the author object is also updated @@ -213,52 +205,45 @@ exports.listPadsOfAuthor = thenify(function(authorID, callback) */ // get the globalAuthor - db.get("globalAuthor:" + authorID, function(err, author) { - if (ERR(err, callback)) return; + let author = await db.get("globalAuthor:" + authorID); - if (author === null) { - // author does not exist - callback(new customError("authorID does not exist", "apierror")); + if (author === null) { + // author does not exist + throw new customError("authorID does not exist", "apierror"); + } - return; - } + // everything is fine, return the pad IDs + let padIDs = Object.keys(author.padIDs || {}); - // everything is fine, return the pad IDs - var pads = []; - - if (author.padIDs != null) { - for (var padId in author.padIDs) { - pads.push(padId); - } - } - - callback(null, {padIDs: pads}); - }); -}); + return { padIDs }; +} /** * Adds a new pad to the list of contributions * @param {String} author The id of the author * @param {String} padID The id of the pad the author contributes to */ -exports.addPad = function(authorID, padID) +exports.addPad = async function(authorID, padID) { // get the entry - db.get("globalAuthor:" + authorID, function(err, author) { - if (ERR(err)) return; - if (author === null) return; + let author = await db.get("globalAuthor:" + authorID); - if (author.padIDs == null) { - // the entry doesn't exist so far, let's create it - author.padIDs = {}; - } + if (author === null) return; - // add the entry for this pad - author.padIDs[padID] = 1; // anything, because value is not used + /* + * ACHTUNG: padIDs can also be undefined, not just null, so it is not possible + * to perform a strict check here + */ + if (!author.padIDs) { + // the entry doesn't exist so far, let's create it + author.padIDs = {}; + } - // save the new element back - db.set("globalAuthor:" + authorID, author); - }); + // add the entry for this pad + author.padIDs[padID] = 1; // anything, because value is not used + + // save the new element back + db.set("globalAuthor:" + authorID, author); } /** @@ -266,16 +251,15 @@ exports.addPad = function(authorID, padID) * @param {String} author The id of the author * @param {String} padID The id of the pad the author contributes to */ -exports.removePad = function(authorID, padID) +exports.removePad = async function(authorID, padID) { - db.get("globalAuthor:" + authorID, function(err, author) { - if (ERR(err)) return; - if (author === null) return; + let author = await db.get("globalAuthor:" + authorID); - if (author.padIDs !== null) { - // remove pad from author - delete author.padIDs[padID]; - db.set("globalAuthor:" + authorID, author); - } - }); + if (author === null) return; + + if (author.padIDs !== null) { + // remove pad from author + delete author.padIDs[padID]; + db.set("globalAuthor:" + authorID, author); + } } From 1b6430ae9fa62a6d6bdc090bb94947d98dac5e07 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 18:07:01 +0000 Subject: [PATCH 33/58] db/PadMessageHandler.js: partial conversion to Promises Converted those functions that API.js still depends on, and others that at this point are never called via the nodeback mechanism. --- src/node/handler/PadMessageHandler.js | 63 ++++++++++++--------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 8fc43b156..77d7dc6c5 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -330,7 +330,7 @@ function handleSaveRevisionMessage(client, message){ * @param msg {Object} the message we're sending * @param sessionID {string} the socketIO session to which we're sending this message */ -exports.handleCustomObjectMessage = thenify(function(msg, sessionID, cb) { +exports.handleCustomObjectMessage = async function(msg, sessionID) { if (msg.data.type === "CUSTOM") { if (sessionID){ // a sessionID is targeted: directly to this sessionID @@ -340,9 +340,7 @@ exports.handleCustomObjectMessage = thenify(function(msg, sessionID, cb) { socketio.sockets.in(msg.data.payload.padId).json.send(msg); } } - cb(null, {}); -}); - +} /** * Handles a custom message (sent via HTTP API request) @@ -809,7 +807,7 @@ function handleUserChanges(data, cb) }); } -exports.updatePadClients = function(pad, callback) +exports.updatePadClients = thenify(function(pad, callback) { // skip this if no-one is on this pad var roomClients = _getRoomClients(pad.id); @@ -886,7 +884,7 @@ exports.updatePadClients = function(pad, callback) callback ); },callback); -} +}); /** * Copied from the Etherpad Source Code. Don't know what this method does excatly... @@ -1503,7 +1501,7 @@ function handleChangesetRequest(client, message) * Tries to rebuild the getChangestInfo function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L144 */ -function getChangesetInfo(padId, startNum, endNum, granularity, callback) +let getChangesetInfo = thenify(function getChangesetInfo(padId, startNum, endNum, granularity, callback) { var forwardsChangesets = []; var backwardsChangesets = []; @@ -1643,13 +1641,13 @@ function getChangesetInfo(padId, startNum, endNum, granularity, callback) start: startNum, granularity: granularity }); }); -} +}); /** * Tries to rebuild the getPadLines function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L263 */ -function getPadLines(padId, revNum, callback) +let getPadLines = thenify(function getPadLines(padId, revNum, callback) { var atext; var result = {}; @@ -1693,13 +1691,13 @@ function getPadLines(padId, revNum, callback) callback(null, result); }); -} +}); /** * Tries to rebuild the composePadChangeset function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L241 */ -function composePadChangesets(padId, startNum, endNum, callback) +let composePadChangesets = thenify(function(padId, startNum, endNum, callback) { var pad; var changesets = {}; @@ -1772,7 +1770,7 @@ function composePadChangesets(padId, startNum, endNum, callback) callback(null, changeset); }); -} +}); function _getRoomClients(padID) { var roomClients = []; @@ -1790,39 +1788,32 @@ function _getRoomClients(padID) { /** * Get the number of users in a pad */ -exports.padUsersCount = thenify(function(padID, callback) { - callback(null, { +exports.padUsersCount = function(padID) { + return { padUsersCount: _getRoomClients(padID).length - }); -}); + } +} /** * Get the list of users in a pad */ -exports.padUsers = thenify(function(padID, callback) { - var result = []; +exports.padUsers = async function(padID) { - var roomClients = _getRoomClients(padID); + let padUsers = []; + let roomClients = _getRoomClients(padID); - async.forEach(roomClients, function(roomClient, callback) { - var s = sessioninfos[roomClient.id]; + for (let i = 0, n = roomClients.length; i < n; ++i) { + let roomClient = roomClients[i]; + + let s = sessioninfos[roomClient.id]; if (s) { - authorManager.getAuthor(s.author, function(err, author) { - if (ERR(err, callback)) return; - - author.id = s.author; - result.push(author); - callback(); - }); - } else { - callback(); + let author = await authorManager.getAuthor(s.author); + author.id = s.author; + padUsers.push(author); } - }, - function(err) { - if (ERR(err, callback)) return; + } - callback(null, {padUsers: result}); - }); -}); + return { padUsers }; +} exports.sessioninfos = sessioninfos; From e7dc0766fdcc3c8477e97c83393fcc2683fab72e Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 25 Jan 2019 18:08:34 +0000 Subject: [PATCH 34/58] db/API.js: complete conversion to Promises This patch also contains significant refactoring relating to error checking of arguments supplied to the functions (e.g. rev) facilitated by use of `throw` instead of nodeback errors. --- src/node/db/API.js | 914 +++++++++++++++++---------------------------- 1 file changed, 336 insertions(+), 578 deletions(-) diff --git a/src/node/db/API.js b/src/node/db/API.js index df2e25f10..f3274a22a 100644 --- a/src/node/db/API.js +++ b/src/node/db/API.js @@ -18,7 +18,6 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var customError = require("../utils/customError"); var padManager = require("./PadManager"); @@ -27,13 +26,11 @@ var readOnlyManager = require("./ReadOnlyManager"); var groupManager = require("./GroupManager"); var authorManager = require("./AuthorManager"); var sessionManager = require("./SessionManager"); -var async = require("async"); var exportHtml = require("../utils/ExportHtml"); var exportTxt = require("../utils/ExportTxt"); var importHtml = require("../utils/ImportHtml"); var cleanText = require("./Pad").cleanText; var PadDiff = require("../utils/padDiff"); -const thenify = require("thenify").withCallback; /**********************/ /**GROUP FUNCTIONS*****/ @@ -104,14 +101,11 @@ Example returns: } */ -exports.getAttributePool = thenify(function(padID, callback) +exports.getAttributePool = async function(padID) { - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {pool: pad.pool}); - }); -}); + let pad = await getPadSafe(padID, true); + return { pool: pad.pool }; +} /** getRevisionChangeset (padID, [rev]) @@ -126,61 +120,32 @@ Example returns: } */ -exports.getRevisionChangeset = thenify(function(padID, rev, callback) +exports.getRevisionChangeset = async function(padID, rev) { - // check if rev is a number - if (rev !== undefined && typeof rev !== "number") { - // try to parse the number - if (isNaN(parseInt(rev))) { - callback(new customError("rev is not a number", "apierror")); - return; - } - - rev = parseInt(rev); - } - - // ensure this is not a negative number - if (rev !== undefined && rev < 0) { - callback(new customError("rev is not a negative number", "apierror")); - return; - } - - // ensure this is not a float value - if (rev !== undefined && !is_int(rev)) { - callback(new customError("rev is a float value", "apierror")); - return; + // try to parse the revision number + if (rev !== undefined) { + rev = checkValidRev(rev); } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); + let head = pad.getHeadRevisionNumber(); - // the client asked for a special revision - if (rev !== undefined) { - // check if this is a valid revision - if (rev > pad.getHeadRevisionNumber()) { - callback(new customError("rev is higher than the head revision of the pad", "apierror")); - return; - } + // the client asked for a special revision + if (rev !== undefined) { - // get the changeset for this revision - pad.getRevisionChangeset(rev, function(err, changeset) { - if (ERR(err, callback)) return; - - callback(null, changeset); - }) - - return; + // check if this is a valid revision + if (rev > head) { + throw new customError("rev is higher than the head revision of the pad", "apierror"); } - // the client wants the latest changeset, lets return it to him - pad.getRevisionChangeset(pad.getHeadRevisionNumber(), function(err, changeset) { - if (ERR(err, callback)) return; + // get the changeset for this revision + return pad.getRevisionChangeset(rev); + } - callback(null, changeset); - }) - }); -}); + // the client wants the latest changeset, lets return it to him + return pad.getRevisionChangeset(head); +} /** getText(padID, [rev]) returns the text of a pad @@ -190,60 +155,34 @@ Example returns: {code: 0, message:"ok", data: {text:"Welcome Text"}} {code: 1, message:"padID does not exist", data: null} */ -exports.getText = thenify(function(padID, rev, callback) +exports.getText = async function(padID, rev) { - // check if rev is a number - if (rev !== undefined && typeof rev !== "number") { - // try to parse the number - if (isNaN(parseInt(rev))) { - callback(new customError("rev is not a number", "apierror")); - return; - } - - rev = parseInt(rev); - } - - // ensure this is not a negative number - if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negative number", "apierror")); - return; - } - - // ensure this is not a float value - if (rev !== undefined && !is_int(rev)) { - callback(new customError("rev is a float value", "apierror")); - return; + // try to parse the revision number + if (rev !== undefined) { + rev = checkValidRev(rev); } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); + let head = pad.getHeadRevisionNumber(); - // the client asked for a special revision - if (rev !== undefined) { - // check if this is a valid revision - if (rev > pad.getHeadRevisionNumber()) { - callback(new customError("rev is higher than the head revision of the pad", "apierror")); - return; - } + // the client asked for a special revision + if (rev !== undefined) { - // get the text of this revision - pad.getInternalRevisionAText(rev, function(err, atext) { - if (ERR(err, callback)) return; - - var data = {text: atext.text}; - - callback(null, data); - }) - - return; + // check if this is a valid revision + if (rev > head) { + throw new customError("rev is higher than the head revision of the pad", "apierror"); } - // the client wants the latest text, lets return it to him - var padText = exportTxt.getTXTFromAtext(pad, pad.atext); - callback(null, {"text": padText}); - }); -}); + // get the text of this revision + let text = await pad.getInternalRevisionAText(rev); + return { text }; + } + + // the client wants the latest text, lets return it to him + let text = exportTxt.getTXTFromAtext(pad, pad.atext); + return { text }; +} /** setText(padID, text) sets the text of a pad @@ -254,25 +193,22 @@ Example returns: {code: 1, message:"padID does not exist", data: null} {code: 1, message:"text too long", data: null} */ -exports.setText = thenify(function(padID, text, callback) +exports.setText = async function(padID, text) { // text is required if (typeof text !== "string") { - callback(new customError("text is not a string", "apierror")); - return; + throw new customError("text is not a string", "apierror"); } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); - // set the text - pad.setText(text); + // set the text + pad.setText(text); - // update the clients on the pad - padMessageHandler.updatePadClients(pad, callback); - }); -}); + // update the clients on the pad + padMessageHandler.updatePadClients(pad); +} /** appendText(padID, text) appends text to a pad @@ -283,24 +219,20 @@ Example returns: {code: 1, message:"padID does not exist", data: null} {code: 1, message:"text too long", data: null} */ -exports.appendText = thenify(function(padID, text, callback) +exports.appendText = async function(padID, text) { // text is required if (typeof text !== "string") { - callback(new customError("text is not a string", "apierror")); - return; + throw new customError("text is not a string", "apierror"); } - // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + // get and update the pad + let pad = await getPadSafe(padID, true); + pad.appendText(text); - pad.appendText(text); - - // update the clients on the pad - padMessageHandler.updatePadClients(pad, callback); - }); -}); + // update the clients on the pad + padMessageHandler.updatePadClients(pad); +} /** getHTML(padID, [rev]) returns the html of a pad @@ -310,60 +242,30 @@ Example returns: {code: 0, message:"ok", data: {text:"Welcome Text"}} {code: 1, message:"padID does not exist", data: null} */ -exports.getHTML = thenify(function(padID, rev, callback) +exports.getHTML = async function(padID, rev) { - if (rev !== undefined && typeof rev !== "number") { - if (isNaN(parseInt(rev))) { - callback(new customError("rev is not a number", "apierror")); - return; + if (rev !== undefined) { + rev = checkValidRev(rev); + } + + let pad = await getPadSafe(padID, true); + + // the client asked for a special revision + if (rev !== undefined) { + // check if this is a valid revision + let head = pad.getHeadRevisionNumber(); + if (rev > head) { + throw new customError("rev is higher than the head revision of the pad", "apierror"); } - - rev = parseInt(rev); } - if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negative number", "apierror")); - return; - } + // get the html of this revision + html = await exportHtml.getPadHTML(pad, rev); - if (rev !== undefined && !is_int(rev)) { - callback(new customError("rev is a float value", "apierror")); - return; - } - - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - // the client asked for a special revision - if (rev !== undefined) { - // check if this is a valid revision - if (rev > pad.getHeadRevisionNumber()) { - callback(new customError("rev is higher than the head revision of the pad", "apierror")); - return; - } - - // get the html of this revision - exportHtml.getPadHTML(pad, rev, function(err, html) { - if (ERR(err, callback)) return; - html = "" +html; // adds HTML head - html += ""; - var data = {html: html}; - callback(null, data); - }); - - return; - } - - // the client wants the latest text, lets return it to him - exportHtml.getPadHTML(pad, undefined, function(err, html) { - if (ERR(err, callback)) return; - html = "" +html; // adds HTML head - html += ""; - var data = {html: html}; - callback(null, data); - }); - }); -}); + // wrap the HTML + html = "" + html + ""; + return { html }; +} /** setHTML(padID, html) sets the text of a pad based on HTML @@ -373,30 +275,26 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.setHTML = thenify(function(padID, html, callback) +exports.setHTML = async function(padID, html) { - // html is required + // html string is required if (typeof html !== "string") { - callback(new customError("html is not a string", "apierror")); - return; + throw new customError("html is not a string", "apierror"); } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); - // add a new changeset with the new html to the pad - importHtml.setPadHTML(pad, cleanText(html), function(e) { - if (e) { - callback(new customError("HTML is malformed", "apierror")); - return; - } + // add a new changeset with the new html to the pad + try { + await importHtml.setPadHTML(pad, cleanText(html)); + } catch (e) { + throw new customError("HTML is malformed", "apierror"); + } - // update the clients on the pad - padMessageHandler.updatePadClients(pad, callback); - }); - }); -}); + // update the clients on the pad + padMessageHandler.updatePadClients(pad); +}; /******************/ /**CHAT FUNCTIONS */ @@ -414,52 +312,43 @@ Example returns: {code: 1, message:"padID does not exist", data: null} */ -exports.getChatHistory = thenify(function(padID, start, end, callback) +exports.getChatHistory = async function(padID, start, end) { if (start && end) { if (start < 0) { - callback(new customError("start is below zero", "apierror")); - return; + throw new customError("start is below zero", "apierror"); } if (end < 0) { - callback(new customError("end is below zero", "apierror")); - return; + throw new customError("end is below zero", "apierror"); } if (start > end) { - callback(new customError("start is higher than end", "apierror")); - return; + throw new customError("start is higher than end", "apierror"); } } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); - var chatHead = pad.chatHead; + var chatHead = pad.chatHead; - // fall back to getting the whole chat-history if a parameter is missing - if (!start || !end) { + // fall back to getting the whole chat-history if a parameter is missing + if (!start || !end) { start = 0; end = pad.chatHead; - } + } - if (start > chatHead) { - callback(new customError("start is higher than the current chatHead", "apierror")); - return; - } - if (end > chatHead) { - callback(new customError("end is higher than the current chatHead", "apierror")); - return; - } + if (start > chatHead) { + throw new customError("start is higher than the current chatHead", "apierror"); + } + if (end > chatHead) { + throw new customError("end is higher than the current chatHead", "apierror"); + } - // the the whole message-log and return it to the client - pad.getChatMessages(start, end, - function(err, msgs) { - if (ERR(err, callback)) return; - callback(null, {messages: msgs}); - }); - }); -}); + // the the whole message-log and return it to the client + let messages = await pad.getChatMessages(start, end); + + return { messages }; +} /** appendChatMessage(padID, text, authorID, time), creates a chat message for the pad id, time is a timestamp @@ -469,25 +358,23 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.appendChatMessage = thenify(function(padID, text, authorID, time, callback) +exports.appendChatMessage = async function(padID, text, authorID, time) { // text is required if (typeof text !== "string") { - callback(new customError("text is not a string", "apierror")); - return; + throw new customError("text is not a string", "apierror"); } - // if time is not an integer value + // if time is not an integer value set time to current timestamp if (time === undefined || !is_int(time)) { - // set time to current timestamp time = Date.now(); } + // @TODO - missing getPadSafe() call ? + // save chat message to database and send message to all connected clients padMessageHandler.sendChatMessageToPadClients(time, authorID, text, padID); - - callback(); -}); +} /*****************/ /**PAD FUNCTIONS */ @@ -501,15 +388,12 @@ Example returns: {code: 0, message:"ok", data: {revisions: 56}} {code: 1, message:"padID does not exist", data: null} */ -exports.getRevisionsCount = thenify(function(padID, callback) +exports.getRevisionsCount = async function(padID) { // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {revisions: pad.getHeadRevisionNumber()}); - }); -}); + let pad = await getPadSafe(padID, true); + return { revisions: pad.getHeadRevisionNumber() }; +} /** getSavedRevisionsCount(padID) returns the number of saved revisions of this pad @@ -519,15 +403,12 @@ Example returns: {code: 0, message:"ok", data: {savedRevisions: 42}} {code: 1, message:"padID does not exist", data: null} */ -exports.getSavedRevisionsCount = thenify(function(padID, callback) +exports.getSavedRevisionsCount = async function(padID) { // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {savedRevisions: pad.getSavedRevisionsNumber()}); - }); -}); + let pad = await getPadSafe(padID, true); + return { savedRevisions: pad.getSavedRevisionsNumber() }; +} /** listSavedRevisions(padID) returns the list of saved revisions of this pad @@ -537,15 +418,12 @@ Example returns: {code: 0, message:"ok", data: {savedRevisions: [2, 42, 1337]}} {code: 1, message:"padID does not exist", data: null} */ -exports.listSavedRevisions = thenify(function(padID, callback) +exports.listSavedRevisions = async function(padID) { // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {savedRevisions: pad.getSavedRevisionsList()}); - }); -}); + let pad = await getPadSafe(padID, true); + return { savedRevisions: pad.getSavedRevisionsList() }; +} /** saveRevision(padID) returns the list of saved revisions of this pad @@ -555,54 +433,29 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.saveRevision = thenify(function(padID, rev, callback) +exports.saveRevision = async function(padID, rev) { // check if rev is a number - if (rev !== undefined && typeof rev !== "number") { - // try to parse the number - if (isNaN(parseInt(rev))) { - callback(new customError("rev is not a number", "apierror")); - return; - } - - rev = parseInt(rev); - } - - // ensure this is not a negative number - if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negative number", "apierror")); - return; - } - - // ensure this is not a float value - if (rev !== undefined && !is_int(rev)) { - callback(new customError("rev is a float value", "apierror")); - return; + if (rev !== undefined) { + rev = checkValidRev(rev); } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); + let head = pad.getHeadRevisionNumber(); - // the client asked for a special revision - if (rev !== undefined) { - // check if this is a valid revision - if (rev > pad.getHeadRevisionNumber()) { - callback(new customError("rev is higher than the head revision of the pad", "apierror")); - return; - } - } else { - rev = pad.getHeadRevisionNumber(); + // the client asked for a special revision + if (rev !== undefined) { + if (rev > head) { + throw new customError("rev is higher than the head revision of the pad", "apierror"); } + } else { + rev = pad.getHeadRevisionNumber(); + } - authorManager.createAuthor('API', function(err, author) { - if (ERR(err, callback)) return; - - pad.addSavedRevision(rev, author.authorID, 'Saved through API call'); - callback(); - }); - }); -}); + let author = authorManager.createAuthor('API'); + pad.addSavedRevision(rev, author.authorID, 'Saved through API call'); +} /** getLastEdited(padID) returns the timestamp of the last revision of the pad @@ -612,18 +465,13 @@ Example returns: {code: 0, message:"ok", data: {lastEdited: 1340815946602}} {code: 1, message:"padID does not exist", data: null} */ -exports.getLastEdited = thenify(function(padID, callback) +exports.getLastEdited = async function(padID) { // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - pad.getLastEdit(function(err, value) { - if (ERR(err, callback)) return; - callback(null, {lastEdited: value}); - }); - }); -}); + let pad = await getPadSafe(padID, true); + let lastEdited = await pad.getLastEdit(); + return { lastEdited }; +} /** createPad(padName [, text]) creates a new pad in this group @@ -633,28 +481,23 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"pad does already exist", data: null} */ -exports.createPad = thenify(function(padID, text, callback) +exports.createPad = async function(padID, text) { if (padID) { // ensure there is no $ in the padID if (padID.indexOf("$") !== -1) { - callback(new customError("createPad can't create group pads", "apierror")); - return; + throw new customError("createPad can't create group pads", "apierror"); } // check for url special characters if (padID.match(/(\/|\?|&|#)/)) { - callback(new customError("malformed padID: Remove special characters", "apierror")); - return; + throw new customError("malformed padID: Remove special characters", "apierror"); } } // create pad - getPadSafe(padID, false, text, function(err) { - if (ERR(err, callback)) return; - callback(); - }); -}); + await getPadSafe(padID, false, text); +} /** deletePad(padID) deletes a pad @@ -664,14 +507,12 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.deletePad = thenify(function(padID, callback) +exports.deletePad = async function(padID) { - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); + await pad.remove(); +} - pad.remove(callback); - }); -}); /** restoreRevision(padID, [rev]) Restores revision from past as new changeset @@ -680,90 +521,66 @@ exports.deletePad = thenify(function(padID, callback) {code:0, message:"ok", data:null} {code: 1, message:"padID does not exist", data: null} */ -exports.restoreRevision = thenify(function(padID, rev, callback) +exports.restoreRevision = async function(padID, rev) { // check if rev is a number - if (rev !== undefined && typeof rev !== "number") { - // try to parse the number - if (isNaN(parseInt(rev))) { - callback(new customError("rev is not a number", "apierror")); - return; - } - - rev = parseInt(rev); - } - - // ensure this is not a negative number - if (rev !== undefined && rev < 0) { - callback(new customError("rev is a negative number", "apierror")); - return; - } - - // ensure this is not a float value - if (rev !== undefined && !is_int(rev)) { - callback(new customError("rev is a float value", "apierror")); - return; + if (rev === undefined) { + throw new customeError("rev is not defined", "apierror"); } + rev = checkValidRev(rev); // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); + // check if this is a valid revision + if (rev > pad.getHeadRevisionNumber()) { + throw new customError("rev is higher than the head revision of the pad", "apierror"); + } - // check if this is a valid revision - if (rev > pad.getHeadRevisionNumber()) { - callback(new customError("rev is higher than the head revision of the pad", "apierror")); - return; + let atext = await pad.getInternalRevisionAText(rev); + + var oldText = pad.text(); + atext.text += "\n"; + + function eachAttribRun(attribs, func) { + var attribsIter = Changeset.opIterator(attribs); + var textIndex = 0; + var newTextStart = 0; + var newTextEnd = atext.text.length; + while (attribsIter.hasNext()) { + var op = attribsIter.next(); + var nextIndex = textIndex + op.chars; + if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) { + func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs); + } + textIndex = nextIndex; } + } - pad.getInternalRevisionAText(rev, function(err, atext) { - if (ERR(err, callback)) return; - - var oldText = pad.text(); - atext.text += "\n"; - function eachAttribRun(attribs, func) { - var attribsIter = Changeset.opIterator(attribs); - var textIndex = 0; - var newTextStart = 0; - var newTextEnd = atext.text.length; - while (attribsIter.hasNext()) { - var op = attribsIter.next(); - var nextIndex = textIndex + op.chars; - if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) { - func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs); - } - textIndex = nextIndex; - } - } - - // create a new changeset with a helper builder object - var builder = Changeset.builder(oldText.length); - - // assemble each line into the builder - eachAttribRun(atext.attribs, function(start, end, attribs) { - builder.insert(atext.text.substring(start, end), attribs); - }); - - var lastNewlinePos = oldText.lastIndexOf('\n'); - if (lastNewlinePos < 0) { - builder.remove(oldText.length - 1, 0); - } else { - builder.remove(lastNewlinePos, oldText.match(/\n/g).length - 1); - builder.remove(oldText.length - lastNewlinePos - 1, 0); - } - - var changeset = builder.toString(); - - // append the changeset - pad.appendRevision(changeset); - - // update the clients on the pad - padMessageHandler.updatePadClients(pad, function() {}); - callback(null, null); - }); + // create a new changeset with a helper builder object + var builder = Changeset.builder(oldText.length); + // assemble each line into the builder + eachAttribRun(atext.attribs, function(start, end, attribs) { + builder.insert(atext.text.substring(start, end), attribs); }); -}); + + var lastNewlinePos = oldText.lastIndexOf('\n'); + if (lastNewlinePos < 0) { + builder.remove(oldText.length - 1, 0); + } else { + builder.remove(lastNewlinePos, oldText.match(/\n/g).length - 1); + builder.remove(oldText.length - lastNewlinePos - 1, 0); + } + + var changeset = builder.toString(); + + // append the changeset + pad.appendRevision(changeset); + + // update the clients on the pad + padMessageHandler.updatePadClients(pad); +} /** copyPad(sourceID, destinationID[, force=false]) copies a pad. If force is true, @@ -774,14 +591,11 @@ Example returns: {code: 0, message:"ok", data: {padID: destinationID}} {code: 1, message:"padID does not exist", data: null} */ -exports.copyPad = thenify(function(sourceID, destinationID, force, callback) +exports.copyPad = async function(sourceID, destinationID, force) { - getPadSafe(sourceID, true, function(err, pad) { - if (ERR(err, callback)) return; - - pad.copy(destinationID, force, callback); - }); -}); + let pad = await getPadSafe(sourceID, true); + await pad.copy(destinationID, force); +} /** movePad(sourceID, destinationID[, force=false]) moves a pad. If force is true, @@ -792,17 +606,13 @@ Example returns: {code: 0, message:"ok", data: {padID: destinationID}} {code: 1, message:"padID does not exist", data: null} */ -exports.movePad = thenify(function(sourceID, destinationID, force, callback) +exports.movePad = async function(sourceID, destinationID, force) { - getPadSafe(sourceID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(sourceID, true); + await pad.copy(destinationID, force); + await pad.remove(); +} - pad.copy(destinationID, force, function(err) { - if (ERR(err, callback)) return; - pad.remove(callback); - }); - }); -}); /** getReadOnlyLink(padID) returns the read only link of a pad @@ -811,19 +621,16 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.getReadOnlyID = thenify(function(padID, callback) +exports.getReadOnlyID = async function(padID) { // we don't need the pad object, but this function does all the security stuff for us - getPadSafe(padID, true, function(err) { - if (ERR(err, callback)) return; + await getPadSafe(padID, true); - // get the readonlyId - readOnlyManager.getReadOnlyId(padID, function(err, readOnlyId) { - if (ERR(err, callback)) return; - callback(null, {readOnlyID: readOnlyId}); - }); - }); -}); + // get the readonlyId + let readOnlyID = await readOnlyManager.getReadOnlyId(padID); + + return { readOnlyID }; +} /** getPadID(roID) returns the padID of a pad based on the readonlyID(roID) @@ -833,20 +640,16 @@ Example returns: {code: 0, message:"ok", data: {padID: padID}} {code: 1, message:"padID does not exist", data: null} */ -exports.getPadID = thenify(function(roID, callback) +exports.getPadID = async function(roID) { // get the PadId - readOnlyManager.getPadId(roID, function(err, retrievedPadID) { - if (ERR(err, callback)) return; + let padID = await readOnlyManager.getPadId(roID); + if (padID === null) { + throw new customError("padID does not exist", "apierror"); + } - if (retrievedPadID === null) { - callback(new customError("padID does not exist", "apierror")); - return; - } - - callback(null, {padID: retrievedPadID}); - }); -}); + return { padID }; +} /** setPublicStatus(padID, publicStatus) sets a boolean for the public status of a pad @@ -856,28 +659,22 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.setPublicStatus = thenify(function(padID, publicStatus, callback) +exports.setPublicStatus = async function(padID, publicStatus) { // ensure this is a group pad - if (padID && padID.indexOf("$") === -1) { - callback(new customError("You can only get/set the publicStatus of pads that belong to a group", "apierror")); - return; - } + checkGroupPad(padID, "publicStatus"); // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); - // convert string to boolean - if (typeof publicStatus === "string") - publicStatus = publicStatus == "true" ? true : false; + // convert string to boolean + if (typeof publicStatus === "string") { + publicStatus = (publicStatus.toLowerCase() === "true"); + } - // set the password - pad.setPublicStatus(publicStatus); - - callback(); - }); -}); + // set the password + pad.setPublicStatus(publicStatus); +} /** getPublicStatus(padID) return true of false @@ -887,21 +684,15 @@ Example returns: {code: 0, message:"ok", data: {publicStatus: true}} {code: 1, message:"padID does not exist", data: null} */ -exports.getPublicStatus = thenify(function(padID, callback) +exports.getPublicStatus = async function(padID) { // ensure this is a group pad - if (padID && padID.indexOf("$") == -1) { - callback(new customError("You can only get/set the publicStatus of pads that belong to a group", "apierror")); - return; - } + checkGroupPad(padID, "publicStatus"); // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {publicStatus: pad.getPublicStatus()}); - }); -}); + let pad = await getPadSafe(padID, true); + return { publicStatus: pad.getPublicStatus() }; +} /** setPassword(padID, password) returns ok or a error message @@ -911,24 +702,17 @@ Example returns: {code: 0, message:"ok", data: null} {code: 1, message:"padID does not exist", data: null} */ -exports.setPassword = thenify(function(padID, password, callback) +exports.setPassword = async function(padID, password) { // ensure this is a group pad - if (padID && padID.indexOf("$") == -1) { - callback(new customError("You can only get/set the password of pads that belong to a group", "apierror")); - return; - } + checkGroupPad(padID, "password"); // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; + let pad = await getPadSafe(padID, true); - // set the password - pad.setPassword(password == "" ? null : password); - - callback(); - }); -}); + // set the password + pad.setPassword(password == "" ? null : password); +} /** isPasswordProtected(padID) returns true or false @@ -938,21 +722,15 @@ Example returns: {code: 0, message:"ok", data: {passwordProtection: true}} {code: 1, message:"padID does not exist", data: null} */ -exports.isPasswordProtected = thenify(function(padID, callback) +exports.isPasswordProtected = async function(padID) { // ensure this is a group pad - if (padID && padID.indexOf("$") == -1) { - callback(new customError("You can only get/set the password of pads that belong to a group", "apierror")); - return; - } + checkGroupPad(padID, "password"); // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {isPasswordProtected: pad.isPasswordProtected()}); - }); -}); + let pad = await getPadSafe(padID, true); + return { isPasswordProtected: pad.isPasswordProtected() }; +} /** listAuthorsOfPad(padID) returns an array of authors who contributed to this pad @@ -962,15 +740,13 @@ Example returns: {code: 0, message:"ok", data: {authorIDs : ["a.s8oes9dhwrvt0zif", "a.akf8finncvomlqva"]} {code: 1, message:"padID does not exist", data: null} */ -exports.listAuthorsOfPad = thenify(function(padID, callback) +exports.listAuthorsOfPad = async function(padID) { // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {authorIDs: pad.getAllAuthors()}); - }); -}); + let pad = await getPadSafe(padID, true); + let authorIDs = pad.getAllAuthors(); + return { authorIDs }; +} /** sendClientsMessage(padID, msg) sends a message to all clients connected to the @@ -995,15 +771,10 @@ Example returns: {code: 1, message:"padID does not exist"} */ -exports.sendClientsMessage = thenify(function(padID, msg, callback) { - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) { - return; - } - - padMessageHandler.handleCustomMessage(padID, msg, callback); - } ); -}); +exports.sendClientsMessage = async function(padID, msg) { + let pad = await getPadSafe(padID, true); + padMessageHandler.handleCustomMessage(padID, msg); +} /** checkToken() returns ok when the current api token is valid @@ -1013,10 +784,9 @@ Example returns: {"code":0,"message":"ok","data":null} {"code":4,"message":"no or wrong API Key","data":null} */ -exports.checkToken = thenify(function(callback) +exports.checkToken = async function() { - callback(); -}); +} /** getChatHead(padID) returns the chatHead (last number of the last chat-message) of the pad @@ -1026,15 +796,12 @@ Example returns: {code: 0, message:"ok", data: {chatHead: 42}} {code: 1, message:"padID does not exist", data: null} */ -exports.getChatHead = thenify(function(padID, callback) +exports.getChatHead = async function(padID) { // get the pad - getPadSafe(padID, true, function(err, pad) { - if (ERR(err, callback)) return; - - callback(null, {chatHead: pad.chatHead}); - }); -}); + let pad = await getPadSafe(padID, true); + return { chatHead: pad.chatHead }; +} /** createDiffHTML(padID, startRev, endRev) returns an object of diffs from 2 points in a pad @@ -1044,68 +811,31 @@ Example returns: {"code":0,"message":"ok","data":{"html":"Welcome to Etherpad!

    This pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!

    Get involved with Etherpad at http://etherpad.org
    aw

    ","authors":["a.HKIv23mEbachFYfH",""]}} {"code":4,"message":"no or wrong API Key","data":null} */ -exports.createDiffHTML = thenify(function(padID, startRev, endRev, callback) { - // check if startRev is a number - if (startRev !== undefined && typeof startRev !== "number") { - // try to parse the number - if (isNaN(parseInt(startRev))) { - callback({stop: "startRev is not a number"}); - return; - } +exports.createDiffHTML = async function(padID, startRev, endRev) { - startRev = parseInt(startRev, 10); + // check if startRev is a number + if (startRev !== undefined) { + startRev = checkValidRev(startRev); } // check if endRev is a number - if (endRev !== undefined && typeof endRev !== "number") { - // try to parse the number - if (isNaN(parseInt(endRev))) { - callback({stop: "endRev is not a number"}); - return; - } - - endRev = parseInt(endRev, 10); + if (endRev !== undefined) { + endRev = checkValidRev(endRev); } // get the pad - getPadSafe(padID, true, function(err, pad) { - if (err) { - return callback(err); - } + let pad = await getPadSafe(padID, true); + try { + var padDiff = new PadDiff(pad, startRev, endRev); + } catch (e) { + throw { stop: e.message }; + } - try { - var padDiff = new PadDiff(pad, startRev, endRev); - } catch(e) { - return callback({stop:e.message}); - } - var html, authors; + let html = await padDiff.getHtml(); + let authors = await padDiff.getAuthors(); - async.series([ - function(callback) { - padDiff.getHtml(function(err, _html) { - if (err) { - return callback(err); - } - - html = _html; - callback(); - }); - }, - function(callback) { - padDiff.getAuthors(function(err, _authors) { - if (err) { - return callback(err); - } - - authors = _authors; - callback(); - }); - } - ], function(err) { - callback(err, {html: html, authors: authors}) - }); - }); -}); + return { html, authors }; +} /******************************/ /** INTERNAL HELPER FUNCTIONS */ @@ -1114,42 +844,70 @@ exports.createDiffHTML = thenify(function(padID, startRev, endRev, callback) { // checks if a number is an int function is_int(value) { - return (parseFloat(value) == parseInt(value)) && !isNaN(value) + return (parseFloat(value) == parseInt(value, 10)) && !isNaN(value) } // gets a pad safe -function getPadSafe(padID, shouldExist, text, callback) +async function getPadSafe(padID, shouldExist, text) { - if (typeof text === "function") { - callback = text; - text = null; - } - // check if padID is a string if (typeof padID !== "string") { - callback(new customError("padID is not a string", "apierror")); - return; + throw new customError("padID is not a string", "apierror"); } // check if the padID maches the requirements if (!padManager.isValidPadId(padID)) { - callback(new customError("padID did not match requirements", "apierror")); - return; + throw new customError("padID did not match requirements", "apierror"); } // check if the pad exists - padManager.doesPadExists(padID, function(err, exists) { - if (ERR(err, callback)) return; + let exists = await padManager.doesPadExists(padID); + if (!exists && shouldExist) { // does not exist, but should - if (exists == false && shouldExist == true) { - callback(new customError("padID does not exist", "apierror")); - } else if (exists == true && shouldExist == false) { - // does exist, but shouldn't - callback(new customError("padID does already exist", "apierror")); - } else { - // pad exists, let's get it - padManager.getPad(padID, text, callback); - } - }); + throw new customError("padID does not exist", "apierror"); + } + + if (exists && !shouldExist) { + // does exist, but shouldn't + throw new customError("padID does already exist", "apierror"); + } + + // pad exists, let's get it + return padManager.getPad(padID, text); +} + +// checks if a rev is a legal number +// pre-condition is that `rev` is not undefined +function checkValidRev(rev) +{ + if (typeof rev !== "number") { + rev = parseInt(rev, 10); + } + + // check if rev is a number + if (isNaN(rev)) { + throw new customError("rev is not a number", "apierror"); + } + + // ensure this is not a negative number + if (rev < 0) { + throw new customError("rev is not a negative number", "apierror"); + } + + // ensure this is not a float value + if (!is_int(rev)) { + throw new customError("rev is a float value", "apierror"); + } + + return rev; +} + +// checks if a padID is part of a group +function checkGroupPad(padID, field) +{ + // ensure this is a group pad + if (padID && padID.indexOf("$") === -1) { + throw new customError(`You can only get/set the ${field} of pads that belong to a group`, "apierror"); + } } From 7709fd46e53141fc6fdfd534dab48f8c88e4f75e Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Sat, 26 Jan 2019 23:52:02 +0000 Subject: [PATCH 35/58] utility scripts: converted to use the Promise interface --- bin/checkAllPads.js | 178 ++++++++++++++++-------------------------- bin/checkPad.js | 160 +++++++++++++++---------------------- bin/deletePad.js | 70 ++++++----------- bin/extractPadData.js | 112 +++++++++++--------------- 4 files changed, 202 insertions(+), 318 deletions(-) diff --git a/bin/checkAllPads.js b/bin/checkAllPads.js index c4467fa79..0d4e8bb8d 100644 --- a/bin/checkAllPads.js +++ b/bin/checkAllPads.js @@ -7,128 +7,88 @@ if (process.argv.length != 2) { process.exit(1); } -// initialize the variables -var db, settings, padManager; -var npm = require('../src/node_modules/npm'); -var async = require('../src/node_modules/async'); - -var Changeset = require('../src/static/js/Changeset'); - -async.series([ - // load npm - function(callback) { - npm.load({}, callback); - }, - - // load modules - function(callback) { - settings = require('../src/node/utils/Settings'); - db = require('../src/node/db/DB'); +// load and initialize NPM +let npm = require('../src/node_modules/npm'); +npm.load({}, async function() { + try { // initialize the database - db.init(callback); - }, + let settings = require('../src/node/utils/Settings'); + let db = require('../src/node/db/DB'); + await db.init(); - // load pads - function (callback) { - padManager = require('../src/node/db/PadManager'); + // load modules + let Changeset = require('../src/static/js/Changeset'); + let padManager = require('../src/node/db/PadManager'); - padManager.listAllPads(function(err, res) { - padIds = res.padIDs; - callback(err); - }); - }, + // get all pads + let res = await padManager.listAllPads(); - function (callback) { - async.forEach(padIds, function(padId, callback) { - padManager.getPad(padId, function(err, pad) { - if (err) { - callback(err); + for (let padId of res.padIDs) { + + let pad = await padManager.getPad(padId); + + // check if the pad has a pool + if (pad.pool === undefined) { + console.error("[" + pad.id + "] Missing attribute pool"); + continue; + } + + // create an array with key kevisions + // key revisions always save the full pad atext + let head = pad.getHeadRevisionNumber(); + let keyRevisions = []; + for (let rev = 0; rev < head; rev += 100) { + keyRevisions.push(rev); + } + + // run through all key revisions + for (let keyRev of keyRevisions) { + + // create an array of revisions we need till the next keyRevision or the End + var revisionsNeeded = []; + for (let rev = keyRev ; rev <= keyRev + 100 && rev <= head; rev++) { + revisionsNeeded.push(rev); } - // check if the pad has a pool - if (pad.pool === undefined ) { - console.error("[" + pad.id + "] Missing attribute pool"); - callback(); + // this array will hold all revision changesets + var revisions = []; - return; + // run through all needed revisions and get them from the database + for (let revNum of revisionsNeeded) { + let revision = await db.get("pad:" + pad.id + ":revs:" + revNum); + revisions[revNum] = revision; } - // create an array with key kevisions - // key revisions always save the full pad atext - var head = pad.getHeadRevisionNumber(); - var keyRevisions = []; - for (var i = 0; i < head; i += 100) { - keyRevisions.push(i); + // check if the revision exists + if (revisions[keyRev] == null) { + console.error("[" + pad.id + "] Missing revision " + keyRev); + continue; } - // run through all key revisions - async.forEachSeries(keyRevisions, function(keyRev, callback) { - // create an array of revisions we need till the next keyRevision or the End - var revisionsNeeded = []; + // check if there is a atext in the keyRevisions + if (revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) { + console.error("[" + pad.id + "] Missing atext in revision " + keyRev); + continue; + } - for(var i = keyRev; i <= keyRev + 100 && i <= head; i++) { - revisionsNeeded.push(i); + let apool = pad.pool; + let atext = revisions[keyRev].meta.atext; + + for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) { + try { + let cs = revisions[rev].changeset; + atext = Changeset.applyToAText(cs, atext, apool); + } catch (e) { + console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message); } - - // this array will hold all revision changesets - var revisions = []; - - // run through all needed revisions and get them from the database - async.forEach(revisionsNeeded, function(revNum, callback) { - db.db.get("pad:" + pad.id + ":revs:" + revNum, function(err, revision) { - revisions[revNum] = revision; - callback(err); - }); - }, - - function(err) { - if (err) { - callback(err); - return; - } - - // check if the revision exists - if (revisions[keyRev] == null) { - console.error("[" + pad.id + "] Missing revision " + keyRev); - callback(); - return; - } - - // check if there is a atext in the keyRevisions - if (revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) { - console.error("[" + pad.id + "] Missing atext in revision " + keyRev); - callback(); - return; - } - - var apool = pad.pool; - var atext = revisions[keyRev].meta.atext; - - for(var i = keyRev + 1; i <= keyRev + 100 && i <= head; i++) { - try { - // console.log("[" + pad.id + "] check revision " + i); - var cs = revisions[i].changeset; - atext = Changeset.applyToAText(cs, atext, apool); - } catch(e) { - console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message); - callback(); - return; - } - } - - callback(); - }); - }, callback); - }); - }, callback); - } -], -function (err) { - if (err) { - throw err; - } else { - console.log("finished"); - process.exit(0); + } + } + console.log("finished"); + process.exit(0); + } + } catch (err) { + console.trace(err); + process.exit(1); } }); diff --git a/bin/checkPad.js b/bin/checkPad.js index 4d864d106..c6a3a1971 100644 --- a/bin/checkPad.js +++ b/bin/checkPad.js @@ -7,121 +7,89 @@ if (process.argv.length != 3) { process.exit(1); } -//get the padID -var padId = process.argv[2]; +// get the padID +const padId = process.argv[2]; -// initialize the variables -var db, settings, padManager; -var npm = require('../src/node_modules/npm'); -var async = require('../src/node_modules/async'); +// load and initialize NPM; +let npm = require('../src/node_modules/npm'); +npm.load({}, async function() { -var Changeset = require('ep_etherpad-lite/static/js/Changeset'); + try { + // initialize database + let settings = require('../src/node/utils/Settings'); + let db = require('../src/node/db/DB'); + await db.init(); -async.series([ - // load npm - function(callback) { - npm.load({}, function(er) { - callback(er); - }); - }, + // load modules + let Changeset = require('ep_etherpad-lite/static/js/Changeset'); + let padManager = require('../src/node/db/PadManager'); - // load modules - function(callback) { - settings = require('../src/node/utils/Settings'); - db = require('../src/node/db/DB'); + let exists = await padManager.doesPadExists(padId); + if (!exists) { + console.error("Pad does not exist"); + process.exit(1); + } - // initialize the database - db.init(callback); - }, + // get the pad + let pad = await padManager.getPad(padId); - // get the pad - function (callback) { - padManager = require('../src/node/db/PadManager'); - - padManager.doesPadExists(padId, function(err, exists) { - if (!exists) { - console.error("Pad does not exist"); - process.exit(1); - } - - padManager.getPad(padId, function(err, _pad) { - pad = _pad; - callback(err); - }); - }); - }, - - function (callback) { // create an array with key revisions // key revisions always save the full pad atext - var head = pad.getHeadRevisionNumber(); - var keyRevisions = []; - for (var i = 0; i < head; i += 100) { - keyRevisions.push(i); + let head = pad.getHeadRevisionNumber(); + let keyRevisions = []; + for (let rev = 0; rev < head; rev += 100) { + keyRevisions.push(rev); } // run through all key revisions - async.forEachSeries(keyRevisions, function(keyRev, callback) { + for (let keyRev of keyRevisions) { + // create an array of revisions we need till the next keyRevision or the End - var revisionsNeeded = []; - for(var i = keyRev; i <= keyRev + 100 && i <= head; i++) { - revisionsNeeded.push(i); + let revisionsNeeded = []; + for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) { + revisionsNeeded.push(rev); } // this array will hold all revision changesets var revisions = []; // run through all needed revisions and get them from the database - async.forEach(revisionsNeeded, function(revNum, callback) { - db.db.get("pad:" + padId + ":revs:" + revNum, function(err, revision) { - revisions[revNum] = revision; - callback(err); - }); - }, - function(err) { - if (err) { - callback(err); - return; + for (let revNum of revisionsNeeded) { + let revision = await db.get("pad:" + padId + ":revs:" + revNum); + revisions[revNum] = revision; + } + + // check if the pad has a pool + if (pad.pool === undefined ) { + console.error("Attribute pool is missing"); + process.exit(1); + } + + // check if there is an atext in the keyRevisions + if (revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) { + console.error("No atext in key revision " + keyRev); + continue; + } + + let apool = pad.pool; + let atext = revisions[keyRev].meta.atext; + + for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) { + try { + // console.log("check revision " + rev); + let cs = revisions[rev].changeset; + atext = Changeset.applyToAText(cs, atext, apool); + } catch(e) { + console.error("Bad changeset at revision " + rev + " - " + e.message); + continue; } + } + console.log("finished"); + process.exit(0); + } - // check if the pad has a pool - if (pad.pool === undefined) { - console.error("Attribute pool is missing"); - process.exit(1); - } - - // check if there is an atext in the keyRevisions - if (revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) { - console.error("No atext in key revision " + keyRev); - callback(); - return; - } - - var apool = pad.pool; - var atext = revisions[keyRev].meta.atext; - - for (var i = keyRev + 1; i <= keyRev + 100 && i <= head; i++) { - try { - // console.log("check revision " + i); - var cs = revisions[i].changeset; - atext = Changeset.applyToAText(cs, atext, apool); - } catch(e) { - console.error("Bad changeset at revision " + i + " - " + e.message); - callback(); - return; - } - } - - callback(); - }); - }, callback); - } -], -function (err) { - if(err) { - throw err; - } else { - console.log("finished"); - process.exit(0); + } catch (e) { + console.trace(e); + process.exit(1); } }); diff --git a/bin/deletePad.js b/bin/deletePad.js index 31856d382..e9e273e3e 100644 --- a/bin/deletePad.js +++ b/bin/deletePad.js @@ -9,55 +9,33 @@ if (process.argv.length != 3) { } // get the padID -var padId = process.argv[2]; +let padId = process.argv[2]; -var db, padManager, pad, settings; -var neededDBValues = ["pad:"+padId]; +let npm = require('../src/node_modules/npm'); -var npm = require('../src/node_modules/npm'); -var async = require('../src/node_modules/async'); - -async.series([ - // load npm - function(callback) { - npm.load({}, function(er) { - if (er) { - console.error("Could not load NPM: " + er) - process.exit(1); - } else { - callback(); - } - }); - }, - - // load modules - function(callback) { - settings = require('../src/node/utils/Settings'); - db = require('../src/node/db/DB'); - callback(); - }, - - // initialize the database - function (callback) { - db.init(callback); - }, - - // delete the pad and its links - function (callback) { - padManager = require('../src/node/db/PadManager'); - - padManager.removePad(padId, function(err){ - callback(err); - }); - - callback(); +npm.load({}, async function(er) { + if (er) { + console.error("Could not load NPM: " + er) + process.exit(1); } -], -function (err) { - if(err) { - throw err; - } else { + + try { + let settings = require('../src/node/utils/Settings'); + let db = require('../src/node/db/DB'); + await db.init(); + + padManager = require('../src/node/db/PadManager'); + await padManager.removePad(padId); + console.log("Finished deleting padId: " + padId); - process.exit(); + process.exit(0); + + } catch (e) { + if (err.name === "apierror") { + console.error(e); + } else { + console.trace(e); + } + process.exit(1); } }); diff --git a/bin/extractPadData.js b/bin/extractPadData.js index ac22e11b7..cce297f71 100644 --- a/bin/extractPadData.js +++ b/bin/extractPadData.js @@ -10,88 +10,66 @@ if (process.argv.length != 3) { } // get the padID -var padId = process.argv[2]; +let padId = process.argv[2]; -var db, dirty, padManager, pad, settings; -var neededDBValues = ["pad:"+padId]; +let npm = require('../src/node_modules/npm'); -var npm = require('../node_modules/ep_etherpad-lite/node_modules/npm'); -var async = require('../node_modules/ep_etherpad-lite/node_modules/async'); +npm.load({}, async function(er) { + if (er) { + console.error("Could not load NPM: " + er) + process.exit(1); + } -async.series([ - // load npm - function(callback) { - npm.load({}, function(er) { - if (er) { - console.error("Could not load NPM: " + er) - process.exit(1); - } else { - callback(); - } - }) - }, + try { + // initialize database + let settings = require('../src/node/utils/Settings'); + let db = require('../src/node/db/DB'); + await db.init(); - // load modules - function(callback) { - settings = require('../node_modules/ep_etherpad-lite/node/utils/Settings'); - db = require('../node_modules/ep_etherpad-lite/node/db/DB'); - dirty = require('../node_modules/ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty')(padId + ".db"); - callback(); - }, + // load extra modules + let dirtyDB = require('../src/node_modules/dirty'); + let padManager = require('../src/node/db/PadManager'); + let util = require('util'); - // initialize the database - function (callback) { - db.init(callback); - }, + // initialize output database + let dirty = dirtyDB(padId + '.db'); - // get the pad - function (callback) { - padManager = require('../node_modules/ep_etherpad-lite/node/db/PadManager'); + // Promise wrapped get and set function + let wrapped = db.db.db.wrappedDB; + let get = util.promisify(wrapped.get.bind(wrapped)); + let set = util.promisify(dirty.set.bind(dirty)); - padManager.getPad(padId, function(err, _pad) { - pad = _pad; - callback(err); - }); - }, + // array in which required key values will be accumulated + let neededDBValues = ['pad:' + padId]; + + // get the actual pad object + let pad = await padManager.getPad(padId); - function (callback) { // add all authors - var authors = pad.getAllAuthors(); - for (var i = 0; i < authors.length; i++) { - neededDBValues.push('globalAuthor:' + authors[i]); - } + neededDBValues.push(...pad.getAllAuthors().map(author => 'globalAuthor:' + author)); // add all revisions - var revHead = pad.head; - for (var i = 0; i <= revHead; i++) { - neededDBValues.push('pad:' + padId + ':revs:' + i); + for (let rev = 0; rev <= pad.head; ++rev) { + neededDBValues.push('pad:' + padId + ':revs:' + rev); } - // get all chat values - var chatHead = pad.chatHead; - for (var i = 0; i <= chatHead; i++) { - neededDBValues.push('pad:' + padId + ':chat:' + i); + // add all chat values + for (let chat = 0; chat <= pad.chatHead; ++chat) { + neededDBValues.push('pad:' + padId + ':chat:' + chat); } - // get and set all values - async.forEach(neededDBValues, function(dbkey, callback) { - db.db.db.wrappedDB.get(dbkey, function(err, dbvalue) { - if (err) { callback(err); return} + for (let dbkey of neededDBValues) { + let dbvalue = await get(dbkey); + if (dbvalue && typeof dbvalue !== 'object') { + dbvalue = JSON.parse(dbvalue); + } + await set(dbkey, dbvalue); + } - if (dbvalue && typeof dbvalue != 'object') { - dbvalue = JSON.parse(dbvalue); // if it's not json then parse it as json - } - - dirty.set(dbkey, dbvalue, callback); - }); - }, callback); - } -], -function (err) { - if (err) { - throw err; - } else { - console.log("finished"); - process.exit(); + console.log('finished'); + process.exit(0); + } catch (er) { + console.error(er); + process.exit(1); } }); From e58da69cfbb9af955e0e6424ec9496a34182fe7c Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Mon, 28 Jan 2019 13:13:24 +0000 Subject: [PATCH 36/58] db/SecurityManager.js: converted checkAccess() to pure Promises Also converted the handler functions that depend on checkAccess() into async functions too. NB: this commit needs specific attention to it because it touches a lot of security related code! --- src/node/db/SecurityManager.js | 410 +++++++------- src/node/handler/PadMessageHandler.js | 749 +++++++++++--------------- src/node/handler/SocketIORouter.js | 36 +- 3 files changed, 508 insertions(+), 687 deletions(-) diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index e9aefcd61..dcdbe6a62 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -18,8 +18,6 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); -var async = require("async"); var authorManager = require("./AuthorManager"); var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); var padManager = require("./PadManager"); @@ -35,270 +33,222 @@ const thenify = require("thenify").withCallback; * @param sessionCookie the session the user has (set via api) * @param token the token of the author (randomly generated at client side, used for public pads) * @param password the password the user has given to access this pad, can be null - * @param callback will be called with (err, {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx}) + * @return {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx}) */ -exports.checkAccess = thenify(function(padID, sessionCookie, token, password, callback) +exports.checkAccess = async function(padID, sessionCookie, token, password) { - var statusObject; + // immutable object + let deny = Object.freeze({ accessStatus: "deny" }); if (!padID) { - callback(null, {accessStatus: "deny"}); - return; + return deny; } // allow plugins to deny access var deniedByHook = hooks.callAll("onAccessCheck", {'padID': padID, 'password': password, 'token': token, 'sessionCookie': sessionCookie}).indexOf(false) > -1; if (deniedByHook) { - callback(null, {accessStatus: "deny"}); - return; + return deny; } + // get author for this token + let tokenAuthor = await authorManager.getAuthor4Token(token); + + // check if pad exists + let padExists = await padManager.doesPadExist(padID); + if (settings.requireSession) { // a valid session is required (api-only mode) if (!sessionCookie) { // without sessionCookie, access is denied - callback(null, {accessStatus: "deny"}); - - return; + return deny; } } else { // a session is not required, so we'll check if it's a public pad if (padID.indexOf("$") === -1) { // it's not a group pad, means we can grant access - // get author for this token - authorManager.getAuthor4Token(token, function(err, author) { - if (ERR(err, callback)) return; + // assume user has access + let statusObject = { accessStatus: "grant", authorID: tokenAuthor }; - // assume user has access - statusObject = { accessStatus: "grant", authorID: author }; + if (settings.editOnly) { + // user can't create pads - if (settings.editOnly) { - // user can't create pads - - // check if pad exists - padManager.doesPadExists(padID, function(err, exists) { - if (ERR(err, callback)) return; - - if (!exists) { - // pad doesn't exist - user can't have access - statusObject.accessStatus = "deny"; - } - - // grant or deny access, with author of token - callback(null, statusObject); - }); - - return; + if (!padExists) { + // pad doesn't exist - user can't have access + statusObject.accessStatus = "deny"; } + } - // user may create new pads - no need to check anything - // grant access, with author of token - callback(null, statusObject); - }); - - // don't continue - return; + // user may create new pads - no need to check anything + // grant access, with author of token + return statusObject; } } - var groupID = padID.split("$")[0]; - var padExists = false; - var validSession = false; - var sessionAuthor; - var tokenAuthor; - var isPublic; - var isPasswordProtected; - var passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong + let validSession = false; + let sessionAuthor; + let isPublic; + let isPasswordProtected; + let passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong - async.series([ - // get basic informations from the database - function(callback) { - async.parallel([ - // does pad exist - function(callback) { - padManager.doesPadExists(padID, function(err, exists) { - if (ERR(err, callback)) return; + // get information about all sessions contained in this cookie + if (sessionCookie) { + let groupID = padID.split("$")[0]; + let sessionIDs = sessionCookie.split(','); - padExists = exists; - callback(); - }); - }, + // was previously iterated in parallel using async.forEach + for (let sessionID of sessionIDs) { + try { + let sessionInfo = await sessionManager.getSessionInfo(sessionID); - // get information about all sessions contained in this cookie - function(callback) { - if (!sessionCookie) { - callback(); - return; - } - - var sessionIDs = sessionCookie.split(','); - - async.forEach(sessionIDs, function(sessionID, callback) { - sessionManager.getSessionInfo(sessionID, function(err, sessionInfo) { - // skip session if it doesn't exist - if (err && err.message == "sessionID does not exist") { - authLogger.debug("Auth failed: unknown session"); - callback(); - - return; - } - - if (ERR(err, callback)) return; - - var now = Math.floor(Date.now()/1000); - - // is it for this group? - if (sessionInfo.groupID != groupID) { - authLogger.debug("Auth failed: wrong group"); - callback(); - - return; - } - - // is validUntil still ok? - if (sessionInfo.validUntil <= now) { - authLogger.debug("Auth failed: validUntil"); - callback(); - - return; - } - - // There is a valid session - validSession = true; - sessionAuthor = sessionInfo.authorID; - - callback(); - }); - }, callback); - }, - - // get author for token - function(callback) { - // get author for this token - authorManager.getAuthor4Token(token, function(err, author) { - if (ERR(err, callback)) return; - - tokenAuthor = author; - callback(); - }); - } - ], callback); - }, - - // get more informations of this pad, if avaiable - function(callback) { - // skip this if the pad doesn't exist - if (padExists == false) { - callback(); - - return; - } - - padManager.getPad(padID, function(err, pad) { - if (ERR(err, callback)) return; - - // is it a public pad? - isPublic = pad.getPublicStatus(); - - // is it password protected? - isPasswordProtected = pad.isPasswordProtected(); - - // is password correct? - if (isPasswordProtected && password && pad.isCorrectPassword(password)) { - passwordStatus = "correct"; + // is it for this group? + if (sessionInfo.groupID != groupID) { + authLogger.debug("Auth failed: wrong group"); + continue; } - callback(); - }); - }, + // is validUntil still ok? + let now = Math.floor(Date.now() / 1000); + if (sessionInfo.validUntil <= now) { + authLogger.debug("Auth failed: validUntil"); + continue; + } - function(callback) { - if (validSession && padExists) { - // - a valid session for this group is avaible AND pad exists - if (!isPasswordProtected) { - // - the pad is not password protected - - // --> grant access - statusObject = { accessStatus: "grant", authorID: sessionAuthor }; - } else if (settings.sessionNoPassword) { - // - the setting to bypass password validation is set - - // --> grant access - statusObject = { accessStatus: "grant", authorID: sessionAuthor }; - } else if (isPasswordProtected && passwordStatus === "correct") { - // - the pad is password protected and password is correct - - // --> grant access - statusObject = { accessStatus: "grant", authorID: sessionAuthor }; - } else if (isPasswordProtected && passwordStatus === "wrong") { - // - the pad is password protected but wrong password given - - // --> deny access, ask for new password and tell them that the password is wrong - statusObject = { accessStatus: "wrongPassword" }; - } else if (isPasswordProtected && passwordStatus === "notGiven") { - // - the pad is password protected but no password given - - // --> ask for password - statusObject = { accessStatus: "needPassword" }; + // fall-through - there is a valid session + validSession = true; + sessionAuthor = sessionInfo.authorID; + break; + } catch (err) { + // skip session if it doesn't exist + if (err.message == "sessionID does not exist") { + authLogger.debug("Auth failed: unknown session"); } else { - throw new Error("Ops, something wrong happend"); + throw err; } - } else if (validSession && !padExists) { - // - a valid session for this group avaible but pad doesn't exist - - // --> grant access - statusObject = {accessStatus: "grant", authorID: sessionAuthor}; - - if (settings.editOnly) { - // --> deny access if user isn't allowed to create the pad - authLogger.debug("Auth failed: valid session & pad does not exist"); - statusObject.accessStatus = "deny"; - } - } else if (!validSession && padExists) { - // there is no valid session avaiable AND pad exists - - // -- it's public and not password protected - if (isPublic && !isPasswordProtected) { - // --> grant access, with author of token - statusObject = {accessStatus: "grant", authorID: tokenAuthor}; - } else if (isPublic && isPasswordProtected && passwordStatus === "correct") { - // - it's public and password protected and password is correct - - // --> grant access, with author of token - statusObject = {accessStatus: "grant", authorID: tokenAuthor}; - } else if (isPublic && isPasswordProtected && passwordStatus === "wrong") { - // - it's public and the pad is password protected but wrong password given - - // --> deny access, ask for new password and tell them that the password is wrong - statusObject = {accessStatus: "wrongPassword"}; - } else if (isPublic && isPasswordProtected && passwordStatus === "notGiven") { - // - it's public and the pad is password protected but no password given - - // --> ask for password - statusObject = {accessStatus: "needPassword"}; - } else if (!isPublic) { - // - it's not public - - authLogger.debug("Auth failed: invalid session & pad is not public"); - // --> deny access - statusObject = {accessStatus: "deny"}; - } else { - throw new Error("Ops, something wrong happend"); - } - } else { - // there is no valid session avaiable AND pad doesn't exist - authLogger.debug("Auth failed: invalid session & pad does not exist"); - // --> deny access - statusObject = {accessStatus: "deny"}; } - - callback(); } - ], - function(err) { - if (ERR(err, callback)) return; + } - callback(null, statusObject); - }); -}); + if (padExists) { + let pad = await padManager.getPad(padID); + + // is it a public pad? + isPublic = pad.getPublicStatus(); + + // is it password protected? + isPasswordProtected = pad.isPasswordProtected(); + + // is password correct? + if (isPasswordProtected && password && pad.isCorrectPassword(password)) { + passwordStatus = "correct"; + } + } + + // - a valid session for this group is avaible AND pad exists + if (validSession && padExists) { + let authorID = sessionAuthor; + let grant = Object.freeze({ accessStatus: "grant", authorID }); + + if (!isPasswordProtected) { + // - the pad is not password protected + + // --> grant access + return grant; + } + + if (settings.sessionNoPassword) { + // - the setting to bypass password validation is set + + // --> grant access + return grant; + } + + if (isPasswordProtected && passwordStatus === "correct") { + // - the pad is password protected and password is correct + + // --> grant access + return grant; + } + + if (isPasswordProtected && passwordStatus === "wrong") { + // - the pad is password protected but wrong password given + + // --> deny access, ask for new password and tell them that the password is wrong + return { accessStatus: "wrongPassword" }; + } + + if (isPasswordProtected && passwordStatus === "notGiven") { + // - the pad is password protected but no password given + + // --> ask for password + return { accessStatus: "needPassword" }; + } + + throw new Error("Oops, something wrong happend"); + } + + if (validSession && !padExists) { + // - a valid session for this group avaible but pad doesn't exist + + // --> grant access by default + let accessStatus = "grant"; + let authorID = sessionAuthor; + + // --> deny access if user isn't allowed to create the pad + if (settings.editOnly) { + authLogger.debug("Auth failed: valid session & pad does not exist"); + accessStatus = "deny"; + } + + return { accessStatus, authorID }; + } + + if (!validSession && padExists) { + // there is no valid session avaiable AND pad exists + + let authorID = tokenAuthor; + let grant = Object.freeze({ accessStatus: "grant", authorID }); + + if (isPublic && !isPasswordProtected) { + // -- it's public and not password protected + + // --> grant access, with author of token + return grant; + } + + if (isPublic && isPasswordProtected && passwordStatus === "correct") { + // - it's public and password protected and password is correct + + // --> grant access, with author of token + return grant; + } + + if (isPublic && isPasswordProtected && passwordStatus === "wrong") { + // - it's public and the pad is password protected but wrong password given + + // --> deny access, ask for new password and tell them that the password is wrong + return { accessStatus: "wrongPassword" }; + } + + if (isPublic && isPasswordProtected && passwordStatus === "notGiven") { + // - it's public and the pad is password protected but no password given + + // --> ask for password + return { accessStatus: "needPassword" }; + } + + if (!isPublic) { + // - it's not public + + authLogger.debug("Auth failed: invalid session & pad is not public"); + // --> deny access + return { accessStatus: "deny" }; + } + + throw new Error("Oops, something wrong happend"); + } + + // there is no valid session avaiable AND pad doesn't exist + authLogger.debug("Auth failed: invalid session & pad does not exist"); + return { accessStatus: "deny" }; +} diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 77d7dc6c5..3004ae49c 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -164,7 +164,7 @@ exports.handleDisconnect = function(client) * @param client the client that send this message * @param message the message from the client */ -exports.handleMessage = function(client, message) +exports.handleMessage = async function(client, message) { if (message == null) { return; @@ -181,35 +181,33 @@ exports.handleMessage = function(client, message) return; } - var handleMessageHook = function(callback) { + async function handleMessageHook() { // Allow plugins to bypass the readonly message blocker - hooks.aCallAll("handleMessageSecurity", { client: client, message: message }, function( err, messages ) { - if(ERR(err, callback)) return; - _.each(messages, function(newMessage){ - if ( newMessage === true ) { - thisSession.readonly = false; - } - }); - }); + let messages = await hooks.aCallAll("handleMessageSecurity", { client: client, message: message }); + + for (let message of messages) { + if (message === true) { + thisSession.readonly = false; + break; + } + } + + let dropMessage = false; - var dropMessage = false; // Call handleMessage hook. If a plugin returns null, the message will be dropped. Note that for all messages // handleMessage will be called, even if the client is not authorized - hooks.aCallAll("handleMessage", { client: client, message: message }, function( err, messages ) { - if(ERR(err, callback)) return; - _.each(messages, function(newMessage){ - if ( newMessage === null ) { - dropMessage = true; - } - }); - - // If no plugins explicitly told us to drop the message, its ok to proceed - if(!dropMessage){ callback() }; - }); + messages = await hooks.aCallAll("handleMessage", { client: client, message: message }); + for (let message of messages) { + if (message === null ) { + dropMessage = true; + break; + } + } + return dropMessage; } - var finalHandler = function() { + function finalHandler() { // Check what type of message we get and delegate to the other methods if (message.type == "CLIENT_READY") { handleClientReady(client, message); @@ -256,54 +254,49 @@ exports.handleMessage = function(client, message) return; } - async.series([ - handleMessageHook, + let dropMessage = await handleMessageHook(); + if (!dropMessage) { // check permissions - function(callback) { - // client tried to auth for the first time (first msg from the client) - if (message.type == "CLIENT_READY") { + + // client tried to auth for the first time (first msg from the client) + if (message.type == "CLIENT_READY") { createSessionInfo(client, message); - } + } - // Note: message.sessionID is an entirely different kind of - // session from the sessions we use here! Beware! - // FIXME: Call our "sessions" "connections". - // FIXME: Use a hook instead - // FIXME: Allow to override readwrite access with readonly + // Note: message.sessionID is an entirely different kind of + // session from the sessions we use here! Beware! + // FIXME: Call our "sessions" "connections". + // FIXME: Use a hook instead + // FIXME: Allow to override readwrite access with readonly - // Simulate using the load testing tool - if (!sessioninfos[client.id].auth) { - console.error("Auth was never applied to a session. If you are using the stress-test tool then restart Etherpad and the Stress test tool.") - return; - } + // Simulate using the load testing tool + if (!sessioninfos[client.id].auth) { + console.error("Auth was never applied to a session. If you are using the stress-test tool then restart Etherpad and the Stress test tool.") + return; + } - var auth = sessioninfos[client.id].auth; - var checkAccessCallback = function(err, statusObject) { - if (ERR(err, callback)) return; + let auth = sessioninfos[client.id].auth; - if (statusObject.accessStatus == "grant") { - // access was granted - callback(); - } else { - // no access, send the client a message that tells him why - client.json.send({accessStatus: statusObject.accessStatus}) - } - }; + // check if pad is requested via readOnly + let padId = auth.padID; - // check if pad is requested via readOnly - if (auth.padID.indexOf("r.") === 0) { - // Pad is readOnly, first get the real Pad ID - readOnlyManager.getPadId(auth.padID, function(err, value) { - ERR(err); - securityManager.checkAccess(value, auth.sessionID, auth.token, auth.password, checkAccessCallback); - }); - } else { - securityManager.checkAccess(auth.padID, auth.sessionID, auth.token, auth.password, checkAccessCallback); - } - }, - finalHandler - ]); + // Pad is readOnly, first get the real Pad ID + if (padId.indexOf("r.") === 0) { + padId = await readOnlyManager.getPadId(padID); + } + + let { accessStatus } = await securityManager.checkAccess(padId, auth.sessionID, auth.token, auth.password); + + // no access, send the client a message that tells him why + if (accessStatus !== "grant") { + client.json.send({ accessStatus }); + return; + } + + // access was granted + finalHandler(); + } } @@ -977,7 +970,7 @@ function createSessionInfo(client, message) * @param client the client that send this message * @param message the message from the client */ -function handleClientReady(client, message) +async function handleClientReady(client, message) { // check if all ok if (!message.token) { @@ -1000,434 +993,319 @@ function handleClientReady(client, message) return; } - var author; - var authorName; - var authorColorId; - var pad; var historicalAuthorData = {}; - var currentTime; - var padIds; hooks.callAll("clientReady", message); - async.series([ - // Get ro/rw id:s - function(callback) { - readOnlyManager.getIds(message.padId, function(err, value) { - if (ERR(err, callback)) return; + // Get ro/rw id:s + let padIds = await readOnlyManager.getIds(message.padId); - padIds = value; - callback(); - }); - }, + // check permissions - // check permissions - function(callback) { - // Note: message.sessionID is an entierly different kind of - // session from the sessions we use here! Beware! - // FIXME: Call our "sessions" "connections". - // FIXME: Use a hook instead - // FIXME: Allow to override readwrite access with readonly - securityManager.checkAccess(padIds.padId, message.sessionID, message.token, message.password, function(err, statusObject) { - if (ERR(err, callback)) return; + // Note: message.sessionID is an entierly different kind of + // session from the sessions we use here! Beware! + // FIXME: Call our "sessions" "connections". + // FIXME: Use a hook instead + // FIXME: Allow to override readwrite access with readonly + let statusObject = await securityManager.checkAccess(padIds.padId, message.sessionID, message.token, message.password); + let accessStatus = statusObject.accessStatus; - if (statusObject.accessStatus == "grant") { - // access was granted - author = statusObject.authorID; - callback(); - } else { - // no access, send the client a message that tells him why - client.json.send({accessStatus: statusObject.accessStatus}) - } - }); - }, + // no access, send the client a message that tells him why + if (accessStatus !== "grant") { + client.json.send({ accessStatus }); + return; + } - // get all authordata of this new user, and load the pad-object from the database - function(callback) - { - async.parallel([ - // get colorId and name - function(callback) { - authorManager.getAuthor(author, function(err, value) { - if (ERR(err, callback)) return; + let author = statusObject.authorID; - authorColorId = value.colorId; - authorName = value.name; - callback(); - }); - }, + // get all authordata of this new user, and load the pad-object from the database + let value = await authorManager.getAuthor(author); + let authorColorId = value.colorId; + let authorName = value.name; - // get pad - function(callback) { - padManager.getPad(padIds.padId, function(err, value) { - if (ERR(err, callback)) return; + // get pad + let pad = await padManager.getPad(padIds.padId); - pad = value; - callback(); - }); - } - ], callback); - }, + // these db requests all need the pad object (timestamp of latest revision, author data) + let authors = pad.getAllAuthors(); - // these db requests all need the pad object (timestamp of latest revission, author data) - function(callback) { - var authors = pad.getAllAuthors(); + // get timestamp of latest revision needed for timeslider + let currentTime = await pad.getRevisionDate(pad.getHeadRevisionNumber()); - async.parallel([ - // get timestamp of latest revission needed for timeslider - function(callback) { - pad.getRevisionDate(pad.getHeadRevisionNumber(), function(err, date) { - if (ERR(err, callback)) return; + // get all author data out of the database + for (let authorId of authors) { + try { + let author = await authorManager.getAuthor(authorId); + historicalAuthorData[authorId] = { name: author.name, colorId: author.colorId }; // Filter author attribs (e.g. don't send author's pads to all clients) + } catch (err) { + messageLogger.error("There is no author for authorId:", authorId); + } + } - currentTime = date; - callback(); - }); - }, + // glue the clientVars together, send them and tell the other clients that a new one is there - // get all author data out of the database - function(callback) { - async.forEach(authors, function(authorId, callback) { - authorManager.getAuthor(authorId, function(err, author) { - if (!author && !err) { - messageLogger.error("There is no author for authorId:", authorId); + // Check that the client is still here. It might have disconnected between callbacks. + if (sessioninfos[client.id] === undefined) { + return; + } - return callback(); - } + // Check if this author is already on the pad, if yes, kick the other sessions! + let roomClients = _getRoomClients(pad.id); - if (ERR(err, callback)) return; + for (let client of roomClients) { + let sinfo = sessioninfos[client.id]; + if (sinfo && sinfo.author == author) { + // fix user's counter, works on page refresh or if user closes browser window and then rejoins + sessioninfos[client.id] = {}; + client.leave(padIds.padId); + client.json.send({disconnect:"userdup"}); + } + } - historicalAuthorData[authorId] = { name: author.name, colorId: author.colorId }; // Filter author attribs (e.g. don't send author's pads to all clients) - callback(); - }); - }, callback); - } - ], callback); + // Save in sessioninfos that this session belonges to this pad + sessioninfos[client.id].padId = padIds.padId; + sessioninfos[client.id].readOnlyPadId = padIds.readOnlyPadId; + sessioninfos[client.id].readonly = padIds.readonly; - }, + // Log creation/(re-)entering of a pad + let ip = remoteAddress[client.id]; - // glue the clientVars together, send them and tell the other clients that a new one is there - function(callback) { - // Check that the client is still here. It might have disconnected between callbacks. - if(sessioninfos[client.id] === undefined) { - return callback(); - } + // Anonymize the IP address if IP logging is disabled + if (settings.disableIPlogging) { + ip = 'ANONYMOUS'; + } - // Check if this author is already on the pad, if yes, kick the other sessions! - var roomClients = _getRoomClients(pad.id); + if (pad.head > 0) { + accessLogger.info('[ENTER] Pad "' + padIds.padId + '": Client ' + client.id + ' with IP "' + ip + '" entered the pad'); + } else if (pad.head == 0) { + accessLogger.info('[CREATE] Pad "' + padIds.padId + '": Client ' + client.id + ' with IP "' + ip + '" created the pad'); + } - async.forEach(roomClients, function(client, callback) { - var sinfo = sessioninfos[client.id]; + if (message.reconnect) { + // If this is a reconnect, we don't have to send the client the ClientVars again + // Join the pad and start receiving updates + client.join(padIds.padId); - if (sinfo && sinfo.author == author) { - // fix user's counter, works on page refresh or if user closes browser window and then rejoins - sessioninfos[client.id] = {}; - client.leave(padIds.padId); - client.json.send({ disconnect:"userdup" }); - } - }); + // Save the revision in sessioninfos, we take the revision from the info the client send to us + sessioninfos[client.id].rev = message.client_rev; - // Save in sessioninfos that this session belonges to this pad - sessioninfos[client.id].padId = padIds.padId; - sessioninfos[client.id].readOnlyPadId = padIds.readOnlyPadId; - sessioninfos[client.id].readonly = padIds.readonly; + // During the client reconnect, client might miss some revisions from other clients. By using client revision, + // this below code sends all the revisions missed during the client reconnect + var revisionsNeeded = []; + var changesets = {}; - // Log creation/(re-)entering of a pad - var ip = remoteAddress[client.id]; + var startNum = message.client_rev + 1; + var endNum = pad.getHeadRevisionNumber() + 1; - // Anonymize the IP address if IP logging is disabled - if (settings.disableIPlogging) { - ip = 'ANONYMOUS'; - } + var headNum = pad.getHeadRevisionNumber(); - if (pad.head > 0) { - accessLogger.info('[ENTER] Pad "' + padIds.padId + '": Client ' + client.id + ' with IP "' + ip + '" entered the pad'); - } else if (pad.head == 0) { - accessLogger.info('[CREATE] Pad "' + padIds.padId + '": Client ' + client.id + ' with IP "' + ip + '" created the pad'); - } + if (endNum > headNum + 1) { + endNum = headNum + 1; + } - if (message.reconnect == true) { - // If this is a reconnect, we don't have to send the client the ClientVars again - // Join the pad and start receiving updates - client.join(padIds.padId); + if (startNum < 0) { + startNum = 0; + } - // Save the revision in sessioninfos, we take the revision from the info the client send to us - sessioninfos[client.id].rev = message.client_rev; + for (let r = startNum; r < endNum; r++) { + revisionsNeeded.push(r); + changesets[r] = {}; + } - // During the client reconnect, client might miss some revisions from other clients. By using client revision, - // this below code sends all the revisions missed during the client reconnect - var revisionsNeeded = []; - var changesets = {}; + // get changesets, author and timestamp needed for pending revisions + for (let revNum of revisionsNeeded) { + changesets[revNum]['changeset'] = await pad.getRevisionChangeset(revNum); + changesets[revNum]['author'] = await pad.getRevisionAuthor(revNum); + changesets[revNum]['timestamp'] = await pad.getRevisionDate(revNum); + } - var startNum = message.client_rev + 1; - var endNum = pad.getHeadRevisionNumber() + 1; + // return pending changesets + for (let r of revisionsNeeded) { - async.series([ - // push all the revision numbers needed into revisionsNeeded array - function(callback) { - var headNum = pad.getHeadRevisionNumber(); - - if (endNum > headNum+1) { - endNum = headNum+1; - } - - if (startNum < 0) { - startNum = 0; - } - - for (var r = startNum; r < endNum; r++) { - revisionsNeeded.push(r); - changesets[r] = {}; - } - - callback(); - }, - - // get changesets needed for pending revisions - function(callback) { - async.eachSeries(revisionsNeeded, function(revNum, callback) { - pad.getRevisionChangeset(revNum, function(err, value) { - if (ERR(err)) return; - - changesets[revNum]['changeset'] = value; - callback(); - }); - }, callback); - }, - - // get author for each changeset - function(callback) { - async.eachSeries(revisionsNeeded, function(revNum, callback) { - pad.getRevisionAuthor(revNum, function(err, value) { - if (ERR(err)) return; - - changesets[revNum]['author'] = value; - callback(); - }); - }, callback); - }, - - // get timestamp for each changeset - function(callback) { - async.eachSeries(revisionsNeeded, function(revNum, callback) { - pad.getRevisionDate(revNum, function(err, value) { - if (ERR(err)) return; - - changesets[revNum]['timestamp'] = value; - callback(); - }); - }, callback); - } - ], - - // return error and pending changesets - function(err) { - if (ERR(err, callback)) return; - - async.eachSeries(revisionsNeeded, function(r, callback) { - var forWire = Changeset.prepareForWire(changesets[r]['changeset'], pad.pool); - var wireMsg = {"type":"COLLABROOM", - "data":{type:"CLIENT_RECONNECT", - headRev:pad.getHeadRevisionNumber(), - newRev:r, - changeset:forWire.translated, - apool: forWire.pool, - author: changesets[r]['author'], - currentTime: changesets[r]['timestamp'] + let forWire = Changeset.prepareForWire(changesets[r]['changeset'], pad.pool); + let wireMsg = {"type":"COLLABROOM", + "data":{type:"CLIENT_RECONNECT", + headRev:pad.getHeadRevisionNumber(), + newRev:r, + changeset:forWire.translated, + apool: forWire.pool, + author: changesets[r]['author'], + currentTime: changesets[r]['timestamp'] }}; - client.json.send(wireMsg); - callback(); - }); + client.json.send(wireMsg); + } - if (startNum == endNum) { - var Msg = {"type":"COLLABROOM", - "data":{type:"CLIENT_RECONNECT", - noChanges: true, - newRev: pad.getHeadRevisionNumber() - }}; - client.json.send(Msg); - } - }); - } else { - // This is a normal first connect - // prepare all values for the wire, there's a chance that this throws, if the pad is corrupted - try { - var atext = Changeset.cloneAText(pad.atext); - var attribsForWire = Changeset.prepareForWire(atext.attribs, pad.pool); - var apool = attribsForWire.pool.toJsonable(); - atext.attribs = attribsForWire.translated; - } catch(e) { - console.error(e.stack || e) - client.json.send({ disconnect:"corruptPad" });// pull the brakes - return callback(); - } + if (startNum == endNum) { + var Msg = {"type":"COLLABROOM", + "data":{type:"CLIENT_RECONNECT", + noChanges: true, + newRev: pad.getHeadRevisionNumber() + }}; + client.json.send(Msg); + } - // Warning: never ever send padIds.padId to the client. If the - // client is read only you would open a security hole 1 swedish - // mile wide... - var clientVars = { - "skinName": settings.skinName, - "accountPrivs": { - "maxRevisions": 100 - }, - "automaticReconnectionTimeout": settings.automaticReconnectionTimeout, - "initialRevisionList": [], - "initialOptions": { - "guestPolicy": "deny" - }, - "savedRevisions": pad.getSavedRevisions(), - "collab_client_vars": { - "initialAttributedText": atext, - "clientIp": "127.0.0.1", - "padId": message.padId, - "historicalAuthorData": historicalAuthorData, - "apool": apool, - "rev": pad.getHeadRevisionNumber(), - "time": currentTime, - }, - "colorPalette": authorManager.getColorPalette(), + } else { + // This is a normal first connect + + // prepare all values for the wire, there's a chance that this throws, if the pad is corrupted + try { + var atext = Changeset.cloneAText(pad.atext); + var attribsForWire = Changeset.prepareForWire(atext.attribs, pad.pool); + var apool = attribsForWire.pool.toJsonable(); + atext.attribs = attribsForWire.translated; + } catch(e) { + console.error(e.stack || e) + client.json.send({ disconnect:"corruptPad" }); // pull the brakes + + return; + } + + // Warning: never ever send padIds.padId to the client. If the + // client is read only you would open a security hole 1 swedish + // mile wide... + var clientVars = { + "skinName": settings.skinName, + "accountPrivs": { + "maxRevisions": 100 + }, + "automaticReconnectionTimeout": settings.automaticReconnectionTimeout, + "initialRevisionList": [], + "initialOptions": { + "guestPolicy": "deny" + }, + "savedRevisions": pad.getSavedRevisions(), + "collab_client_vars": { + "initialAttributedText": atext, "clientIp": "127.0.0.1", - "userIsGuest": true, - "userColor": authorColorId, "padId": message.padId, - "padOptions": settings.padOptions, - "padShortcutEnabled": settings.padShortcutEnabled, - "initialTitle": "Pad: " + message.padId, - "opts": {}, - // tell the client the number of the latest chat-message, which will be - // used to request the latest 100 chat-messages later (GET_CHAT_MESSAGES) - "chatHead": pad.chatHead, - "numConnectedUsers": roomClients.length, - "readOnlyId": padIds.readOnlyPadId, - "readonly": padIds.readonly, - "serverTimestamp": Date.now(), - "userId": author, - "abiwordAvailable": settings.abiwordAvailable(), - "sofficeAvailable": settings.sofficeAvailable(), - "exportAvailable": settings.exportAvailable(), - "plugins": { - "plugins": plugins.plugins, - "parts": plugins.parts, - }, - "indentationOnNewLine": settings.indentationOnNewLine, - "scrollWhenFocusLineIsOutOfViewport": { - "percentage" : { - "editionAboveViewport": settings.scrollWhenFocusLineIsOutOfViewport.percentage.editionAboveViewport, - "editionBelowViewport": settings.scrollWhenFocusLineIsOutOfViewport.percentage.editionBelowViewport, - }, - "duration": settings.scrollWhenFocusLineIsOutOfViewport.duration, - "scrollWhenCaretIsInTheLastLineOfViewport": settings.scrollWhenFocusLineIsOutOfViewport.scrollWhenCaretIsInTheLastLineOfViewport, - "percentageToScrollWhenUserPressesArrowUp": settings.scrollWhenFocusLineIsOutOfViewport.percentageToScrollWhenUserPressesArrowUp, - }, - "initialChangesets": [] // FIXME: REMOVE THIS SHIT + "historicalAuthorData": historicalAuthorData, + "apool": apool, + "rev": pad.getHeadRevisionNumber(), + "time": currentTime, + }, + "colorPalette": authorManager.getColorPalette(), + "clientIp": "127.0.0.1", + "userIsGuest": true, + "userColor": authorColorId, + "padId": message.padId, + "padOptions": settings.padOptions, + "padShortcutEnabled": settings.padShortcutEnabled, + "initialTitle": "Pad: " + message.padId, + "opts": {}, + // tell the client the number of the latest chat-message, which will be + // used to request the latest 100 chat-messages later (GET_CHAT_MESSAGES) + "chatHead": pad.chatHead, + "numConnectedUsers": roomClients.length, + "readOnlyId": padIds.readOnlyPadId, + "readonly": padIds.readonly, + "serverTimestamp": Date.now(), + "userId": author, + "abiwordAvailable": settings.abiwordAvailable(), + "sofficeAvailable": settings.sofficeAvailable(), + "exportAvailable": settings.exportAvailable(), + "plugins": { + "plugins": plugins.plugins, + "parts": plugins.parts, + }, + "indentationOnNewLine": settings.indentationOnNewLine, + "scrollWhenFocusLineIsOutOfViewport": { + "percentage" : { + "editionAboveViewport": settings.scrollWhenFocusLineIsOutOfViewport.percentage.editionAboveViewport, + "editionBelowViewport": settings.scrollWhenFocusLineIsOutOfViewport.percentage.editionBelowViewport, + }, + "duration": settings.scrollWhenFocusLineIsOutOfViewport.duration, + "scrollWhenCaretIsInTheLastLineOfViewport": settings.scrollWhenFocusLineIsOutOfViewport.scrollWhenCaretIsInTheLastLineOfViewport, + "percentageToScrollWhenUserPressesArrowUp": settings.scrollWhenFocusLineIsOutOfViewport.percentageToScrollWhenUserPressesArrowUp, + }, + "initialChangesets": [] // FIXME: REMOVE THIS SHIT + } + + // Add a username to the clientVars if one avaiable + if (authorName != null) { + clientVars.userName = authorName; + } + + // call the clientVars-hook so plugins can modify them before they get sent to the client + let messages = await hooks.aCallAll("clientVars", { clientVars: clientVars, pad: pad }); + + // combine our old object with the new attributes from the hook + for (let msg of messages) { + Object.assign(clientVars, msg); + } + + // Join the pad and start receiving updates + client.join(padIds.padId); + + // Send the clientVars to the Client + client.json.send({type: "CLIENT_VARS", data: clientVars}); + + // Save the current revision in sessioninfos, should be the same as in clientVars + sessioninfos[client.id].rev = pad.getHeadRevisionNumber(); + + sessioninfos[client.id].author = author; + + // prepare the notification for the other users on the pad, that this user joined + let messageToTheOtherUsers = { + "type": "COLLABROOM", + "data": { + type: "USER_NEWINFO", + userInfo: { + "ip": "127.0.0.1", + "colorId": authorColorId, + "userAgent": "Anonymous", + "userId": author } + } + }; - // Add a username to the clientVars if one avaiable - if (authorName != null) { - clientVars.userName = authorName; - } + // Add the authorname of this new User, if avaiable + if (authorName != null) { + messageToTheOtherUsers.data.userInfo.name = authorName; + } - // call the clientVars-hook so plugins can modify them before they get sent to the client - hooks.aCallAll("clientVars", { clientVars: clientVars, pad: pad }, function( err, messages ) { - if (ERR(err, callback)) return; + // notify all existing users about new user + client.broadcast.to(padIds.padId).json.send(messageToTheOtherUsers); - _.each(messages, function(newVars) { - // combine our old object with the new attributes from the hook - for(var attr in newVars) { - clientVars[attr] = newVars[attr]; - } - }); + // Get sessions for this pad + roomClients = _getRoomClients(pad.id); + for (let roomClient of roomClients) { - // Join the pad and start receiving updates - client.join(padIds.padId); - - // Send the clientVars to the Client - client.json.send({ type: "CLIENT_VARS", data: clientVars }); - - // Save the current revision in sessioninfos, should be the same as in clientVars - sessioninfos[client.id].rev = pad.getHeadRevisionNumber(); - }); + // Jump over, if this session is the connection session + if (roomClient.id == client.id) { + continue; } - sessioninfos[client.id].author = author; + // Since sessioninfos might change while being enumerated, check if the + // sessionID is still assigned to a valid session + if (sessioninfos[roomClient.id] === undefined) { + continue; + } - // prepare the notification for the other users on the pad, that this user joined - var messageToTheOtherUsers = { + let author = sessioninfos[roomClient.id].author; + + // get the authorname & colorId + + // reuse previously created cache of author's data + let authorInfo = historicalAuthorData[author] || await authorManager.getAuthor(author); + + // Send the new User a Notification about this other user + let msg = { "type": "COLLABROOM", "data": { type: "USER_NEWINFO", userInfo: { "ip": "127.0.0.1", - "colorId": authorColorId, + "colorId": authorInfo.colorId, + "name": authorInfo.name, "userAgent": "Anonymous", "userId": author } } }; - // Add the authorname of this new User, if avaiable - if (authorName != null) { - messageToTheOtherUsers.data.userInfo.name = authorName; - } - - // notify all existing users about new user - client.broadcast.to(padIds.padId).json.send(messageToTheOtherUsers); - - // Get sessions for this pad - var roomClients = _getRoomClients(pad.id); - - async.forEach(roomClients, function(roomClient, callback) { - var author; - - // Jump over, if this session is the connection session - if (roomClient.id == client.id) { - return callback(); - } - - // Since sessioninfos might change while being enumerated, check if the - // sessionID is still assigned to a valid session - if (sessioninfos[roomClient.id] !== undefined) { - author = sessioninfos[roomClient.id].author; - } else { - // If the client id is not valid, callback(); - return callback(); - } - - async.waterfall([ - // get the authorname & colorId - function(callback) { - // reuse previously created cache of author's data - if (historicalAuthorData[author]) { - callback(null, historicalAuthorData[author]); - } else { - authorManager.getAuthor(author, callback); - } - }, - - function(authorInfo, callback) { - // Send the new User a Notification about this other user - var msg = { - "type": "COLLABROOM", - "data": { - type: "USER_NEWINFO", - userInfo: { - "ip": "127.0.0.1", - "colorId": authorInfo.colorId, - "name": authorInfo.name, - "userAgent": "Anonymous", - "userId": author - } - } - }; - - client.json.send(msg); - } - ], callback); - }, callback); + client.json.send(msg); } - ], - function(err) { - ERR(err); - }); + } } /** @@ -1496,7 +1374,6 @@ function handleChangesetRequest(client, message) ]); } - /** * Tries to rebuild the getChangestInfo function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L144 diff --git a/src/node/handler/SocketIORouter.js b/src/node/handler/SocketIORouter.js index 08f9f47e8..27c1e9a22 100644 --- a/src/node/handler/SocketIORouter.js +++ b/src/node/handler/SocketIORouter.js @@ -19,7 +19,6 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var log4js = require('log4js'); var messageLogger = log4js.getLogger("message"); var securityManager = require("../db/SecurityManager"); @@ -80,7 +79,7 @@ exports.setSocketIO = function(_socket) { components[i].handleConnect(client); } - client.on('message', function(message) { + client.on('message', async function(message) { if (message.protocolVersion && message.protocolVersion != 2) { messageLogger.warn("Protocolversion header is not correct:" + stringifyWithoutPassword(message)); return; @@ -92,27 +91,22 @@ exports.setSocketIO = function(_socket) { } else { // try to authorize the client if (message.padId !== undefined && message.sessionID !== undefined && message.token !== undefined && message.password !== undefined) { - var checkAccessCallback = function(err, statusObject) { - ERR(err); + // check for read-only pads + let padId = message.padId; + if (padId.indexOf("r.") === 0) { + padId = await readOnlyManager.getPadId(message.padId); + } - if (statusObject.accessStatus === "grant") { - // access was granted, mark the client as authorized and handle the message - clientAuthorized = true; - handleMessage(client, message); - } else { - // no access, send the client a message that tells him why - messageLogger.warn("Authentication try failed:" + stringifyWithoutPassword(message)); - client.json.send({accessStatus: statusObject.accessStatus}); - } - }; - if (message.padId.indexOf("r.") === 0) { - readOnlyManager.getPadId(message.padId, function(err, value) { - ERR(err); - securityManager.checkAccess(value, message.sessionID, message.token, message.password, checkAccessCallback); - }); + let { accessStatus } = await securityManager.checkAccess(padId, message.sessionID, message.token, message.password); + + if (accessStatus === "grant") { + // access was granted, mark the client as authorized and handle the message + clientAuthorized = true; + handleMessage(client, message); } else { - // this message has everything to try an authorization - securityManager.checkAccess (message.padId, message.sessionID, message.token, message.password, checkAccessCallback); + // no access, send the client a message that tells him why + messageLogger.warn("Authentication try failed:" + stringifyWithoutPassword(message)); + client.json.send({ accessStatus }); } } else { // drop message From 005c0afa97e348e95a9ff8ee8ff0e480c6daf4f1 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Mon, 28 Jan 2019 14:44:36 +0000 Subject: [PATCH 37/58] db/SessionManager.js: completely converted to Promises/async --- src/node/db/SessionManager.js | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/node/db/SessionManager.js b/src/node/db/SessionManager.js index 67d3e3816..9161205d7 100644 --- a/src/node/db/SessionManager.js +++ b/src/node/db/SessionManager.js @@ -18,13 +18,11 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); var randomString = require("../utils/randomstring"); var db = require("./DB"); var groupManager = require("./GroupManager"); var authorManager = require("./AuthorManager"); -const thenify = require("thenify").withCallback; exports.doesSessionExist = async function(sessionID) { @@ -120,23 +118,19 @@ exports.createSession = async function(groupID, authorID, validUntil) return { sessionID }; } -// @TODO once external dependencies are using Promises -exports.getSessionInfo = thenify(function(sessionID, callback) +exports.getSessionInfo = async function(sessionID) { // check if the database entry of this session exists - db.get("session:" + sessionID, function (err, session) - { - if(ERR(err, callback)) return; + let session = await db.get("session:" + sessionID); - if (session == null) { - // session does not exist - callback(new customError("sessionID does not exist", "apierror")) - } else { - // everything is fine, return the sessioninfos - callback(null, session); - } - }); -}); + if (session == null) { + // session does not exist + throw new customError("sessionID does not exist", "apierror"); + } + + // everything is fine, return the sessioninfos + return session; +} /** * Deletes a session @@ -199,7 +193,7 @@ exports.listSessionsOfAuthor = async function(authorID) // this function is basically the code listSessionsOfAuthor and listSessionsOfGroup has in common // required to return null rather than an empty object if there are none -async function listSessionsWithDBKey(dbkey, callback) +async function listSessionsWithDBKey(dbkey) { // get the group2sessions entry let sessionObject = await db.get(dbkey); From 81089644725e9510503af6c017c2887799895b49 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Mon, 28 Jan 2019 15:36:36 +0000 Subject: [PATCH 38/58] db/AuthorManager.js: further conversion also fixes a missing await calling `.createAuthor` in db/Pad.js --- src/node/db/API.js | 2 +- src/node/db/AuthorManager.js | 90 ++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/src/node/db/API.js b/src/node/db/API.js index f3274a22a..3fcf187cf 100644 --- a/src/node/db/API.js +++ b/src/node/db/API.js @@ -453,7 +453,7 @@ exports.saveRevision = async function(padID, rev) rev = pad.getHeadRevisionNumber(); } - let author = authorManager.createAuthor('API'); + let author = await authorManager.createAuthor('API'); pad.addSavedRevision(rev, author.authorID, 'Saved through API call'); } diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index d204b64ab..99584da19 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -18,7 +18,6 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var db = require("./DB"); var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; @@ -53,17 +52,14 @@ exports.doesAuthorExists = exports.doesAuthorExist; /** * Returns the AuthorID for a token. * @param {String} token The token - * @param {Function} callback callback (err, author) */ -exports.getAuthor4Token = thenify(function(token, callback) +exports.getAuthor4Token = async function(token) { - mapAuthorWithDBKey("token2author", token, function(err, author) { - if (ERR(err, callback)) return; + let author = await mapAuthorWithDBKey("token2author", token); - // return only the sub value authorID - callback(null, author ? author.authorID : author); - }); -}); + // return only the sub value authorID + return author ? author.authorID : author; +} /** * Returns the AuthorID for a mapper. @@ -87,69 +83,63 @@ exports.createAuthorIfNotExistsFor = async function(authorMapper, name) * so far this is token2author and mapper2author * @param {String} mapperkey The database key name for this mapper * @param {String} mapper The mapper - * @param {Function} callback callback (err, author) */ -let mapAuthorWithDBKey = thenify(function mapAuthorWithDBKey (mapperkey, mapper, callback) +async function mapAuthorWithDBKey (mapperkey, mapper) { // try to map to an author - db.db.get(mapperkey + ":" + mapper, function(err, author) { - if (ERR(err, callback)) return; + let author = await db.get(mapperkey + ":" + mapper); - if (author === null) { - // there is no author with this mapper, so create one - exports.createAuthor(null, function(err, author) { - if (ERR(err, callback)) return; + if (author === null) { + // there is no author with this mapper, so create one + let author = await exports.createAuthor(null); - // create the token2author relation - db.db.set(mapperkey + ":" + mapper, author.authorID); - - // return the author - callback(null, author); - }); - - return; - } - - // there is an author with this mapper - // update the timestamp of this author - db.db.setSub("globalAuthor:" + author, ["timestamp"], Date.now()); + // create the token2author relation + await db.set(mapperkey + ":" + mapper, author.authorID); // return the author - callback(null, {authorID: author}); - }); -}); + return author; + } + + // there is an author with this mapper + // update the timestamp of this author + await db.setSub("globalAuthor:" + author, ["timestamp"], Date.now()); + + // return the author + return { authorID: author}; +} /** * Internal function that creates the database entry for an author * @param {String} name The name of the author */ -exports.createAuthor = thenify(function(name, callback) +exports.createAuthor = function(name) { // create the new author name - var author = "a." + randomString(16); + let author = "a." + randomString(16); // create the globalAuthors db entry - var authorObj = { + let authorObj = { "colorId": Math.floor(Math.random() * (exports.getColorPalette().length)), "name": name, "timestamp": Date.now() }; // set the global author db entry - db.db.set("globalAuthor:" + author, authorObj); + // NB: no await, since we're not waiting for the DB set to finish + db.set("globalAuthor:" + author, authorObj); - callback(null, {authorID: author}); -}); + return { authorID: author }; +} /** * Returns the Author Obj of the author * @param {String} author The id of the author - * @param {Function} callback callback(err, authorObj) */ -exports.getAuthor = thenify(function(author, callback) +exports.getAuthor = function(author) { - db.db.get("globalAuthor:" + author, callback); -}); + // NB: result is already a Promise + return db.get("globalAuthor:" + author); +} /** * Returns the color Id of the author @@ -165,12 +155,11 @@ exports.getAuthorColorId = thenify(function(author, callback) * Sets the color Id of the author * @param {String} author The id of the author * @param {String} colorId The color id of the author - * @param {Function} callback (optional) */ -exports.setAuthorColorId = thenify(function(author, colorId, callback) +exports.setAuthorColorId = function(author, colorId) { - db.db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback); -}); + return db.setSub("globalAuthor:" + author, ["colorId"], colorId); +} /** * Returns the name of the author @@ -186,12 +175,11 @@ exports.getAuthorName = thenify(function(author, callback) * Sets the name of the author * @param {String} author The id of the author * @param {String} name The name of the author - * @param {Function} callback (optional) */ -exports.setAuthorName = thenify(function(author, name, callback) +exports.setAuthorName = function(author, name) { - db.db.setSub("globalAuthor:" + author, ["name"], name, callback); -}); + return db.setSub("globalAuthor:" + author, ["name"], name); +} /** * Returns an array of all pads this author contributed to From bbe4a5f756ed0150064cad098d36ce964d0b54b1 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Mon, 28 Jan 2019 16:20:30 +0000 Subject: [PATCH 39/58] db/PadManager.js: more conversion to Promises/async --- src/node/db/PadManager.js | 61 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/src/node/db/PadManager.js b/src/node/db/PadManager.js index e478636b0..858ce5b45 100644 --- a/src/node/db/PadManager.js +++ b/src/node/db/PadManager.js @@ -50,46 +50,43 @@ var globalPads = { * * Updated without db access as new pads are created/old ones removed. */ -var padList = { +let padList = { list: [], sorted : false, initiated: false, - init: thenify(function(cb) { - db.findKeys("pad:*", "*:*:*", function(err, dbData) { - if (ERR(err, cb)) return; + init: async function() { + let dbData = await db.findKeys("pad:*", "*:*:*"); - if (dbData != null) { - padList.initiated = true - dbData.forEach(function(val) { - padList.addPad(val.replace(/pad:/,""),false); - }); + if (dbData != null) { + this.initiated = true; - cb && cb(); + for (let val of dbData) { + this.addPad(val.replace(/pad:/,""), false); } - }); + } return this; - }), - load: thenify(function(cb) { - if (this.initiated) { - cb && cb(); - } else { - this.init(cb); + }, + load: async function() { + if (!this.initiated) { + return this.init(); } - }), + + return this; + }, /** * Returns all pads in alphabetical order as array. */ - getPads: thenify(function(cb) { - this.load(function() { - if (!padList.sorted) { - padList.list = padList.list.sort(); - padList.sorted = true; - } + getPads: async function() { + await this.load(); - cb && cb(padList.list); - }) - }), + if (!this.sorted) { + this.list.sort(); + this.sorted = true; + } + + return this.list; + }, addPad: function(name) { if (!this.initiated) return; @@ -171,12 +168,12 @@ exports.getPad = thenify(function(id, text, callback) }); }); -exports.listAllPads = thenify(function(cb) +exports.listAllPads = async function() { - padList.getPads(function(list) { - cb && cb(null, {padIDs: list}); - }); -}); + let padIDs = await padList.getPads(); + + return { padIDs }; +} // checks if a pad exists exports.doesPadExist = thenify(function(padId, callback) From 7f19033cc06e2cc0dbd0482dd1bad87494f3c87f Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 10:25:01 +0000 Subject: [PATCH 40/58] SocketIORouter: code formatting cleanups --- src/node/handler/SocketIORouter.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/node/handler/SocketIORouter.js b/src/node/handler/SocketIORouter.js index 27c1e9a22..077a62beb 100644 --- a/src/node/handler/SocketIORouter.js +++ b/src/node/handler/SocketIORouter.js @@ -75,7 +75,7 @@ exports.setSocketIO = function(_socket) { } // tell all components about this connect - for (var i in components) { + for (let i in components) { components[i].handleConnect(client); } @@ -117,7 +117,7 @@ exports.setSocketIO = function(_socket) { client.on('disconnect', function() { // tell all components about this disconnect - for (var i in components) { + for (let i in components) { components[i].handleDisconnect(client); } }); @@ -142,14 +142,10 @@ function handleMessage(client, message) // this ensures there are no passwords in the log function stringifyWithoutPassword(message) { - var newMessage = {}; + let newMessage = Object.assign({}, message); - for (var i in message) { - if (i == "password" && message[i] != null) { - newMessage["password"] = "xxx"; - } else { - newMessage[i] = message[i]; - } + if (newMessage.password != null) { + newMessage.password = "xxx"; } return JSON.stringify(newMessage); From 982d4f380a284a6d64e6314bf232c8e4ea542b83 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 10:25:46 +0000 Subject: [PATCH 41/58] db/Pad.js: start use of promise DB methods --- src/node/db/Pad.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index 34d000631..4abf2cab2 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -6,7 +6,7 @@ var ERR = require("async-stacktrace"); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var AttributePool = require("ep_etherpad-lite/static/js/AttributePool"); -var db = require("./DB").db; +var db = require("./DB"); var async = require("async"); var settings = require('../utils/Settings'); var authorManager = require("./AuthorManager"); @@ -128,25 +128,25 @@ Pad.prototype.saveToDatabase = function saveToDatabase() { } } - db.set("pad:" + this.id, dbObject); + db.db.set("pad:" + this.id, dbObject); } // get time of last edit (changeset application) -Pad.prototype.getLastEdit = thenify(function getLastEdit(callback) { +Pad.prototype.getLastEdit = function getLastEdit() { var revNum = this.getHeadRevisionNumber(); - db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); -}); + return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]); +} Pad.prototype.getRevisionChangeset = thenify(function getRevisionChangeset(revNum, callback) { - db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"], callback); + db.db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"], callback); }); Pad.prototype.getRevisionAuthor = thenify(function getRevisionAuthor(revNum, callback) { - db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"], callback); + db.db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"], callback); }); Pad.prototype.getRevisionDate = thenify(function getRevisionDate(revNum, callback) { - db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); + db.db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); }); Pad.prototype.getAllAuthors = function getAllAuthors() { @@ -182,7 +182,7 @@ Pad.prototype.getInternalRevisionAText = thenify(function getInternalRevisionATe async.parallel([ // get the atext of the key revision function (callback) { - db.getSub("pad:" + _this.id + ":revs:" + keyRev, ["meta", "atext"], function(err, _atext) { + db.db.getSub("pad:" + _this.id + ":revs:" + keyRev, ["meta", "atext"], function(err, _atext) { if (ERR(err, callback)) return; try { atext = Changeset.cloneAText(_atext); @@ -232,7 +232,7 @@ Pad.prototype.getInternalRevisionAText = thenify(function getInternalRevisionATe }); Pad.prototype.getRevision = thenify(function getRevisionChangeset(revNum, callback) { - db.get("pad:" + this.id + ":revs:" + revNum, callback); + db.db.get("pad:" + this.id + ":revs:" + revNum, callback); }); Pad.prototype.getAllAuthorColors = thenify(function getAllAuthorColors(callback) { @@ -333,7 +333,7 @@ Pad.prototype.getChatMessage = thenify(function getChatMessage(entryNum, callbac async.series([ // get the chat entry function(callback) { - db.get("pad:" + _this.id + ":chat:" + entryNum, function(err, _entry) { + db.db.get("pad:" + _this.id + ":chat:" + entryNum, function(err, _entry) { if (ERR(err, callback)) return; entry = _entry; callback(); @@ -410,7 +410,7 @@ Pad.prototype.init = thenify(function init(text, callback) { } // try to load the pad - db.get("pad:" + this.id, function(err, value) { + db.db.get("pad:" + this.id, function(err, value) { if (ERR(err, callback)) return; // if this pad exists, load it @@ -509,7 +509,7 @@ Pad.prototype.copy = thenify(function copy(destinationID, force, callback) { // copy the 'pad' entry function(callback) { - db.get("pad:" + sourceID, function(err, pad) { + db.db.get("pad:" + sourceID, function(err, pad) { db.set("pad:" + destinationID, pad); }); @@ -524,7 +524,7 @@ Pad.prototype.copy = thenify(function copy(destinationID, force, callback) { var chatHead = _this.chatHead; for (var i=0; i <= chatHead; i++) { - db.get("pad:" + sourceID + ":chat:" + i, function (err, chat) { + db.db.get("pad:" + sourceID + ":chat:" + i, function (err, chat) { if (ERR(err, callback)) return; db.set("pad:" + destinationID + ":chat:" + i, chat); }); @@ -537,7 +537,7 @@ Pad.prototype.copy = thenify(function copy(destinationID, force, callback) { function(callback) { var revHead = _this.head; for (var i=0; i <= revHead; i++) { - db.get("pad:" + sourceID + ":revs:" + i, function (err, rev) { + db.db.get("pad:" + sourceID + ":revs:" + i, function (err, rev) { if (ERR(err, callback)) return; db.set("pad:" + destinationID + ":revs:" + i, rev); }); @@ -606,7 +606,7 @@ Pad.prototype.remove = thenify(function remove(callback) { // it is a group pad var groupID = padID.substring(0, padID.indexOf("$")); - db.get("group:" + groupID, function (err, group) { + db.db.get("group:" + groupID, function (err, group) { if (ERR(err, callback)) return; // remove the pad entry From 58d0e6cea462cac9571d2f7b03f2b6a07a7ed0df Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 10:41:10 +0000 Subject: [PATCH 42/58] APIHandler.js: further cleanup - removed possible issue with failing to sanitize `padName` if `padId` was also supplied - removed unnecessary `try` block - simplified API and function name matching tests --- src/node/handler/APIHandler.js | 48 +++++++++++----------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/src/node/handler/APIHandler.js b/src/node/handler/APIHandler.js index 07b9b55c5..3898daaf5 100644 --- a/src/node/handler/APIHandler.js +++ b/src/node/handler/APIHandler.js @@ -19,7 +19,6 @@ */ var absolutePaths = require('../utils/AbsolutePaths'); -var ERR = require("async-stacktrace"); var fs = require("fs"); var api = require("../db/API"); var log4js = require('log4js'); @@ -152,35 +151,16 @@ exports.version = version; */ exports.handle = async function(apiVersion, functionName, fields, req, res) { - //check if this is a valid apiversion - var isKnownApiVersion = false; - - for (var knownApiVersion in version) { - if (knownApiVersion == apiVersion) { - isKnownApiVersion = true; - break; - } - } - // say goodbye if this is an unknown API version - if (!isKnownApiVersion) { + if (!(apiVersion in version)) { res.statusCode = 404; res.send({code: 3, message: "no such api version", data: null}); return; } - // check if this is a valid function name - var isKnownFunctionname = false; - - for (var knownFunctionname in version[apiVersion]) { - if (knownFunctionname == functionName) { - isKnownFunctionname = true; - break; - } - } - // say goodbye if this is an unknown function - if (!isKnownFunctionname) { + if (!(functionName in version[apiVersion])) { + // no status code?! res.send({code: 3, message: "no such function", data: null}); return; } @@ -194,17 +174,19 @@ exports.handle = async function(apiVersion, functionName, fields, req, res) return; } - try { - // sanitize any padIDs before continuing - if (fields["padID"]) { - fields["padID"] = await padManager.sanitizePadId(fields["padID"]); - } else if (fields["padName"]) { - fields["padName"] = await padManager.sanitizePadId(fields["padName"]); - } - await callAPI(apiVersion, functionName, fields, req, res); - } catch (e) { - ERR(e); + // sanitize any padIDs before continuing + if (fields["padID"]) { + fields["padID"] = await padManager.sanitizePadId(fields["padID"]); } + // there was an 'else' here before - removed it to ensure + // that this sanitize step can't be circumvented by forcing + // the first branch to be taken + if (fields["padName"]) { + fields["padName"] = await padManager.sanitizePadId(fields["padName"]); + } + + // no need to await - callAPI returns a promise + return callAPI(apiVersion, functionName, fields, req, res); } // calls the api function From c499a08030b76fc0a5e5835d9b1c986117e2a903 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 10:47:50 +0000 Subject: [PATCH 43/58] bin/repairPad.js: conversion to promise/async - but see also github issue #3545 --- bin/repairPad.js | 144 +++++++++++++++++++---------------------------- 1 file changed, 58 insertions(+), 86 deletions(-) diff --git a/bin/repairPad.js b/bin/repairPad.js index 28f28cb6e..d495baef5 100644 --- a/bin/repairPad.js +++ b/bin/repairPad.js @@ -1,106 +1,78 @@ /* - This is a repair tool. It extracts all datas of a pad, removes and inserts them again. -*/ + * This is a repair tool. It extracts all datas of a pad, removes and inserts them again. + */ console.warn("WARNING: This script must not be used while etherpad is running!"); -if(process.argv.length != 3) -{ +if (process.argv.length != 3) { console.error("Use: node bin/repairPad.js $PADID"); process.exit(1); } -//get the padID + +// get the padID var padId = process.argv[2]; -var db, padManager, pad, settings; -var neededDBValues = ["pad:"+padId]; +let npm = require("../src/node_modules/npm"); +npm.load({}, async function(er) { + if (er) { + console.error("Could not load NPM: " + er) + process.exit(1); + } -var npm = require("../src/node_modules/npm"); -var async = require("../src/node_modules/async"); + try { + // intialize database + let settings = require('../src/node/utils/Settings'); + let db = require('../src/node/db/DB'); + await db.init(); -async.series([ - // load npm - function(callback) { - npm.load({}, function(er) { - if(er) - { - console.error("Could not load NPM: " + er) - process.exit(1); - } - else - { - callback(); - } - }) - }, - // load modules - function(callback) { - settings = require('../src/node/utils/Settings'); - db = require('../src/node/db/DB'); - callback(); - }, - //initialize the database - function (callback) - { - db.init(callback); - }, - //get the pad - function (callback) - { - padManager = require('../src/node/db/PadManager'); - - padManager.getPad(padId, function(err, _pad) - { - pad = _pad; - callback(err); - }); - }, - function (callback) - { - //add all authors - var authors = pad.getAllAuthors(); - for(var i=0;i "globalAuthor:")); + + // add all revisions + for (let rev = 0; rev <= pad.head; ++rev) { + neededDBValues.push("pad:" + padId + ":revs:" + rev); } - - //add all revisions - var revHead = pad.head; - for(var i=0;i<=revHead;i++) - { - neededDBValues.push("pad:"+padId+":revs:" + i); + + // add all chat values + for (let chat = 0; chat <= pad.chatHead; ++chat) { + neededDBValues.push("pad:" + padId + ":chat:" + chat); } - - //get all chat values - var chatHead = pad.chatHead; - for(var i=0;i<=chatHead;i++) - { - neededDBValues.push("pad:"+padId+":chat:" + i); - } - callback(); - }, - function (callback) { - db = db.db; + + // + // NB: this script doesn't actually does what's documented + // since the `value` fields in the following `.forEach` + // block are just the array index numbers + // + // the script therefore craps out now before it can do + // any damage. + // + // See gitlab issue #3545 + // + console.info("aborting [gitlab #3545]"); + process.exit(1); + + // now fetch and reinsert every key neededDBValues.forEach(function(key, value) { - console.debug("Key: "+key+", value: "+value); + console.log("Key: " + key+ ", value: " + value); db.remove(key); db.set(key, value); }); - callback(); - } -], function (err) -{ - if(err) throw err; - else - { + console.info("finished"); - process.exit(); + process.exit(0); + + } catch (er) { + if (er.name === "apierror") { + console.error(er); + } else { + console.trace(er); + } } }); - -//get the pad object -//get all revisions of this pad -//get all authors related to this pad -//get the readonly link related to this pad -//get the chat entries related to this pad -//remove all keys from database and insert them again From d543d5ae6af0cbad75e942e1c7e3a9f892d46e5a Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 10:43:01 +0000 Subject: [PATCH 44/58] PadMessageHandler.js: convert handleUserChanges() to Promises - the call site still expects a nodeback function, so also introduced the `nodeify` module to allow that function to work as expected. --- src/node/handler/PadMessageHandler.js | 253 +++++++++++--------------- src/package.json | 2 +- 2 files changed, 109 insertions(+), 146 deletions(-) diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 3004ae49c..1e9496cff 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -39,6 +39,7 @@ var channels = require("channels"); var stats = require('../stats'); var remoteAddress = require("../utils/RemoteAddress").remoteAddress; const thenify = require("thenify").withCallback; +const nodeify = require("nodeify"); /** * A associative array that saves informations about a session @@ -61,7 +62,11 @@ stats.gauge('totalUsers', function() { /** * A changeset queue per pad that is processed by handleUserChanges() */ -var padChannels = new channels.channels(thenify(handleUserChanges)); +var padChannels = new channels.channels(handleUserChangesCB); + +function handleUserChangesCB(data, callback) { + return nodeify(handleUserChanges(data), callback); +} /** * Saves the Socket class we need to send and receive data from the client @@ -591,7 +596,7 @@ function handleUserInfoUpdate(client, message) * @param client the client that send this message * @param message the message from the client */ -function handleUserChanges(data, cb) +async function handleUserChanges(data) { var client = data.client , message = data.message @@ -602,17 +607,17 @@ function handleUserChanges(data, cb) // Make sure all required fields are present if (message.data.baseRev == null) { messageLogger.warn("Dropped message, USER_CHANGES Message has no baseRev!"); - return cb(); + return; } if (message.data.apool == null) { messageLogger.warn("Dropped message, USER_CHANGES Message has no apool!"); - return cb(); + return; } if (message.data.changeset == null) { messageLogger.warn("Dropped message, USER_CHANGES Message has no changeset!"); - return cb(); + return; } // TODO: this might happen with other messages too => find one place to copy the session @@ -620,7 +625,7 @@ function handleUserChanges(data, cb) // if the session was valid when the message arrived in the first place if (!sessioninfos[client.id]) { messageLogger.warn("Dropped message, disconnect happened in the mean time"); - return cb(); + return; } // get all Vars we need @@ -632,172 +637,130 @@ function handleUserChanges(data, cb) // finish processing the changeset, so keep a reference to the session. var thisSession = sessioninfos[client.id]; - var r, apool, pad; - // Measure time to process edit var stopWatch = stats.timer('edits').start(); - async.series([ - // get the pad - function(callback) { - padManager.getPad(thisSession.padId, function(err, value) { - if (ERR(err, callback)) return; + // get the pad + let pad = await padManager.getPad(thisSession.padId); - pad = value; - callback(); + // create the changeset + try { + try { + // Verify that the changeset has valid syntax and is in canonical form + Changeset.checkRep(changeset); + + // Verify that the attribute indexes used in the changeset are all + // defined in the accompanying attribute pool. + Changeset.eachAttribNumber(changeset, function(n) { + if (!wireApool.getAttrib(n)) { + throw new Error("Attribute pool is missing attribute " + n + " for changeset " + changeset); + } }); - }, - // create the changeset - function(callback) { - // ex. _checkChangesetAndPool + // Validate all added 'author' attribs to be the same value as the current user + var iterator = Changeset.opIterator(Changeset.unpack(changeset).ops) + , op; - try { - // Verify that the changeset has valid syntax and is in canonical form - Changeset.checkRep(changeset); + while (iterator.hasNext()) { + op = iterator.next() - // Verify that the attribute indexes used in the changeset are all - // defined in the accompanying attribute pool. - Changeset.eachAttribNumber(changeset, function(n) { - if (!wireApool.getAttrib(n)) { - throw new Error("Attribute pool is missing attribute " + n + " for changeset " + changeset); + // + can add text with attribs + // = can change or add attribs + // - can have attribs, but they are discarded and don't show up in the attribs - but do show up in the pool + + op.attribs.split('*').forEach(function(attr) { + if (!attr) return; + + attr = wireApool.getAttrib(attr); + if (!attr) return; + + // the empty author is used in the clearAuthorship functionality so this should be the only exception + if ('author' == attr[0] && (attr[1] != thisSession.author && attr[1] != '')) { + throw new Error("Trying to submit changes as another author in changeset " + changeset); } }); - - // Validate all added 'author' attribs to be the same value as the current user - var iterator = Changeset.opIterator(Changeset.unpack(changeset).ops) - , op - while(iterator.hasNext()) { - op = iterator.next() - - // + can add text with attribs - // = can change or add attribs - // - can have attribs, but they are discarded and don't show up in the attribs - but do show up in the pool - - op.attribs.split('*').forEach(function(attr) { - if (!attr) return; - - attr = wireApool.getAttrib(attr) - if (!attr) return; - - // the empty author is used in the clearAuthorship functionality so this should be the only exception - if ('author' == attr[0] && (attr[1] != thisSession.author && attr[1] != '')) { - throw new Error("Trying to submit changes as another author in changeset " + changeset); - } - }); - } - - // ex. adoptChangesetAttribs - - // Afaik, it copies the new attributes from the changeset, to the global Attribute Pool - changeset = Changeset.moveOpsToNewPool(changeset, wireApool, pad.pool); - } catch(e) { - // There is an error in this changeset, so just refuse it - client.json.send({disconnect:"badChangeset"}); - stats.meter('failedChangesets').mark(); - return callback(new Error("Can't apply USER_CHANGES, because "+e.message)); } - // ex. applyUserChanges - apool = pad.pool; - r = baseRev; + // ex. adoptChangesetAttribs - // The client's changeset might not be based on the latest revision, - // since other clients are sending changes at the same time. - // Update the changeset so that it can be applied to the latest revision. - // https://github.com/caolan/async#whilst - async.whilst( - function() { return r < pad.getHeadRevisionNumber(); }, - function(callback) - { - r++; + // Afaik, it copies the new attributes from the changeset, to the global Attribute Pool + changeset = Changeset.moveOpsToNewPool(changeset, wireApool, pad.pool); - pad.getRevisionChangeset(r, function(err, c) { - if (ERR(err, callback)) return; + } catch(e) { + // There is an error in this changeset, so just refuse it + client.json.send({ disconnect: "badChangeset" }); + stats.meter('failedChangesets').mark(); + throw new Error("Can't apply USER_CHANGES, because " + e.message); + } - // At this point, both "c" (from the pad) and "changeset" (from the - // client) are relative to revision r - 1. The follow function - // rebases "changeset" so that it is relative to revision r - // and can be applied after "c". - try { - // a changeset can be based on an old revision with the same changes in it - // prevent eplite from accepting it TODO: better send the client a NEW_CHANGES - // of that revision - if (baseRev + 1 == r && c == changeset) { - client.json.send({disconnect:"badChangeset"}); - stats.meter('failedChangesets').mark(); + // ex. applyUserChanges + let apool = pad.pool; + let r = baseRev; - return callback(new Error("Won't apply USER_CHANGES, because it contains an already accepted changeset")); - } + // The client's changeset might not be based on the latest revision, + // since other clients are sending changes at the same time. + // Update the changeset so that it can be applied to the latest revision. + while (r < pad.getHeadRevisionNumber()) { + r++; - changeset = Changeset.follow(c, changeset, false, apool); - } catch(e) { - client.json.send({disconnect:"badChangeset"}); - stats.meter('failedChangesets').mark(); + let c = await pad.getRevisionChangeset(r); - return callback(new Error("Can't apply USER_CHANGES, because "+e.message)); - } - - if ((r - baseRev) % 200 == 0) { - // don't let the stack get too deep - async.nextTick(callback); - } else { - callback(null); - } - }); - }, - - // use the callback of the series function - callback - ); - }, - - // do correction changesets, and send it to all users - function(callback) { - var prevText = pad.text(); - - if (Changeset.oldLen(changeset) != prevText.length) { - client.json.send({disconnect:"badChangeset"}); - stats.meter('failedChangesets').mark(); - - return callback(new Error("Can't apply USER_CHANGES "+changeset+" with oldLen " + Changeset.oldLen(changeset) + " to document of length " + prevText.length)); - } + // At this point, both "c" (from the pad) and "changeset" (from the + // client) are relative to revision r - 1. The follow function + // rebases "changeset" so that it is relative to revision r + // and can be applied after "c". try { - pad.appendRevision(changeset, thisSession.author); + // a changeset can be based on an old revision with the same changes in it + // prevent eplite from accepting it TODO: better send the client a NEW_CHANGES + // of that revision + if (baseRev + 1 == r && c == changeset) { + client.json.send({disconnect:"badChangeset"}); + stats.meter('failedChangesets').mark(); + throw new Error("Won't apply USER_CHANGES, because it contains an already accepted changeset"); + } + + changeset = Changeset.follow(c, changeset, false, apool); } catch(e) { client.json.send({disconnect:"badChangeset"}); stats.meter('failedChangesets').mark(); - - return callback(e) + throw new Error("Can't apply USER_CHANGES, because " + e.message); } - - var correctionChangeset = _correctMarkersInPad(pad.atext, pad.pool); - if (correctionChangeset) { - pad.appendRevision(correctionChangeset); - } - - // Make sure the pad always ends with an empty line. - if (pad.text().lastIndexOf("\n") != pad.text().length-1) { - var nlChangeset = Changeset.makeSplice(pad.text(), pad.text().length-1, - 0, "\n"); - pad.appendRevision(nlChangeset); - } - - exports.updatePadClients(pad, function(er) { - ERR(er) - }); - callback(); } - ], - function(err) { - stopWatch.end() - cb(); - if(err) { - console.warn(err.stack || err); + let prevText = pad.text(); + + if (Changeset.oldLen(changeset) != prevText.length) { + client.json.send({disconnect:"badChangeset"}); + stats.meter('failedChangesets').mark(); + throw new Error("Can't apply USER_CHANGES "+changeset+" with oldLen " + Changeset.oldLen(changeset) + " to document of length " + prevText.length); } - }); + + try { + pad.appendRevision(changeset, thisSession.author); + } catch(e) { + client.json.send({ disconnect: "badChangeset" }); + stats.meter('failedChangesets').mark(); + throw e; + } + + let correctionChangeset = _correctMarkersInPad(pad.atext, pad.pool); + if (correctionChangeset) { + pad.appendRevision(correctionChangeset); + } + + // Make sure the pad always ends with an empty line. + if (pad.text().lastIndexOf("\n") != pad.text().length-1) { + var nlChangeset = Changeset.makeSplice(pad.text(), pad.text().length - 1, 0, "\n"); + pad.appendRevision(nlChangeset); + } + + await exports.updatePadClients(pad); + } catch (err) { + console.warn(err.stack || err); + } + + stopWatch.end(); } exports.updatePadClients = thenify(function(pad, callback) diff --git a/src/package.json b/src/package.json index fa63b0710..11259e76d 100644 --- a/src/package.json +++ b/src/package.json @@ -48,6 +48,7 @@ "languages4translatewiki": "0.1.3", "log4js": "0.6.35", "measured-core": "1.11.2", + "nodeify": "^1.0.1", "npm": "6.4.1", "object.values": "^1.0.4", "request": "2.88.0", @@ -87,4 +88,3 @@ "version": "1.7.5", "license": "Apache-2.0" } - From 9246a1de26cdf94238dd9296be060b64e3d9f346 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 13:55:49 +0000 Subject: [PATCH 45/58] PadMessageHandler.js: further conversion --- src/node/handler/PadMessageHandler.js | 397 +++++++++----------------- 1 file changed, 135 insertions(+), 262 deletions(-) diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 1e9496cff..25f9879bc 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -1274,7 +1274,7 @@ async function handleClientReady(client, message) /** * Handles a request for a rough changeset, the timeslider client needs it */ -function handleChangesetRequest(client, message) +async function handleChangesetRequest(client, message) { // check if all ok if (message.data == null) { @@ -1308,309 +1308,182 @@ function handleChangesetRequest(client, message) return; } - var granularity = message.data.granularity; - var start = message.data.start; - var end = start + (100 * granularity); - var padIds; + let granularity = message.data.granularity; + let start = message.data.start; + let end = start + (100 * granularity); - async.series([ - function(callback) { - readOnlyManager.getIds(message.padId, function(err, value) { - if (ERR(err, callback)) return; + let padIds = await readOnlyManager.getIds(message.padId); - padIds = value; - callback(); - }); - }, - - function(callback) { - // build the requested rough changesets and send them back - getChangesetInfo(padIds.padId, start, end, granularity, function(err, changesetInfo) { - if (err) return console.error('Error while handling a changeset request for ' + padIds.padId, err, message.data); - - var data = changesetInfo; - data.requestID = message.data.requestID; - - client.json.send({ type: "CHANGESET_REQ", data: data }); - }); - } - ]); + // build the requested rough changesets and send them back + try { + let data = await getChangesetInfo(padIds.padId, start, end, granularity); + data.requestID = message.data.requestID; + client.json.send({ type: "CHANGESET_REQ", data }); + } catch (err) { + console.error('Error while handling a changeset request for ' + padIds.padId, err, message.data); + } } /** * Tries to rebuild the getChangestInfo function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L144 */ -let getChangesetInfo = thenify(function getChangesetInfo(padId, startNum, endNum, granularity, callback) +async function getChangesetInfo(padId, startNum, endNum, granularity) { - var forwardsChangesets = []; - var backwardsChangesets = []; - var timeDeltas = []; - var apool = new AttributePool(); - var pad; - var composedChangesets = {}; - var revisionDate = []; - var lines; - var head_revision = 0; + let pad = await padManager.getPad(padId); + let head_revision = pad.getHeadRevisionNumber(); - async.series([ - // get the pad from the database - function(callback) { - padManager.getPad(padId, function(err, _pad) { - if (ERR(err, callback)) return; + // calculate the last full endnum + if (endNum > head_revision + 1) { + endNum = head_revision + 1; + } + endNum = Math.floor(endNum / granularity) * granularity; - pad = _pad; - head_revision = pad.getHeadRevisionNumber(); - callback(); - }); - }, + let compositesChangesetNeeded = []; + let revTimesNeeded = []; - function(callback) { - // calculate the last full endnum - var lastRev = pad.getHeadRevisionNumber(); - if (endNum > lastRev + 1) { - endNum = lastRev + 1; - } + // figure out which composite Changeset and revTimes we need, to load them in bulk + for (let start = startNum; start < endNum; start += granularity) { + let end = start + granularity; - endNum = Math.floor(endNum / granularity) * granularity; + // add the composite Changeset we needed + compositesChangesetNeeded.push({ start, end }); - var compositesChangesetNeeded = []; - var revTimesNeeded = []; + // add the t1 time we need + revTimesNeeded.push(start == 0 ? 0 : start - 1); - // figure out which composite Changeset and revTimes we need, to load them in bulk - var compositeStart = startNum; - while (compositeStart < endNum) { - var compositeEnd = compositeStart + granularity; + // add the t2 time we need + revTimesNeeded.push(end - 1); + } - // add the composite Changeset we needed - compositesChangesetNeeded.push({ start: compositeStart, end: compositeEnd }); + // get all needed db values parallel - no await here since + // it would make all the lookups run in series - // add the t1 time we need - revTimesNeeded.push(compositeStart == 0 ? 0 : compositeStart - 1); + // get all needed composite Changesets + let composedChangesets = {}; + let p1 = Promise.all(compositesChangesetNeeded.map(item => { + return composePadChangesets(padId, item.start, item.end).then(changeset => { + composedChangesets[item.start + "/" + item.end] = changeset; + }); + })); - // add the t2 time we need - revTimesNeeded.push(compositeEnd - 1); + // get all needed revision Dates + let revisionDate = []; + let p2 = Promise.all(revTimesNeeded.map(revNum => { + return pad.getRevisionDate(revNum).then(revDate => { + revisionDate[revNum] = Math.floor(revDate / 1000); + }); + })); - compositeStart += granularity; - } - - // get all needed db values parallel - async.parallel([ - function(callback) { - // get all needed composite Changesets - async.forEach(compositesChangesetNeeded, function(item, callback) { - composePadChangesets(padId, item.start, item.end, function(err, changeset) { - if (ERR(err, callback)) return; - - composedChangesets[item.start + "/" + item.end] = changeset; - callback(); - }); - }, callback); - }, - - function(callback) { - // get all needed revision Dates - async.forEach(revTimesNeeded, function(revNum, callback) { - pad.getRevisionDate(revNum, function(err, revDate) { - if (ERR(err, callback)) return; - - revisionDate[revNum] = Math.floor(revDate/1000); - callback(); - }); - }, callback); - }, - - // get the lines - function(callback) { - getPadLines(padId, startNum-1, function(err, _lines) { - if (ERR(err, callback)) return; - - lines = _lines; - callback(); - }); - } - ], callback); - }, - - // don't know what happens here exactly :/ - function(callback) { - var compositeStart = startNum; - - while (compositeStart < endNum) { - var compositeEnd = compositeStart + granularity; - if (compositeEnd > endNum || compositeEnd > head_revision+1) { - break; - } - - var forwards = composedChangesets[compositeStart + "/" + compositeEnd]; - var backwards = Changeset.inverse(forwards, lines.textlines, lines.alines, pad.apool()); - - Changeset.mutateAttributionLines(forwards, lines.alines, pad.apool()); - Changeset.mutateTextLines(forwards, lines.textlines); - - var forwards2 = Changeset.moveOpsToNewPool(forwards, pad.apool(), apool); - var backwards2 = Changeset.moveOpsToNewPool(backwards, pad.apool(), apool); - - var t1, t2; - if (compositeStart == 0) { - t1 = revisionDate[0]; - } else { - t1 = revisionDate[compositeStart - 1]; - } - - t2 = revisionDate[compositeEnd - 1]; - - timeDeltas.push(t2 - t1); - forwardsChangesets.push(forwards2); - backwardsChangesets.push(backwards2); - - compositeStart += granularity; - } - - callback(); - } - ], - function(err) { - if (ERR(err, callback)) return; - - callback(null, {forwardsChangesets: forwardsChangesets, - backwardsChangesets: backwardsChangesets, - apool: apool.toJsonable(), - actualEndNum: endNum, - timeDeltas: timeDeltas, - start: startNum, - granularity: granularity }); + // get the lines + let lines; + let p3 = getPadLines(padId, startNum - 1).then(_lines => { + lines = _lines; }); -}); + + // wait for all of the above to complete + await Promise.all([p1, p2, p3]); + + // doesn't know what happens here exactly :/ + let timeDeltas = []; + let forwardsChangesets = []; + let backwardsChangesets = []; + let apool = new AttributePool(); + + for (let compositeStart = startNum; compositeStart < endNum; compositeStart += granularity) { + let compositeEnd = compositeStart + granularity; + if (compositeEnd > endNum || compositeEnd > head_revision + 1) { + break; + } + + let forwards = composedChangesets[compositeStart + "/" + compositeEnd]; + let backwards = Changeset.inverse(forwards, lines.textlines, lines.alines, pad.apool()); + + Changeset.mutateAttributionLines(forwards, lines.alines, pad.apool()); + Changeset.mutateTextLines(forwards, lines.textlines); + + let forwards2 = Changeset.moveOpsToNewPool(forwards, pad.apool(), apool); + let backwards2 = Changeset.moveOpsToNewPool(backwards, pad.apool(), apool); + + let t1 = (compositeStart == 0) ? revisionDate[0] : revisionDate[compositeStart - 1]; + let t2 = revisionDate[compositeEnd - 1]; + + timeDeltas.push(t2 - t1); + forwardsChangesets.push(forwards2); + backwardsChangesets.push(backwards2); + } + + return { forwardsChangesets, backwardsChangesets, + apool: apool.toJsonable(), actualEndNum: endNum, + timeDeltas, start: startNum, granularity }; +} /** * Tries to rebuild the getPadLines function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L263 */ -let getPadLines = thenify(function getPadLines(padId, revNum, callback) +async function getPadLines(padId, revNum) { - var atext; - var result = {}; - var pad; + let pad = padManager.getPad(padId); - async.series([ - // get the pad from the database - function(callback) { - padManager.getPad(padId, function(err, _pad) { - if (ERR(err, callback)) return; + // get the atext + let atext; - pad = _pad; - callback(); - }); - }, + if (revNum >= 0) { + atext = await pad.getInternalRevisionAText(revNum); + } else { + atext = Changeset.makeAText("\n"); + } - // get the atext - function(callback) { - if (revNum >= 0) { - pad.getInternalRevisionAText(revNum, function(err, _atext) { - if (ERR(err, callback)) return; - - atext = _atext; - callback(); - }); - } else { - atext = Changeset.makeAText("\n"); - callback(null); - } - }, - - function(callback) { - result.textlines = Changeset.splitTextLines(atext.text); - result.alines = Changeset.splitAttributionLines(atext.attribs, atext.text); - callback(null); - } - ], - - function(err) { - if (ERR(err, callback)) return; - - callback(null, result); - }); -}); + return { + textlines: Changeset.splitTextLines(atext.text), + alines: Changeset.splitAttributionLines(atext.attribs, atext.text) + }; +} /** * Tries to rebuild the composePadChangeset function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L241 */ -let composePadChangesets = thenify(function(padId, startNum, endNum, callback) +async function composePadChangesets (padId, startNum, endNum) { - var pad; - var changesets = {}; - var changeset; + let pad = await padManager.getPad(padId); - async.series([ - // get the pad from the database - function(callback) { - padManager.getPad(padId, function(err, _pad) { - if (ERR(err, callback)) return; + // fetch all changesets we need + let headNum = pad.getHeadRevisionNumber(); + endNum = Math.min(endNum, headNum + 1); + startNum = Math.max(startNum, 0); - pad = _pad; - callback(); - }); - }, + // create an array for all changesets, we will + // replace the values with the changeset later + let changesetsNeeded = []; + for (let r = startNum ; r < endNum; r++) { + changesetsNeeded.push(r); + } - // fetch all changesets we need - function(callback) { - var changesetsNeeded=[]; + // get all changesets + let changesets = {}; + await Promise.all(changesetsNeeded.map(revNum => { + return pad.getRevisionChangeset(revNum).then(changeset => changesets[revNum] = changeset); + })); - var headNum = pad.getHeadRevisionNumber(); - if (endNum > headNum + 1) { - endNum = headNum + 1; - } + // compose Changesets + try { + let changeset = changesets[startNum]; + let pool = pad.apool(); - if (startNum < 0) { - startNum = 0; - } - - // create an array for all changesets, we will - // replace the values with the changeset later - for (var r = startNum; r < endNum; r++) { - changesetsNeeded.push(r); - } - - // get all changesets - async.forEach(changesetsNeeded, function(revNum,callback) { - pad.getRevisionChangeset(revNum, function(err, value) { - if (ERR(err, callback)) return; - - changesets[revNum] = value; - callback(); - }); - },callback); - }, - - // compose Changesets - function(callback) { - changeset = changesets[startNum]; - var pool = pad.apool(); - - try { - for (var r = startNum + 1; r < endNum; r++) { - var cs = changesets[r]; - changeset = Changeset.compose(changeset, cs, pool); - } - } catch(e) { - // r-1 indicates the rev that was build starting with startNum, applying startNum+1, +2, +3 - console.warn("failed to compose cs in pad:", padId, " startrev:", startNum, " current rev:", r); - return callback(e); - } - - callback(null); + for (let r = startNum + 1; r < endNum; r++) { + let cs = changesets[r]; + changeset = Changeset.compose(changeset, cs, pool); } - ], + return changeset; - // return err and changeset - function(err) { - if (ERR(err, callback)) return; - - callback(null, changeset); - }); -}); + } catch (e) { + // r-1 indicates the rev that was build starting with startNum, applying startNum+1, +2, +3 + console.warn("failed to compose cs in pad:", padId, " startrev:", startNum," current rev:", r); + throw e; + } +} function _getRoomClients(padID) { var roomClients = []; From bb80325d2cf8e653cb8fc7f8b40e0a64c6b3c071 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 15:27:42 +0000 Subject: [PATCH 46/58] PadMessageHandler.js: completed conversion --- src/node/handler/ImportHandler.js | 6 +- src/node/handler/PadMessageHandler.js | 297 ++++++++++---------------- 2 files changed, 115 insertions(+), 188 deletions(-) diff --git a/src/node/handler/ImportHandler.js b/src/node/handler/ImportHandler.js index ec07bf6e5..e92df4434 100644 --- a/src/node/handler/ImportHandler.js +++ b/src/node/handler/ImportHandler.js @@ -289,9 +289,9 @@ exports.doImport = function(req, res, padId) return; } - padMessageHandler.updatePadClients(pad, function(){ - callback(); - }); + // @TODO: not waiting for updatePadClients to finish + padMessageHandler.updatePadClients(pad); + callback(); }); }, diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 25f9879bc..b41af86b9 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -19,8 +19,6 @@ */ -var ERR = require("async-stacktrace"); -var async = require("async"); var padManager = require("../db/PadManager"); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var AttributePool = require("ep_etherpad-lite/static/js/AttributePool"); @@ -38,7 +36,6 @@ var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); var channels = require("channels"); var stats = require('../stats'); var remoteAddress = require("../utils/RemoteAddress").remoteAddress; -const thenify = require("thenify").withCallback; const nodeify = require("nodeify"); /** @@ -62,16 +59,14 @@ stats.gauge('totalUsers', function() { /** * A changeset queue per pad that is processed by handleUserChanges() */ -var padChannels = new channels.channels(handleUserChangesCB); - -function handleUserChangesCB(data, callback) { +var padChannels = new channels.channels(function(data, callback) { return nodeify(handleUserChanges(data), callback); -} +}); /** * Saves the Socket class we need to send and receive data from the client */ -var socketio; +let socketio; /** * This Method is called by server.js to tell the message handler on which socket it should send @@ -115,17 +110,17 @@ exports.kickSessionsFromPad = function(padID) * Handles the disconnection of a user * @param client the client that leaves */ -exports.handleDisconnect = function(client) +exports.handleDisconnect = async function(client) { stats.meter('disconnects').mark(); // save the padname of this session - var session = sessioninfos[client.id]; + let session = sessioninfos[client.id]; // if this connection was already etablished with a handshake, send a disconnect message to the others if (session && session.author) { // Get the IP address from our persistant object - var ip = remoteAddress[client.id]; + let ip = remoteAddress[client.id]; // Anonymize the IP address if IP logging is disabled if (settings.disableIPlogging) { @@ -135,29 +130,27 @@ exports.handleDisconnect = function(client) accessLogger.info('[LEAVE] Pad "' + session.padId + '": Author "' + session.author + '" on client ' + client.id + ' with IP "' + ip + '" left the pad'); // get the author color out of the db - authorManager.getAuthorColorId(session.author, function(err, color) { - ERR(err); + let color = await authorManager.getAuthorColorId(session.author); - // prepare the notification for the other users on the pad, that this user left - var messageToTheOtherUsers = { - "type": "COLLABROOM", - "data": { - type: "USER_LEAVE", - userInfo: { - "ip": "127.0.0.1", - "colorId": color, - "userAgent": "Anonymous", - "userId": session.author - } + // prepare the notification for the other users on the pad, that this user left + let messageToTheOtherUsers = { + "type": "COLLABROOM", + "data": { + type: "USER_LEAVE", + userInfo: { + "ip": "127.0.0.1", + "colorId": color, + "userAgent": "Anonymous", + "userId": session.author } - }; + } + }; - // Go through all user that are still on the pad, and send them the USER_LEAVE message - client.broadcast.to(session.padId).json.send(messageToTheOtherUsers); + // Go through all user that are still on the pad, and send them the USER_LEAVE message + client.broadcast.to(session.padId).json.send(messageToTheOtherUsers); - // Allow plugins to hook into users leaving the pad - hooks.callAll("userLeave", session); - }); + // Allow plugins to hook into users leaving the pad + hooks.callAll("userLeave", session); } // Delete the sessioninfos entrys of this session @@ -179,7 +172,7 @@ exports.handleMessage = async function(client, message) return; } - var thisSession = sessioninfos[client.id]; + let thisSession = sessioninfos[client.id]; if (!thisSession) { messageLogger.warn("Dropped message from an unknown connection.") @@ -248,7 +241,7 @@ exports.handleMessage = async function(client, message) /* * In a previous version of this code, an "if (message)" wrapped the - * following async.series(). + * following series of async calls [now replaced with await calls] * This ugly "!Boolean(message)" is a lame way to exactly negate the truthy * condition and replace it with an early return, while being sure to leave * the original behaviour unchanged. @@ -310,15 +303,13 @@ exports.handleMessage = async function(client, message) * @param client the client that send this message * @param message the message from the client */ -function handleSaveRevisionMessage(client, message){ +async function handleSaveRevisionMessage(client, message) +{ var padId = sessioninfos[client.id].padId; var userId = sessioninfos[client.id].author; - padManager.getPad(padId, function(err, pad) { - if (ERR(err)) return; - - pad.addSavedRevision(pad.head, userId); - }); + let pad = await padManager.getPad(padId); + pad.addSavedRevision(pad.head, userId); } /** @@ -328,7 +319,7 @@ function handleSaveRevisionMessage(client, message){ * @param msg {Object} the message we're sending * @param sessionID {string} the socketIO session to which we're sending this message */ -exports.handleCustomObjectMessage = async function(msg, sessionID) { +exports.handleCustomObjectMessage = function(msg, sessionID) { if (msg.data.type === "CUSTOM") { if (sessionID){ // a sessionID is targeted: directly to this sessionID @@ -346,9 +337,9 @@ exports.handleCustomObjectMessage = async function(msg, sessionID) { * @param padID {Pad} the pad to which we're sending this message * @param msgString {String} the message we're sending */ -exports.handleCustomMessage = thenify(function(padID, msgString, cb) { - var time = Date.now(); - var msg = { +exports.handleCustomMessage = function(padID, msgString) { + let time = Date.now(); + let msg = { type: 'COLLABROOM', data: { type: msgString, @@ -356,9 +347,7 @@ exports.handleCustomMessage = thenify(function(padID, msgString, cb) { } }; socketio.sockets.in(padID).json.send(msg); - - cb(null, {}); -}); +} /** * Handles a Chat Message @@ -382,55 +371,24 @@ function handleChatMessage(client, message) * @param text the text of the chat message * @param padId the padId to send the chat message to */ -exports.sendChatMessageToPadClients = function(time, userId, text, padId) { - var pad; - var userName; +exports.sendChatMessageToPadClients = async function(time, userId, text, padId) +{ + // get the pad + let pad = await padManager.getPad(padId); - async.series([ - // get the pad - function(callback) { - padManager.getPad(padId, function(err, _pad) { - if (ERR(err, callback)) return; + // get the author + let userName = await authorManager.getAuthorName(userId); - pad = _pad; - callback(); - }); - }, + // save the chat message + pad.appendChatMessage(text, userId, time); - function(callback) { - authorManager.getAuthorName(userId, function(err, _userName) { - if (ERR(err, callback)) return; + let msg = { + type: "COLLABROOM", + data: { type: "CHAT_MESSAGE", userId, userName, time, text } + }; - userName = _userName; - callback(); - }); - }, - - // save the chat message and broadcast it - function(callback) { - // save the chat message - pad.appendChatMessage(text, userId, time); - - var msg = { - type: "COLLABROOM", - data: { - type: "CHAT_MESSAGE", - userId: userId, - userName: userName, - time: time, - text: text - } - }; - - // broadcast the chat message to everyone on the pad - socketio.sockets.in(padId).json.send(msg); - - callback(); - } - ], - function(err) { - ERR(err); - }); + // broadcast the chat message to everyone on the pad + socketio.sockets.in(padId).json.send(msg); } /** @@ -438,7 +396,7 @@ exports.sendChatMessageToPadClients = function(time, userId, text, padId) { * @param client the client that send this message * @param message the message from the client */ -function handleGetChatMessages(client, message) +async function handleGetChatMessages(client, message) { if (message.data.start == null) { messageLogger.warn("Dropped message, GetChatMessages Message has no start!"); @@ -450,45 +408,29 @@ function handleGetChatMessages(client, message) return; } - var start = message.data.start; - var end = message.data.end; - var count = end - start; + let start = message.data.start; + let end = message.data.end; + let count = end - start; if (count < 0 || count > 100) { messageLogger.warn("Dropped message, GetChatMessages Message, client requested invalid amount of messages!"); return; } - var padId = sessioninfos[client.id].padId; - var pad; + let padId = sessioninfos[client.id].padId; + let pad = await padManager.getPad(padId); - async.series([ - // get the pad - function(callback) { - padManager.getPad(padId, function(err, _pad) { - if (ERR(err, callback)) return; + let chatMessages = await pad.getChatMessages(start, end); + let infoMsg = { + type: "COLLABROOM", + data: { + type: "CHAT_MESSAGES", + messages: chatMessages + } + }; - pad = _pad; - callback(); - }); - }, - - function(callback) { - pad.getChatMessages(start, end, function(err, chatMessages) { - if (ERR(err, callback)) return; - - var infoMsg = { - type: "COLLABROOM", - data: { - type: "CHAT_MESSAGES", - messages: chatMessages - } - }; - - // send the messages back to the client - client.json.send(infoMsg); - }); - }]); + // send the messages back to the client + client.json.send(infoMsg); } /** @@ -763,13 +705,13 @@ async function handleUserChanges(data) stopWatch.end(); } -exports.updatePadClients = thenify(function(pad, callback) +exports.updatePadClients = async function(pad) { // skip this if no-one is on this pad - var roomClients = _getRoomClients(pad.id); + let roomClients = _getRoomClients(pad.id); if (roomClients.length == 0) { - return callback(); + return; } // since all clients usually get the same set of changesets, store them in local cache @@ -778,69 +720,54 @@ exports.updatePadClients = thenify(function(pad, callback) // BEFORE first result will be landed to our cache object. The solution is to replace parallel processing // via async.forEach with sequential for() loop. There is no real benefits of running this in parallel, // but benefit of reusing cached revision object is HUGE - var revCache = {}; + let revCache = {}; // go through all sessions on this pad - async.forEach(roomClients, function(client, callback){ - var sid = client.id; - // https://github.com/caolan/async#whilst + for (let client of roomClients) { + let sid = client.id; + // send them all new changesets - async.whilst( - function() { return sessioninfos[sid] && sessioninfos[sid].rev < pad.getHeadRevisionNumber()}, - function(callback) - { - var r = sessioninfos[sid].rev + 1; + while (sessioninfos[sid] && sessioninfos[sid].rev < pad.getHeadRevisionNumber()) { + let r = sessioninfos[sid].rev + 1; + let revision = revCache[r]; + if (!revision) { + revision = await pad.getRevision(r); + revCache[r] = revision; + } - async.waterfall([ - function(callback) { - if(revCache[r]) { - callback(null, revCache[r]); - } else { - pad.getRevision(r, callback); - } - }, + let author = revision.meta.author, + revChangeset = revision.changeset, + currentTime = revision.meta.timestamp; - function(revision, callback) { - revCache[r] = revision; + // next if session has not been deleted + if (sessioninfos[sid] == null) { + continue; + } - var author = revision.meta.author, - revChangeset = revision.changeset, - currentTime = revision.meta.timestamp; + if (author == sessioninfos[sid].author) { + client.json.send({ "type": "COLLABROOM", "data":{ type: "ACCEPT_COMMIT", newRev: r }}); + } else { + let forWire = Changeset.prepareForWire(revChangeset, pad.pool); + let wireMsg = {"type": "COLLABROOM", + "data": { type:"NEW_CHANGES", + newRev:r, + changeset: forWire.translated, + apool: forWire.pool, + author: author, + currentTime: currentTime, + timeDelta: currentTime - sessioninfos[sid].time + }}; - // next if session has not been deleted - if (sessioninfos[sid] == null) { - return callback(null); - } + client.json.send(wireMsg); + } - if (author == sessioninfos[sid].author) { - client.json.send({"type":"COLLABROOM","data":{type:"ACCEPT_COMMIT", newRev:r}}); - } else { - var forWire = Changeset.prepareForWire(revChangeset, pad.pool); - var wireMsg = {"type":"COLLABROOM", - "data":{type:"NEW_CHANGES", - newRev:r, - changeset: forWire.translated, - apool: forWire.pool, - author: author, - currentTime: currentTime, - timeDelta: currentTime - sessioninfos[sid].time - }}; - - client.json.send(wireMsg); - } - - if (sessioninfos[sid]) { - sessioninfos[sid].time = currentTime; - sessioninfos[sid].rev = r; - } - callback(null); - } - ], callback); - }, - callback - ); - },callback); -}); + if (sessioninfos[sid]) { + sessioninfos[sid].time = currentTime; + sessioninfos[sid].rev = r; + } + } + } +} /** * Copied from the Etherpad Source Code. Don't know what this method does excatly... @@ -893,12 +820,12 @@ function _correctMarkersInPad(atext, apool) { function handleSwitchToPad(client, message) { // clear the session and leave the room - var currentSession = sessioninfos[client.id]; - var padId = currentSession.padId; - var roomClients = _getRoomClients(padId); + let currentSession = sessioninfos[client.id]; + let padId = currentSession.padId; + let roomClients = _getRoomClients(padId); - async.forEach(roomClients, function(client, callback) { - var sinfo = sessioninfos[client.id]; + roomClients.forEach(client => { + let sinfo = sessioninfos[client.id]; if (sinfo && sinfo.author == currentSession.author) { // fix user's counter, works on page refresh or if user closes browser window and then rejoins sessioninfos[client.id] = {}; From 5192a0c498bd99a14bdf9729089b789ef64bfc86 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Wed, 30 Jan 2019 16:19:51 +0000 Subject: [PATCH 47/58] db/ReadOnlyManager.js: completed conversion Requires temporary hack within `Pad.remove()` to allow for the lack of callback on the rewritten version. --- src/node/db/Pad.js | 8 +++- src/node/db/ReadOnlyManager.js | 79 ++++++++++------------------------ 2 files changed, 30 insertions(+), 57 deletions(-) diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index 4abf2cab2..db35bd367 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -19,6 +19,7 @@ var crypto = require("crypto"); var randomString = require("../utils/randomstring"); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); const thenify = require("thenify").withCallback; +const nodeify = require("nodeify"); // serialization/deserialization attributes var attributeBlackList = ["id"]; @@ -621,7 +622,12 @@ Pad.prototype.remove = thenify(function remove(callback) { // remove the readonly entries function(callback) { - readOnlyManager.getReadOnlyId(padID, function(err, readonlyID) { + // @TODO - temporary until surrounding code is Promisified + function getReadOnlyId(padID, callback) { + return nodeify(readOnlyManager.getReadOnlyId(padID), callback); + } + + getReadOnlyId(padID, function(err, readonlyID) { if (ERR(err, callback)) return; db.remove("pad2readonly:" + padID); diff --git a/src/node/db/ReadOnlyManager.js b/src/node/db/ReadOnlyManager.js index 14c6f75c7..96a52d479 100644 --- a/src/node/db/ReadOnlyManager.js +++ b/src/node/db/ReadOnlyManager.js @@ -19,80 +19,47 @@ */ -var ERR = require("async-stacktrace"); -var db = require("./DB").db; -var async = require("async"); +var db = require("./DB"); var randomString = require("../utils/randomstring"); -const thenify = require("thenify").withCallback; /** * returns a read only id for a pad * @param {String} padId the id of the pad */ -exports.getReadOnlyId = thenify(function (padId, callback) +exports.getReadOnlyId = async function (padId) { - var readOnlyId; + // check if there is a pad2readonly entry + let readOnlyId = await db.get("pad2readonly:" + padId); - async.waterfall([ - // check if there is a pad2readonly entry - function(callback) { - db.get("pad2readonly:" + padId, callback); - }, + // there is no readOnly Entry in the database, let's create one + if (readOnlyId == null) { + readOnlyId = "r." + randomString(16); + db.set("pad2readonly:" + padId, readOnlyId); + db.set("readonly2pad:" + readOnlyId, padId); + } - function(dbReadOnlyId, callback) { - if (dbReadOnlyId == null) { - // there is no readOnly Entry in the database, let's create one - readOnlyId = "r." + randomString(16); - - db.set("pad2readonly:" + padId, readOnlyId); - db.set("readonly2pad:" + readOnlyId, padId); - } else { - // there is a readOnly Entry in the database, let's take this one - readOnlyId = dbReadOnlyId; - } - - callback(); - } - ], - function(err) { - if (ERR(err, callback)) return; - - // return the results - callback(null, readOnlyId); - }) -}); + return readOnlyId; +} /** * returns the padId for a read only id * @param {String} readOnlyId read only id */ -exports.getPadId = thenify(function(readOnlyId, callback) +exports.getPadId = function(readOnlyId) { - db.get("readonly2pad:" + readOnlyId, callback); -}); + return db.get("readonly2pad:" + readOnlyId); +} /** * returns the padId and readonlyPadId in an object for any id * @param {String} padIdOrReadonlyPadId read only id or real pad id */ -exports.getIds = thenify(function(id, callback) { - if (id.indexOf("r.") == 0) { - exports.getPadId(id, function (err, value) { - if (ERR(err, callback)) return; +exports.getIds = async function(id) { + let readonly = (id.indexOf("r.") === 0); - callback(null, { - readOnlyPadId: id, - padId: value, // Might be null, if this is an unknown read-only id - readonly: true - }); - }); - } else { - exports.getReadOnlyId(id, function (err, value) { - callback(null, { - readOnlyPadId: value, - padId: id, - readonly: false - }); - }); - } -}); + // Might be null, if this is an unknown read-only id + let readOnlyPadId = readonly ? id : await exports.getReadOnlyId(id); + let padId = readonly ? await exports.getPadId(id) : id; + + return { readOnlyPadId, padId, readonly }; +} From 62345ac8f7c733454fc051c0905cdf2d8dc0cb82 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 08:55:36 +0000 Subject: [PATCH 48/58] import/export: conversion to Promises/async NB1: needs additional review and testing - no abiword available on my test bed NB2: in ImportHandler.js, directly delete the file, and handle the eventual error later: checking before for existence is prone to race conditions, and does not handle any errors anyway. --- src/node/db/API.js | 2 +- src/node/handler/ExportHandler.js | 196 ++++++------- src/node/handler/ImportHandler.js | 449 +++++++++++++----------------- src/node/utils/ExportEtherpad.js | 93 +++---- src/node/utils/ExportHtml.js | 94 ++----- src/node/utils/ExportTxt.js | 53 +--- src/node/utils/ImportEtherpad.js | 52 ++-- src/node/utils/ImportHtml.js | 10 +- 8 files changed, 379 insertions(+), 570 deletions(-) diff --git a/src/node/db/API.js b/src/node/db/API.js index 3fcf187cf..f0b44d92e 100644 --- a/src/node/db/API.js +++ b/src/node/db/API.js @@ -287,7 +287,7 @@ exports.setHTML = async function(padID, html) // add a new changeset with the new html to the pad try { - await importHtml.setPadHTML(pad, cleanText(html)); + importHtml.setPadHTML(pad, cleanText(html)); } catch (e) { throw new customError("HTML is malformed", "apierror"); } diff --git a/src/node/handler/ExportHandler.js b/src/node/handler/ExportHandler.js index 3bcabf233..39638c222 100644 --- a/src/node/handler/ExportHandler.js +++ b/src/node/handler/ExportHandler.js @@ -19,18 +19,20 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var exporthtml = require("../utils/ExportHtml"); var exporttxt = require("../utils/ExportTxt"); var exportEtherpad = require("../utils/ExportEtherpad"); -var async = require("async"); var fs = require("fs"); var settings = require('../utils/Settings'); var os = require('os'); var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks"); var TidyHtml = require('../utils/TidyHtml'); +const util = require("util"); -var convertor = null; +const fsp_writeFile = util.promisify(fs.writeFile); +const fsp_unlink = util.promisify(fs.unlink); + +let convertor = null; // load abiword only if it is enabled if (settings.abiword != null) { @@ -47,122 +49,92 @@ const tempDirectory = os.tmpdir(); /** * do a requested export */ -exports.doExport = function(req, res, padId, type) +async function doExport(req, res, padId, type) { var fileName = padId; // allow fileName to be overwritten by a hook, the type type is kept static for security reasons - hooks.aCallFirst("exportFileName", padId, - function(err, hookFileName){ - // if fileName is set then set it to the padId, note that fileName is returned as an array. - if (hookFileName.length) { - fileName = hookFileName; - } + let hookFileName = await hooks.aCallFirst("exportFileName", padId); - // tell the browser that this is a downloadable file - res.attachment(fileName + "." + type); + // if fileName is set then set it to the padId, note that fileName is returned as an array. + if (hookFileName.length) { + fileName = hookFileName; + } - // if this is a plain text export, we can do this directly - // We have to over engineer this because tabs are stored as attributes and not plain text - if (type == "etherpad") { - exportEtherpad.getPadRaw(padId, function(err, pad) { - if (!err) { - res.send(pad); - // return; - } - }); - } else if (type == "txt") { - exporttxt.getPadTXTDocument(padId, req.params.rev, function(err, txt) { - if (!err) { - res.send(txt); - } - }); - } else { - var html; - var randNum; - var srcFile, destFile; + // tell the browser that this is a downloadable file + res.attachment(fileName + "." + type); - async.series([ - // render the html document - function(callback) { - exporthtml.getPadHTMLDocument(padId, req.params.rev, function(err, _html) { - if (ERR(err, callback)) return; - html = _html; - callback(); - }); - }, + // if this is a plain text export, we can do this directly + // We have to over engineer this because tabs are stored as attributes and not plain text + if (type === "etherpad") { + let pad = await exportEtherpad.getPadRaw(padId); + res.send(pad); + } else if (type === "txt") { + let txt = await exporttxt.getPadTXTDocument(padId, req.params.rev); + res.send(txt); + } else { + // render the html document + let html = await exporthtml.getPadHTMLDocument(padId, req.params.rev); - // decide what to do with the html export - function(callback) { - // if this is a html export, we can send this from here directly - if (type == "html") { - // do any final changes the plugin might want to make - hooks.aCallFirst("exportHTMLSend", html, function(err, newHTML) { - if (newHTML.length) html = newHTML; - res.send(html); - callback("stop"); - }); - } else { - // write the html export to a file - randNum = Math.floor(Math.random()*0xFFFFFFFF); - srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html"; - fs.writeFile(srcFile, html, callback); - } - }, + // decide what to do with the html export - // Tidy up the exported HTML - function(callback) { - // ensure html can be collected by the garbage collector - html = null; - - TidyHtml.tidy(srcFile, callback); - }, - - // send the convert job to the convertor (abiword, libreoffice, ..) - function(callback) { - destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type; - - // Allow plugins to overwrite the convert in export process - hooks.aCallAll("exportConvert", { srcFile: srcFile, destFile: destFile, req: req, res: res }, function(err, result) { - if (!err && result.length > 0) { - // console.log("export handled by plugin", destFile); - handledByPlugin = true; - callback(); - } else { - convertor.convertFile(srcFile, destFile, type, callback); - } - }); - - }, - - // send the file - function(callback) { - res.sendFile(destFile, null, callback); - }, - - // clean up temporary files - function(callback) { - async.parallel([ - function(callback) { - fs.unlink(srcFile, callback); - }, - function(callback) { - // 100ms delay to accommodate for slow windows fs - if (os.type().indexOf("Windows") > -1) { - setTimeout(function() { - fs.unlink(destFile, callback); - }, 100); - } else { - fs.unlink(destFile, callback); - } - } - ], callback); - } - ], - function(err) { - if (err && err != "stop") ERR(err); - }) - } + // if this is a html export, we can send this from here directly + if (type === "html") { + // do any final changes the plugin might want to make + let newHTML = await hooks.aCallFirst("exportHTMLSend", html); + if (newHTML.length) html = newHTML; + res.send(html); + throw "stop"; } - ); -}; + + // else write the html export to a file + let randNum = Math.floor(Math.random()*0xFFFFFFFF); + let srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html"; + await fsp_writeFile(srcFile, html); + + // Tidy up the exported HTML + // ensure html can be collected by the garbage collector + html = null; + await TidyHtml.tidy(srcFile); + + // send the convert job to the convertor (abiword, libreoffice, ..) + let destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type; + + // Allow plugins to overwrite the convert in export process + let result = await hooks.aCallAll("exportConvert", { srcFile, destFile, req, res }); + if (result.length > 0) { + // console.log("export handled by plugin", destFile); + handledByPlugin = true; + } else { + // @TODO no Promise interface for convertors (yet) + await new Promise((resolve, reject) => { + convertor.convertFile(srcFile, destFile, type, function(err) { + err ? reject("convertFailed") : resolve(); + }); + }); + } + + // send the file + let sendFile = util.promisify(res.sendFile); + await res.sendFile(destFile, null); + + // clean up temporary files + await fsp_unlink(srcFile); + + // 100ms delay to accommodate for slow windows fs + if (os.type().indexOf("Windows") > -1) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + + await fsp_unlink(destFile); + } +} + +exports.doExport = function(req, res, padId, type) +{ + doExport(req, res, padId, type).catch(err => { + if (err !== "stop") { + throw err; + } + }); +} diff --git a/src/node/handler/ImportHandler.js b/src/node/handler/ImportHandler.js index e92df4434..fb90c5059 100644 --- a/src/node/handler/ImportHandler.js +++ b/src/node/handler/ImportHandler.js @@ -20,10 +20,8 @@ * limitations under the License. */ -var ERR = require("async-stacktrace") - , padManager = require("../db/PadManager") +var padManager = require("../db/PadManager") , padMessageHandler = require("./PadMessageHandler") - , async = require("async") , fs = require("fs") , path = require("path") , settings = require('../utils/Settings') @@ -32,10 +30,16 @@ var ERR = require("async-stacktrace") , importHtml = require("../utils/ImportHtml") , importEtherpad = require("../utils/ImportEtherpad") , log4js = require("log4js") - , hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); + , hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js") + , util = require("util"); -var convertor = null; -var exportExtension = "htm"; +let fsp_exists = util.promisify(fs.exists); +let fsp_rename = util.promisify(fs.rename); +let fsp_readFile = util.promisify(fs.readFile); +let fsp_unlink = util.promisify(fs.unlink) + +let convertor = null; +let exportExtension = "htm"; // load abiword only if it is enabled and if soffice is disabled if (settings.abiword != null && settings.soffice === null) { @@ -53,292 +57,213 @@ const tmpDirectory = os.tmpdir(); /** * do a requested import */ -exports.doImport = function(req, res, padId) +async function doImport(req, res, padId) { var apiLogger = log4js.getLogger("ImportHandler"); // pipe to a file // convert file to html via abiword or soffice // set html in the pad - - var srcFile, destFile - , pad - , text - , importHandledByPlugin - , directDatabaseAccess - , useConvertor; - var randNum = Math.floor(Math.random()*0xFFFFFFFF); // setting flag for whether to use convertor or not - useConvertor = (convertor != null); + let useConvertor = (convertor != null); - async.series([ - // save the uploaded file to /tmp - function(callback) { - var form = new formidable.IncomingForm(); - form.keepExtensions = true; - form.uploadDir = tmpDirectory; + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + form.uploadDir = tmpDirectory; - form.parse(req, function(err, fields, files) { - if (err || files.file === undefined) { - // the upload failed, stop at this point - if (err) { - console.warn("Uploading Error: " + err.stack); - } - callback("uploadFailed"); - - return; + // locally wrapped Promise, since form.parse requires a callback + let srcFile = await new Promise((resolve, reject) => { + form.parse(req, function(err, fields, files) { + if (err || files.file === undefined) { + // the upload failed, stop at this point + if (err) { + console.warn("Uploading Error: " + err.stack); } - - // everything ok, continue - // save the path of the uploaded file - srcFile = files.file.path; - callback(); - }); - }, - - // ensure this is a file ending we know, else we change the file ending to .txt - // this allows us to accept source code files like .c or .java - function(callback) { - var fileEnding = path.extname(srcFile).toLowerCase() - , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"] - , fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1); - - // if the file ending is known, continue as normal - if (fileEndingKnown) { - callback(); - - return; + reject("uploadFailed"); } + resolve(files.file.path); + }); + }); + // ensure this is a file ending we know, else we change the file ending to .txt + // this allows us to accept source code files like .c or .java + let fileEnding = path.extname(srcFile).toLowerCase() + , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"] + , fileEndingUnknown = (knownFileEndings.indexOf(fileEnding) < 0); + + if (fileEndingUnknown) { + // the file ending is not known + + if (settings.allowUnknownFileEnds === true) { // we need to rename this file with a .txt ending - if (settings.allowUnknownFileEnds === true) { - var oldSrcFile = srcFile; - srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt"); - fs.rename(oldSrcFile, srcFile, callback); - } else { - console.warn("Not allowing unknown file type to be imported", fileEnding); - callback("uploadFailed"); - } - }, + let oldSrcFile = srcFile; - function(callback) { - destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension); + srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt"); + await fs.rename(oldSrcFile, srcFile); + } else { + console.warn("Not allowing unknown file type to be imported", fileEnding); + throw "uploadFailed"; + } + } - // Logic for allowing external Import Plugins - hooks.aCallAll("import", { srcFile: srcFile, destFile: destFile }, function(err, result) { - if (ERR(err, callback)) return callback(); + let destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension); - if (result.length > 0) { // This feels hacky and wrong.. - importHandledByPlugin = true; - } - callback(); - }); - }, + // Logic for allowing external Import Plugins + let result = await hooks.aCallAll("import", { srcFile, destFile }); + let importHandledByPlugin = (result.length > 0); // This feels hacky and wrong.. - function(callback) { - var fileEnding = path.extname(srcFile).toLowerCase() - var fileIsNotEtherpad = (fileEnding !== ".etherpad"); + let fileIsEtherpad = (fileEnding === ".etherpad"); + let fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm"); + let fileIsTXT = (fileEnding === ".txt"); - if (fileIsNotEtherpad) { - callback(); + let directDatabaseAccess = false; - return; - } + if (fileIsEtherpad) { + // we do this here so we can see if the pad has quite a few edits + let _pad = await padManager.getPad(padId); + let headCount = _pad.head; - // we do this here so we can see if the pad has quite a few edits - padManager.getPad(padId, function(err, _pad) { - var headCount = _pad.head; - if (headCount >= 10) { - apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this"); - return callback("padHasData"); - } + if (headCount >= 10) { + apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this"); + throw "padHasData"; + } - fs.readFile(srcFile, "utf8", function(err, _text) { - directDatabaseAccess = true; - importEtherpad.setPadRaw(padId, _text, function(err) { - callback(); - }); + const fsp_readFile = util.promisify(fs.readFile); + let _text = await fsp_readFile(srcFile, "utf8"); + directDatabaseAccess = true; + await importEtherpad.setPadRaw(padId, _text); + } + + // convert file to html if necessary + if (!importHandledByPlugin && !directDatabaseAccess) { + if (fileIsTXT) { + // Don't use convertor for text files + useConvertor = false; + } + + // See https://github.com/ether/etherpad-lite/issues/2572 + if (fileIsHTML || !useConvertor) { + // if no convertor only rename + fs.renameSync(srcFile, destFile); + } else { + // @TODO - no Promise interface for convertors (yet) + await new Promise((resolve, reject) => { + convertor.convertFile(srcFile, destFile, exportExtension, function(err) { + // catch convert errors + if (err) { + console.warn("Converting Error:", err); + reject("convertFailed"); + } + resolve(); }); }); - }, - - // convert file to html if necessary - function(callback) { - if (importHandledByPlugin || directDatabaseAccess) { - callback(); - - return; - } - - var fileEnding = path.extname(srcFile).toLowerCase(); - var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm"); - var fileIsTXT = (fileEnding === ".txt"); - - if (fileIsTXT) useConvertor = false; // Don't use convertor for text files - - // See https://github.com/ether/etherpad-lite/issues/2572 - if (fileIsHTML || (useConvertor === false)) { - // if no convertor only rename - fs.rename(srcFile, destFile, callback); - - return; - } - - convertor.convertFile(srcFile, destFile, exportExtension, function(err) { - // catch convert errors - if (err) { - console.warn("Converting Error:", err); - return callback("convertFailed"); - } - - callback(); - }); - }, - - function(callback) { - if (useConvertor || directDatabaseAccess) { - callback(); - - return; - } - - // Read the file with no encoding for raw buffer access. - fs.readFile(destFile, function(err, buf) { - if (err) throw err; - var isAscii = true; - // Check if there are only ascii chars in the uploaded file - for (var i=0, len=buf.length; i 240) { - isAscii=false; - break; - } - } - - if (!isAscii) { - callback("uploadFailed"); - - return; - } - - callback(); - }); - }, - - // get the pad object - function(callback) { - padManager.getPad(padId, function(err, _pad) { - if (ERR(err, callback)) return; - pad = _pad; - callback(); - }); - }, - - // read the text - function(callback) { - if (directDatabaseAccess) { - callback(); - - return; - } - - fs.readFile(destFile, "utf8", function(err, _text) { - if (ERR(err, callback)) return; - text = _text; - // Title needs to be stripped out else it appends it to the pad.. - text = text.replace("", "<!-- <title>"); - text = text.replace("","-->"); - - // node on windows has a delay on releasing of the file lock. - // We add a 100ms delay to work around this - if (os.type().indexOf("Windows") > -1) { - setTimeout(function() {callback();}, 100); - } else { - callback(); - } - }); - }, - - // change text of the pad and broadcast the changeset - function(callback) { - if (!directDatabaseAccess) { - var fileEnding = path.extname(srcFile).toLowerCase(); - if (importHandledByPlugin || useConvertor || fileEnding == ".htm" || fileEnding == ".html") { - importHtml.setPadHTML(pad, text, function(e){ - if (e) { - apiLogger.warn("Error importing, possibly caused by malformed HTML"); - } - }); - } else { - pad.setText(text); - } - } - - // Load the Pad into memory then broadcast updates to all clients - padManager.unloadPad(padId); - padManager.getPad(padId, function(err, _pad) { - var pad = _pad; - padManager.unloadPad(padId); - - // direct Database Access means a pad user should perform a switchToPad - // and not attempt to receive updated pad data - if (directDatabaseAccess) { - callback(); - - return; - } - - // @TODO: not waiting for updatePadClients to finish - padMessageHandler.updatePadClients(pad); - callback(); - }); - - }, - - // clean up temporary files - function(callback) { - if (directDatabaseAccess) { - callback(); - - return; - } - - try { - fs.unlinkSync(srcFile); - } catch (e) { - console.log(e); - } - - try { - fs.unlinkSync(destFile); - } catch (e) { - console.log(e); - } - - callback(); } - ], function(err) { - var status = "ok"; + } + if (!useConvertor && !directDatabaseAccess) { + // Read the file with no encoding for raw buffer access. + let buf = await fsp_readFile(destFile); + + // Check if there are only ascii chars in the uploaded file + let isAscii = ! Array.prototype.some.call(buf, c => (c > 240)); + + if (!isAscii) { + throw "uploadFailed"; + } + } + + // get the pad object + let pad = await padManager.getPad(padId); + + // read the text + let text; + + if (!directDatabaseAccess) { + text = await fsp_readFile(destFile, "utf8"); + + // Title needs to be stripped out else it appends it to the pad.. + text = text.replace("", "<!-- <title>"); + text = text.replace("","-->"); + + // node on windows has a delay on releasing of the file lock. + // We add a 100ms delay to work around this + if (os.type().indexOf("Windows") > -1){ + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + + // change text of the pad and broadcast the changeset + if (!directDatabaseAccess) { + if (importHandledByPlugin || useConvertor || fileIsHTML) { + try { + importHtml.setPadHTML(pad, text); + } catch (e) { + apiLogger.warn("Error importing, possibly caused by malformed HTML"); + } + } else { + pad.setText(text); + } + } + + // Load the Pad into memory then broadcast updates to all clients + padManager.unloadPad(padId); + pad = await padManager.getPad(padId); + padManager.unloadPad(padId); + + // direct Database Access means a pad user should perform a switchToPad + // and not attempt to receive updated pad data + if (!directDatabaseAccess) { + // tell clients to update + await padMessageHandler.updatePadClients(pad); + } + + if (!directDatabaseAccess) { + // clean up temporary files + + /* + * TODO: directly delete the file and handle the eventual error. Checking + * before for existence is prone to race conditions, and does not handle any + * errors anyway. + */ + if (await fsp_exists(srcFile)) { + fsp_unlink(srcFile); + } + + if (await fsp_exists(destFile)) { + fsp_unlink(destFile); + } + } + + return directDatabaseAccess; +} + +exports.doImport = function (req, res, padId) +{ + let status = "ok"; + let directDatabaseAccess; + + doImport(req, res, padId).then(result => { + directDatabaseAccess = result; + }).catch(err => { // check for known errors and replace the status if (err == "uploadFailed" || err == "convertFailed" || err == "padHasData") { status = err; - err = null; + } else { + throw err; } - - ERR(err); - - // close the connection - res.send( - " \ - \ - \ - " - ); }); + + // close the connection + res.send( + " \ + \ + \ + " + ); } diff --git a/src/node/utils/ExportEtherpad.js b/src/node/utils/ExportEtherpad.js index 84fe80333..0e8ef3bf1 100644 --- a/src/node/utils/ExportEtherpad.js +++ b/src/node/utils/ExportEtherpad.js @@ -15,59 +15,48 @@ */ -var async = require("async"); -var db = require("../db/DB").db; -var ERR = require("async-stacktrace"); -const thenify = require("thenify").withCallback; +let db = require("../db/DB"); -exports.getPadRaw = thenify(function(padId, callback){ - async.waterfall([ - function(cb){ - db.get("pad:"+padId, cb); - }, - function(padcontent,cb){ - var records = ["pad:"+padId]; - for (var i = 0; i <= padcontent.head; i++) { - records.push("pad:"+padId+":revs:" + i); - } +exports.getPadRaw = async function(padId) { - for (var i = 0; i <= padcontent.chatHead; i++) { - records.push("pad:"+padId+":chat:" + i); - } + let padKey = "pad:" + padId; + let padcontent = await db.get(padKey); - var data = {}; - - async.forEachSeries(Object.keys(records), function(key, r){ - - // For each piece of info about a pad. - db.get(records[key], function(err, entry){ - data[records[key]] = entry; - - // Get the Pad Authors - if(entry.pool && entry.pool.numToAttrib){ - var authors = entry.pool.numToAttrib; - async.forEachSeries(Object.keys(authors), function(k, c){ - if(authors[k][0] === "author"){ - var authorId = authors[k][1]; - - // Get the author info - db.get("globalAuthor:"+authorId, function(e, authorEntry){ - if(authorEntry && authorEntry.padIDs) authorEntry.padIDs = padId; - if(!e) data["globalAuthor:"+authorId] = authorEntry; - }); - - } - // console.log("authorsK", authors[k]); - c(null); - }); - } - r(null); // callback; - }); - }, function(err){ - cb(err, data); - }) + let records = [ padKey ]; + for (let i = 0; i <= padcontent.head; i++) { + records.push(padKey + ":revs:" + i); } - ], function(err, data){ - callback(null, data); - }); -}); + + for (let i = 0; i <= padcontent.chatHead; i++) { + records.push(padKey + ":chat:" + i); + } + + let data = {}; + for (let key of records) { + + // For each piece of info about a pad. + let entry = data[key] = await db.get(key); + + // Get the Pad Authors + if (entry.pool && entry.pool.numToAttrib) { + let authors = entry.pool.numToAttrib; + + for (let k of Object.keys(authors)) { + if (authors[k][0] === "author") { + let authorId = authors[k][1]; + + // Get the author info + let authorEntry = await db.get("globalAuthor:" + authorId); + if (authorEntry) { + data["globalAuthor:" + authorId] = authorEntry; + if (authorEntry.padIDs) { + authorEntry.padIDs = padId; + } + } + } + } + } + } + + return data; +} diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index fb15d867e..9cbcd2aa0 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -14,61 +14,29 @@ * limitations under the License. */ - -var async = require("async"); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var padManager = require("../db/PadManager"); -var ERR = require("async-stacktrace"); var _ = require('underscore'); var Security = require('ep_etherpad-lite/static/js/security'); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); var eejs = require('ep_etherpad-lite/node/eejs'); var _analyzeLine = require('./ExportHelper')._analyzeLine; var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace; -const thenify = require("thenify").withCallback; -function getPadHTML(pad, revNum, callback) +async function getPadHTML(pad, revNum) { - var atext = pad.atext; - var html; - async.waterfall([ + let atext = pad.atext; + // fetch revision atext - function (callback) - { - if (revNum != undefined) - { - pad.getInternalRevisionAText(revNum, function (err, revisionAtext) - { - if(ERR(err, callback)) return; - atext = revisionAtext; - callback(); - }); - } - else - { - callback(null); - } - }, + if (revNum != undefined) { + atext = await pad.getInternalRevisionAText(revNum); + } // convert atext to html - - - function (callback) - { - html = getHTMLFromAtext(pad, atext); - callback(null); - }], - // run final callback - - - function (err) - { - if(ERR(err, callback)) return; - callback(null, html); - }); + return getHTMLFromAtext(pad, atext); } -exports.getPadHTML = thenify(getPadHTML); +exports.getPadHTML = getPadHTML; exports.getHTMLFromAtext = getHTMLFromAtext; function getHTMLFromAtext(pad, atext, authorColors) @@ -82,15 +50,16 @@ function getHTMLFromAtext(pad, atext, authorColors) // prepare tags stored as ['tag', true] to be exported hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){ - newProps.forEach(function (propName, i){ + newProps.forEach(function (propName, i) { tags.push(propName); props.push(propName); }); }); + // prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML // with tags like hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){ - newProps.forEach(function (propName, i){ + newProps.forEach(function (propName, i) { tags.push('span data-' + propName[0] + '="' + propName[1] + '"'); props.push(propName); }); @@ -454,38 +423,31 @@ function getHTMLFromAtext(pad, atext, authorColors) hooks.aCallAll("getLineHTMLForExport", context); pieces.push(context.lineContent, "
    "); - } } + } return pieces.join(''); } -exports.getPadHTMLDocument = thenify(function (padId, revNum, callback) +exports.getPadHTMLDocument = async function (padId, revNum) { - padManager.getPad(padId, function (err, pad) - { - if(ERR(err, callback)) return; + let pad = await padManager.getPad(padId); - var stylesForExportCSS = ""; - // Include some Styles into the Head for Export - hooks.aCallAll("stylesForExport", padId, function(err, stylesForExport){ - stylesForExport.forEach(function(css){ - stylesForExportCSS += css; - }); - - getPadHTML(pad, revNum, function (err, html) - { - if(ERR(err, callback)) return; - var exportedDoc = eejs.require("ep_etherpad-lite/templates/export_html.html", { - body: html, - padId: Security.escapeHTML(padId), - extraCSS: stylesForExportCSS - }); - callback(null, exportedDoc); - }); - }); + // Include some Styles into the Head for Export + let stylesForExportCSS = ""; + let stylesForExport = await hooks.aCallAll("stylesForExport", padId); + stylesForExport.forEach(function(css){ + stylesForExportCSS += css; }); -}); + + let html = await getPadHTML(pad, revNum); + + return eejs.require("ep_etherpad-lite/templates/export_html.html", { + body: html, + padId: Security.escapeHTML(padId), + extraCSS: stylesForExportCSS + }); +} // copied from ACE var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; diff --git a/src/node/utils/ExportTxt.js b/src/node/utils/ExportTxt.js index a42878df6..304f77b8a 100644 --- a/src/node/utils/ExportTxt.js +++ b/src/node/utils/ExportTxt.js @@ -18,46 +18,22 @@ * limitations under the License. */ -var async = require("async"); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var padManager = require("../db/PadManager"); -var ERR = require("async-stacktrace"); var _analyzeLine = require('./ExportHelper')._analyzeLine; // This is slightly different than the HTML method as it passes the output to getTXTFromAText -function getPadTXT(pad, revNum, callback) +var getPadTXT = async function(pad, revNum) { - var atext = pad.atext; - var html; - async.waterfall([ + let atext = pad.atext; - // fetch revision atext - function(callback) { - if (revNum != undefined) { - pad.getInternalRevisionAText(revNum, function(err, revisionAtext) { - if (ERR(err, callback)) return; - - atext = revisionAtext; - callback(); - }); - } else { - callback(null); - } - }, + if (revNum != undefined) { + // fetch revision atext + atext = await pad.getInternalRevisionAText(revNum); + } // convert atext to html - function(callback) { - // only this line is different to the HTML function - html = getTXTFromAtext(pad, atext); - callback(null); - }], - - // run final callback - function(err) { - if (ERR(err, callback)) return; - - callback(null, html); - }); + return getTXTFromAtext(pad, atext); } // This is different than the functionality provided in ExportHtml as it provides formatting @@ -244,15 +220,8 @@ function getTXTFromAtext(pad, atext, authorColors) exports.getTXTFromAtext = getTXTFromAtext; -exports.getPadTXTDocument = function(padId, revNum, callback) +exports.getPadTXTDocument = async function(padId, revNum) { - padManager.getPad(padId, function(err, pad) { - if (ERR(err, callback)) return; - - getPadTXT(pad, revNum, function(err, html) { - if (ERR(err, callback)) return; - - callback(null, html); - }); - }); -}; + let pad = await padManager.getPad(padId); + return getPadTXT(pad, revNum); +} diff --git a/src/node/utils/ImportEtherpad.js b/src/node/utils/ImportEtherpad.js index 138c42936..a5b1074e6 100644 --- a/src/node/utils/ImportEtherpad.js +++ b/src/node/utils/ImportEtherpad.js @@ -15,43 +15,44 @@ */ var log4js = require('log4js'); -var async = require("async"); -var db = require("../db/DB").db; -const thenify = require("thenify").withCallback; +const db = require("../db/DB"); -exports.setPadRaw = thenify(function(padId, records, callback) +exports.setPadRaw = function(padId, records) { records = JSON.parse(records); - async.eachSeries(Object.keys(records), function(key, cb) { - var value = records[key]; + Object.keys(records).forEach(async function(key) { + let value = records[key]; if (!value) { - return setImmediate(cb); + return; } + let newKey; + if (value.padIDs) { // Author data - rewrite author pad ids value.padIDs[padId] = 1; - var newKey = key; + newKey = key; // Does this author already exist? - db.get(key, function(err, author) { - if (author) { - // Yes, add the padID to the author - if (Object.prototype.toString.call(author) === '[object Array]') { - author.padIDs.push(padId); - } - value = author; - } else { - // No, create a new array with the author info in - value.padIDs = [padId]; + let author = await db.get(key); + + if (author) { + // Yes, add the padID to the author + if (Object.prototype.toString.call(author) === '[object Array]') { + author.padIDs.push(padId); } - }); + + value = author; + } else { + // No, create a new array with the author info in + value.padIDs = [ padId ]; + } } else { // Not author data, probably pad data // we can split it to look to see if it's pad data - var oldPadId = key.split(":"); + let oldPadId = key.split(":"); // we know it's pad data if (oldPadId[0] === "pad") { @@ -59,16 +60,11 @@ exports.setPadRaw = thenify(function(padId, records, callback) oldPadId[1] = padId; // and create the value - var newKey = oldPadId.join(":"); // create the new key + newKey = oldPadId.join(":"); // create the new key } } // Write the value to the server - db.set(newKey, value); - - setImmediate(cb); - }, - function() { - callback(null, true); + await db.set(newKey, value); }); -}); +} diff --git a/src/node/utils/ImportHtml.js b/src/node/utils/ImportHtml.js index d46b715d3..63b35fa75 100644 --- a/src/node/utils/ImportHtml.js +++ b/src/node/utils/ImportHtml.js @@ -18,9 +18,8 @@ var log4js = require('log4js'); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var contentcollector = require("ep_etherpad-lite/static/js/contentcollector"); var cheerio = require("cheerio"); -const thenify = require("thenify").withCallback; -function setPadHTML(pad, html, callback) +exports.setPadHTML = function(pad, html) { var apiLogger = log4js.getLogger("ImportHtml"); @@ -44,7 +43,7 @@ function setPadHTML(pad, html, callback) apiLogger.warn("HTML was not properly formed", e); // don't process the HTML because it was bad - return callback(e); + throw e; } var result = cc.finish(); @@ -52,7 +51,7 @@ function setPadHTML(pad, html, callback) apiLogger.debug('Lines:'); var i; - for (i = 0; i < result.lines.length; i += 1) { + for (i = 0; i < result.lines.length; i++) { apiLogger.debug('Line ' + (i + 1) + ' text: ' + result.lines[i]); apiLogger.debug('Line ' + (i + 1) + ' attributes: ' + result.lineAttribs[i]); } @@ -92,7 +91,4 @@ function setPadHTML(pad, html, callback) apiLogger.debug('The changeset: ' + theChangeset); pad.setText("\n"); pad.appendRevision(theChangeset); - callback(null); } - -exports.setPadHTML = thenify(setPadHTML); From b664eb488cc59271947d2adb1d0162c5ca7962c9 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 8 Feb 2019 14:46:05 +0000 Subject: [PATCH 49/58] ImportHandler.js: ensure import connection closing happens at the right point --- src/node/handler/ImportHandler.js | 71 ++++++++++++++++--------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/node/handler/ImportHandler.js b/src/node/handler/ImportHandler.js index fb90c5059..d2bd05289 100644 --- a/src/node/handler/ImportHandler.js +++ b/src/node/handler/ImportHandler.js @@ -118,8 +118,6 @@ async function doImport(req, res, padId) let fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm"); let fileIsTXT = (fileEnding === ".txt"); - let directDatabaseAccess = false; - if (fileIsEtherpad) { // we do this here so we can see if the pad has quite a few edits let _pad = await padManager.getPad(padId); @@ -132,12 +130,12 @@ async function doImport(req, res, padId) const fsp_readFile = util.promisify(fs.readFile); let _text = await fsp_readFile(srcFile, "utf8"); - directDatabaseAccess = true; + req.directDatabaseAccess = true; await importEtherpad.setPadRaw(padId, _text); } // convert file to html if necessary - if (!importHandledByPlugin && !directDatabaseAccess) { + if (!importHandledByPlugin && !req.directDatabaseAccess) { if (fileIsTXT) { // Don't use convertor for text files useConvertor = false; @@ -162,7 +160,7 @@ async function doImport(req, res, padId) } } - if (!useConvertor && !directDatabaseAccess) { + if (!useConvertor && !req.directDatabaseAccess) { // Read the file with no encoding for raw buffer access. let buf = await fsp_readFile(destFile); @@ -180,7 +178,7 @@ async function doImport(req, res, padId) // read the text let text; - if (!directDatabaseAccess) { + if (!req.directDatabaseAccess) { text = await fsp_readFile(destFile, "utf8"); // Title needs to be stripped out else it appends it to the pad.. @@ -195,7 +193,7 @@ async function doImport(req, res, padId) } // change text of the pad and broadcast the changeset - if (!directDatabaseAccess) { + if (!req.directDatabaseAccess) { if (importHandledByPlugin || useConvertor || fileIsHTML) { try { importHtml.setPadHTML(pad, text); @@ -214,56 +212,59 @@ async function doImport(req, res, padId) // direct Database Access means a pad user should perform a switchToPad // and not attempt to receive updated pad data - if (!directDatabaseAccess) { - // tell clients to update - await padMessageHandler.updatePadClients(pad); + if (req.directDatabaseAccess) { + return; } - if (!directDatabaseAccess) { - // clean up temporary files + // tell clients to update + await padMessageHandler.updatePadClients(pad); - /* - * TODO: directly delete the file and handle the eventual error. Checking - * before for existence is prone to race conditions, and does not handle any - * errors anyway. - */ - if (await fsp_exists(srcFile)) { - fsp_unlink(srcFile); - } + // clean up temporary files - if (await fsp_exists(destFile)) { - fsp_unlink(destFile); - } + /* + * TODO: directly delete the file and handle the eventual error. Checking + * before for existence is prone to race conditions, and does not handle any + * errors anyway. + */ + if (await fsp_exists(srcFile)) { + fsp_unlink(srcFile); } - return directDatabaseAccess; + if (await fsp_exists(destFile)) { + fsp_unlink(destFile); + } } exports.doImport = function (req, res, padId) { + /** + * NB: abuse the 'req' object by storing an additional + * 'directDatabaseAccess' property on it so that it can + * be passed back in the HTML below. + * + * this is necessary because in the 'throw' paths of + * the function above there's no other way to return + * a value to the caller. + */ let status = "ok"; - let directDatabaseAccess; - - doImport(req, res, padId).then(result => { - directDatabaseAccess = result; - }).catch(err => { + doImport(req, res, padId).catch(err => { // check for known errors and replace the status if (err == "uploadFailed" || err == "convertFailed" || err == "padHasData") { status = err; } else { throw err; } - }); - - // close the connection - res.send( + }).then(() => { + // close the connection + res.send( " \ \ \ " - ); + ); + }); } From ebb8a64e3c9468fa7eeb4b0b44b742ad0374fb79 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 11:14:27 +0000 Subject: [PATCH 50/58] errorhandling.js: use promise db.doShutdown interface --- src/node/hooks/express/errorhandling.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/hooks/express/errorhandling.js b/src/node/hooks/express/errorhandling.js index 9a07dc669..66553621c 100644 --- a/src/node/hooks/express/errorhandling.js +++ b/src/node/hooks/express/errorhandling.js @@ -21,7 +21,7 @@ exports.gracefulShutdown = function(err) { console.log("graceful shutdown..."); // do the db shutdown - db.db.doShutdown(function() { + db.doShutdown().then(function() { console.log("db sucessfully closed."); process.exit(0); From 6d1b6b2796a90911a3ed3ec28a70bbb2cd91b814 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 11:14:38 +0000 Subject: [PATCH 51/58] db/Pad.js: convert to promises/async Also updated some small chunks of dependent code that couldn't be converted until this one had been done. --- src/node/db/AuthorManager.js | 14 +- src/node/db/GroupManager.js | 14 +- src/node/db/Pad.js | 606 ++++++++++++----------------------- src/node/db/PadManager.js | 56 +--- 4 files changed, 236 insertions(+), 454 deletions(-) diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index 99584da19..0c6e3f750 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -144,12 +144,11 @@ exports.getAuthor = function(author) /** * Returns the color Id of the author * @param {String} author The id of the author - * @param {Function} callback callback(err, colorId) */ -exports.getAuthorColorId = thenify(function(author, callback) +exports.getAuthorColorId = function(author) { - db.db.getSub("globalAuthor:" + author, ["colorId"], callback); -}); + return db.getSub("globalAuthor:" + author, ["colorId"]); +} /** * Sets the color Id of the author @@ -164,12 +163,11 @@ exports.setAuthorColorId = function(author, colorId) /** * Returns the name of the author * @param {String} author The id of the author - * @param {Function} callback callback(err, name) */ -exports.getAuthorName = thenify(function(author, callback) +exports.getAuthorName = function(author) { - db.db.getSub("globalAuthor:" + author, ["name"], callback); -}); + return db.getSub("globalAuthor:" + author, ["name"]); +} /** * Sets the name of the author diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index 29a7ff063..238c423b1 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -18,13 +18,11 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; var db = require("./DB"); var padManager = require("./PadManager"); var sessionManager = require("./SessionManager"); -const thenify = require("thenify").withCallback; exports.listAllGroups = async function() { @@ -85,15 +83,13 @@ exports.deleteGroup = async function(groupID) await db.set("groups", newGroups); } -// @TODO: this is the only function still called with a callback -exports.doesGroupExist = thenify(function(groupID, callback) +exports.doesGroupExist = async function(groupID) { // try to get the group entry - db.db.get("group:" + groupID, function (err, group) { - if (ERR(err, callback)) return; - callback(null, group != null); - }); -}); + let group = await db.get("group:" + groupID); + + return (group != null); +} exports.createGroup = async function() { diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index db35bd367..fb36bdf58 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -3,11 +3,9 @@ */ -var ERR = require("async-stacktrace"); var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var AttributePool = require("ep_etherpad-lite/static/js/AttributePool"); var db = require("./DB"); -var async = require("async"); var settings = require('../utils/Settings'); var authorManager = require("./AuthorManager"); var padManager = require("./PadManager"); @@ -18,8 +16,6 @@ var readOnlyManager = require("./ReadOnlyManager"); var crypto = require("crypto"); var randomString = require("../utils/randomstring"); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); -const thenify = require("thenify").withCallback; -const nodeify = require("nodeify"); // serialization/deserialization attributes var attributeBlackList = ["id"]; @@ -34,7 +30,7 @@ exports.cleanText = function (txt) { }; -var Pad = function Pad(id) { +let Pad = function Pad(id) { this.atext = Changeset.makeAText("\n"); this.pool = new AttributePool(); this.head = -1; @@ -129,7 +125,7 @@ Pad.prototype.saveToDatabase = function saveToDatabase() { } } - db.db.set("pad:" + this.id, dbObject); + db.set("pad:" + this.id, dbObject); } // get time of last edit (changeset application) @@ -138,17 +134,17 @@ Pad.prototype.getLastEdit = function getLastEdit() { return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]); } -Pad.prototype.getRevisionChangeset = thenify(function getRevisionChangeset(revNum, callback) { - db.db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"], callback); -}); +Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum) { + return db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"]); +} -Pad.prototype.getRevisionAuthor = thenify(function getRevisionAuthor(revNum, callback) { - db.db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"], callback); -}); +Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum) { + return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"]); +} -Pad.prototype.getRevisionDate = thenify(function getRevisionDate(revNum, callback) { - db.db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"], callback); -}); +Pad.prototype.getRevisionDate = function getRevisionDate(revNum) { + return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]); +} Pad.prototype.getAllAuthors = function getAllAuthors() { var authors = []; @@ -162,101 +158,57 @@ Pad.prototype.getAllAuthors = function getAllAuthors() { return authors; }; -Pad.prototype.getInternalRevisionAText = thenify(function getInternalRevisionAText(targetRev, callback) { - var _this = this; - - var keyRev = this.getKeyRevisionNumber(targetRev); - var atext; - var changesets = []; +Pad.prototype.getInternalRevisionAText = async function getInternalRevisionAText(targetRev) { + let keyRev = this.getKeyRevisionNumber(targetRev); // find out which changesets are needed - var neededChangesets = []; - var curRev = keyRev; - while (curRev < targetRev) { - curRev++; - neededChangesets.push(curRev); + let neededChangesets = []; + for (let curRev = keyRev; curRev < targetRev; ) { + neededChangesets.push(++curRev); } - async.series([ - // get all needed data out of the database - function(callback) { - async.parallel([ - // get the atext of the key revision - function (callback) { - db.db.getSub("pad:" + _this.id + ":revs:" + keyRev, ["meta", "atext"], function(err, _atext) { - if (ERR(err, callback)) return; - try { - atext = Changeset.cloneAText(_atext); - } catch (e) { - return callback(e); - } + // get all needed data out of the database - callback(); - }); - }, + // get the atext of the key revision + let _atext = await db.getSub("pad:" + this.id + ":revs:" + keyRev, ["meta", "atext"]); + let atext = Changeset.cloneAText(_atext); - // get all needed changesets - function (callback) { - async.forEach(neededChangesets, function(item, callback) { - _this.getRevisionChangeset(item, function(err, changeset) { - if (ERR(err, callback)) return; - changesets[item] = changeset; - callback(); - }); - }, callback); - } - ], callback); - }, + // get all needed changesets + let changesets = []; + await Promise.all(neededChangesets.map(item => { + return this.getRevisionChangeset(item).then(changeset => { + changesets[item] = changeset; + }); + })); - // apply all changesets to the key changeset - function(callback) { - var apool = _this.apool(); - var curRev = keyRev; + // apply all changesets to the key changeset + let apool = this.apool(); + for (let curRev = keyRev; curRev < targetRev; ) { + let cs = changesets[++curRev]; + atext = Changeset.applyToAText(cs, atext, apool); + } - while (curRev < targetRev) { - curRev++; - var cs = changesets[curRev]; - try { - atext = Changeset.applyToAText(cs, atext, apool); - } catch(e) { - return callback(e) - } - } + return atext; +} - callback(null); - } - ], - function(err) { - if (ERR(err, callback)) return; - callback(null, atext); - }); -}); +Pad.prototype.getRevision = function getRevisionChangeset(revNum) { + return db.get("pad:" + this.id + ":revs:" + revNum); +} -Pad.prototype.getRevision = thenify(function getRevisionChangeset(revNum, callback) { - db.db.get("pad:" + this.id + ":revs:" + revNum, callback); -}); - -Pad.prototype.getAllAuthorColors = thenify(function getAllAuthorColors(callback) { - var authors = this.getAllAuthors(); - var returnTable = {}; - var colorPalette = authorManager.getColorPalette(); - - async.forEach(authors, function(author, callback) { - authorManager.getAuthorColorId(author, function(err, colorId) { - if (err) { - return callback(err); - } +Pad.prototype.getAllAuthorColors = async function getAllAuthorColors() { + let authors = this.getAllAuthors(); + let returnTable = {}; + let colorPalette = authorManager.getColorPalette(); + await Promise.all(authors.map(author => { + return authorManager.getAuthorColorId(author).then(colorId => { // colorId might be a hex color or an number out of the palette returnTable[author] = colorPalette[colorId] || colorId; - - callback(); }); - }, - function(err) { - callback(err, returnTable); - }); -}); + })); + + return returnTable; +} Pad.prototype.getValidRevisionRange = function getValidRevisionRange(startRev, endRev) { startRev = parseInt(startRev, 10); @@ -327,83 +279,49 @@ Pad.prototype.appendChatMessage = function appendChatMessage(text, userId, time) this.saveToDatabase(); }; -Pad.prototype.getChatMessage = thenify(function getChatMessage(entryNum, callback) { - var _this = this; - var entry; +Pad.prototype.getChatMessage = async function getChatMessage(entryNum) { + // get the chat entry + let entry = await db.get("pad:" + this.id + ":chat:" + entryNum); - async.series([ - // get the chat entry - function(callback) { - db.db.get("pad:" + _this.id + ":chat:" + entryNum, function(err, _entry) { - if (ERR(err, callback)) return; - entry = _entry; - callback(); - }); - }, - - // add the authorName - function(callback) { - // this chat message doesn't exist, return null - if (entry == null) { - callback(); - return; - } - - // get the authorName - authorManager.getAuthorName(entry.userId, function(err, authorName) { - if (ERR(err, callback)) return; - entry.userName = authorName; - callback(); - }); - } - ], - function(err) { - if (ERR(err, callback)) return; - callback(null, entry); - }); -}); - -Pad.prototype.getChatMessages = thenify(function getChatMessages(start, end, callback) { - // collect the numbers of chat entries and in which order we need them - var neededEntries = []; - var order = 0; - for (var i = start; i <= end; i++) { - neededEntries.push({ entryNum: i, order: order }); - order++; + // get the authorName if the entry exists + if (entry != null) { + entry.userName = await authorManager.getAuthorName(entry.userId); } - var _this = this; + return entry; +}; + +Pad.prototype.getChatMessages = async function getChatMessages(start, end) { + + // collect the numbers of chat entries and in which order we need them + let neededEntries = []; + for (let order = 0, entryNum = start; entryNum <= end; ++order, ++entryNum) { + neededEntries.push({ entryNum, order }); + } // get all entries out of the database - var entries = []; - async.forEach(neededEntries, function(entryObject, callback) { - _this.getChatMessage(entryObject.entryNum, function(err, entry) { - if (ERR(err, callback)) return; + let entries = []; + await Promise.all(neededEntries.map(entryObject => { + return this.getChatMessage(entryObject.entryNum).then(entry => { entries[entryObject.order] = entry; - callback(); }); - }, - function(err) { - if (ERR(err, callback)) return; + })); - // sort out broken chat entries - // it looks like in happened in the past that the chat head was - // incremented, but the chat message wasn't added - var cleanedEntries = []; - for (var i=0; i < entries.length; i++) { - if (entries[i] != null) { - cleanedEntries.push(entries[i]); - } else { - console.warn("WARNING: Found broken chat entry in pad " + _this.id); - } + // sort out broken chat entries + // it looks like in happened in the past that the chat head was + // incremented, but the chat message wasn't added + let cleanedEntries = entries.filter(entry => { + let pass = (entry != null); + if (!pass) { + console.warn("WARNING: Found broken chat entry in pad " + this.id); } - - callback(null, cleanedEntries); + return pass; }); -}); -Pad.prototype.init = thenify(function init(text, callback) { - var _this = this; + return cleanedEntries; +} + +Pad.prototype.init = async function init(text) { // replace text with default text if text isn't set if (text == null) { @@ -411,35 +329,31 @@ Pad.prototype.init = thenify(function init(text, callback) { } // try to load the pad - db.db.get("pad:" + this.id, function(err, value) { - if (ERR(err, callback)) return; + let value = await db.get("pad:" + this.id); - // if this pad exists, load it - if (value != null) { - // copy all attr. To a transfrom via fromJsonable if necessary - for (var attr in value) { - if (jsonableList.indexOf(attr) !== -1) { - _this[attr] = _this[attr].fromJsonable(value[attr]); - } else { - _this[attr] = value[attr]; - } + // if this pad exists, load it + if (value != null) { + // copy all attr. To a transfrom via fromJsonable if necassary + for (var attr in value) { + if (jsonableList.indexOf(attr) !== -1) { + this[attr] = this[attr].fromJsonable(value[attr]); + } else { + this[attr] = value[attr]; } - } else { - // this pad doesn't exist, so create it - var firstChangeset = Changeset.makeSplice("\n", 0, 0, exports.cleanText(text)); - - _this.appendRevision(firstChangeset, ''); } + } else { + // this pad doesn't exist, so create it + let firstChangeset = Changeset.makeSplice("\n", 0, 0, exports.cleanText(text)); - hooks.callAll("padLoad", { 'pad': _this }); - callback(null); - }); -}); + this.appendRevision(firstChangeset, ''); + } -Pad.prototype.copy = thenify(function copy(destinationID, force, callback) { - var sourceID = this.id; - var _this = this; - var destGroupID; + hooks.callAll("padLoad", { 'pad': this }); +} + +Pad.prototype.copy = async function copy(destinationID, force) { + + let sourceID = this.id; // allow force to be a string if (typeof force === "string") { @@ -454,236 +368,134 @@ Pad.prototype.copy = thenify(function copy(destinationID, force, callback) { // padMessageHandler.kickSessionsFromPad(sourceID); // flush the source pad: - _this.saveToDatabase(); + this.saveToDatabase(); - async.series([ - // if it's a group pad, let's make sure the group exists. - function(callback) { - if (destinationID.indexOf("$") === -1) { - callback(); - return; - } + // if it's a group pad, let's make sure the group exists. + let destGroupID; + if (destinationID.indexOf("$") >= 0) { - destGroupID = destinationID.split("$")[0]; - groupManager.doesGroupExist(destGroupID, function (err, exists) { - if (ERR(err, callback)) return; + destGroupID = destinationID.split("$")[0] + let groupExists = await groupManager.doesGroupExist(destGroupID); - // group does not exist - if (exists == false) { - callback(new customError("groupID does not exist for destinationID", "apierror")); - return; - } - - // everything is fine, continue - callback(); - }); - }, - - // if the pad exists, we should abort, unless forced. - function(callback) { - padManager.doesPadExists(destinationID, function (err, exists) { - if (ERR(err, callback)) return; - - /* - * this is the negation of a truthy comparison. Has been left in this - * wonky state to keep the old (possibly buggy) behaviour - */ - if (!(exists == true)) { - callback(); - return; - } - - if (!force) { - console.error("erroring out without force"); - callback(new customError("destinationID already exists", "apierror")); - console.error("erroring out without force - after"); - return; - } - - // exists and forcing - padManager.getPad(destinationID, function(err, pad) { - if (ERR(err, callback)) return; - pad.remove(callback); - }); - }); - }, - - // copy the 'pad' entry - function(callback) { - db.db.get("pad:" + sourceID, function(err, pad) { - db.set("pad:" + destinationID, pad); - }); - - callback(); - }, - - // copy all relations - function(callback) { - async.parallel([ - // copy all chat messages - function(callback) { - var chatHead = _this.chatHead; - - for (var i=0; i <= chatHead; i++) { - db.db.get("pad:" + sourceID + ":chat:" + i, function (err, chat) { - if (ERR(err, callback)) return; - db.set("pad:" + destinationID + ":chat:" + i, chat); - }); - } - - callback(); - }, - - // copy all revisions - function(callback) { - var revHead = _this.head; - for (var i=0; i <= revHead; i++) { - db.db.get("pad:" + sourceID + ":revs:" + i, function (err, rev) { - if (ERR(err, callback)) return; - db.set("pad:" + destinationID + ":revs:" + i, rev); - }); - } - - callback(); - }, - - // add the new pad to all authors who contributed to the old one - function(callback) { - var authorIDs = _this.getAllAuthors(); - authorIDs.forEach(function (authorID) { - authorManager.addPad(authorID, destinationID); - }); - - callback(); - }, - // parallel - ], callback); - }, - - function(callback) { - if (destGroupID) { - // Group pad? Add it to the group's list - db.setSub("group:" + destGroupID, ["pads", destinationID], 1); - } - - // Initialize the new pad (will update the listAllPads cache) - setTimeout(function() { - padManager.getPad(destinationID, null, callback) // this runs too early. - }, 10); - }, - - // let the plugins know the pad was copied - function(callback) { - hooks.callAll('padCopy', { 'originalPad': _this, 'destinationID': destinationID }); - callback(); + // group does not exist + if (!groupExists) { + throw new customError("groupID does not exist for destinationID", "apierror"); } - // series - ], - function(err) { - if (ERR(err, callback)) return; - callback(null, { padID: destinationID }); - }); -}); + } -Pad.prototype.remove = thenify(function remove(callback) { + // if the pad exists, we should abort, unless forced. + let exists = await padManager.doesPadExist(destinationID); + + if (exists) { + if (!force) { + console.error("erroring out without force"); + throw new customError("destinationID already exists", "apierror"); + + return; + } + + // exists and forcing + let pad = await padManager.getPad(destinationID); + await pad.remove(callback); + } + + // copy the 'pad' entry + let pad = await db.get("pad:" + sourceID); + db.set("pad:" + destinationID, pad); + + // copy all relations in parallel + let promises = []; + + // copy all chat messages + let chatHead = this.chatHead; + for (let i = 0; i <= chatHead; ++i) { + let p = db.get("pad:" + sourceID + ":chat:" + i).then(chat => { + return db.set("pad:" + destinationID + ":chat:" + i, chat); + }); + promises.push(p); + } + + // copy all revisions + let revHead = this.head; + for (let i = 0; i <= revHead; ++i) { + let p = db.get("pad:" + sourceID + ":revs:" + i).then(rev => { + return db.set("pad:" + destinationID + ":revs:" + i, rev); + }); + promises.push(p); + } + + // add the new pad to all authors who contributed to the old one + this.getAllAuthors().forEach(authorID => { + authorManager.addPad(authorID, destinationID); + }); + + // wait for the above to complete + await Promise.all(promises); + + // Group pad? Add it to the group's list + if (destGroupID) { + await db.setSub("group:" + destGroupID, ["pads", destinationID], 1); + } + + // delay still necessary? + await new Promise(resolve => setTimeout(resolve, 10)); + + // Initialize the new pad (will update the listAllPads cache) + await padManager.getPad(destinationID, null); // this runs too early. + + // let the plugins know the pad was copied + hooks.callAll('padCopy', { 'originalPad': this, 'destinationID': destinationID }); + + return { padID: destinationID }; +} + +Pad.prototype.remove = async function remove() { var padID = this.id; - var _this = this; // kick everyone from this pad padMessageHandler.kickSessionsFromPad(padID); - async.series([ - // delete all relations - function(callback) { - async.parallel([ - // is it a group pad? -> delete the entry of this pad in the group - function(callback) { - if (padID.indexOf("$") === -1) { - // it isn't a group pad, nothing to do here - callback(); - return; - } + // delete all relations + + // is it a group pad? -> delete the entry of this pad in the group + if (padID.indexOf("$") >= 0) { - // it is a group pad - var groupID = padID.substring(0, padID.indexOf("$")); + // it is a group pad + let groupID = padID.substring(0, padID.indexOf("$")); + let group = await db.get("group:" + groupID); - db.db.get("group:" + groupID, function (err, group) { - if (ERR(err, callback)) return; + // remove the pad entry + delete group.pads[padID]; - // remove the pad entry - delete group.pads[padID]; + // set the new value + db.set("group:" + groupID, group); + } - // set the new value - db.set("group:" + groupID, group); + // remove the readonly entries + let readonlyID = readOnlyManager.getReadOnlyId(padID); - callback(); - }); - }, + db.remove("pad2readonly:" + padID); + db.remove("readonly2pad:" + readonlyID); - // remove the readonly entries - function(callback) { - // @TODO - temporary until surrounding code is Promisified - function getReadOnlyId(padID, callback) { - return nodeify(readOnlyManager.getReadOnlyId(padID), callback); - } + // delete all chat messages + for (let i = 0, n = this.chatHead; i <= n; ++i) { + db.remove("pad:" + padID + ":chat:" + i); + } - getReadOnlyId(padID, function(err, readonlyID) { - if (ERR(err, callback)) return; + // delete all revisions + for (let i = 0, n = this.head; i <= n; ++i) { + db.remove("pad:" + padID + ":revs:" + i); + } - db.remove("pad2readonly:" + padID); - db.remove("readonly2pad:" + readonlyID); - - callback(); - }); - }, - - // delete all chat messages - function(callback) { - var chatHead = _this.chatHead; - - for (var i = 0; i <= chatHead; i++) { - db.remove("pad:" + padID + ":chat:" + i); - } - - callback(); - }, - - // delete all revisions - function(callback) { - var revHead = _this.head; - - for (var i = 0; i <= revHead; i++) { - db.remove("pad:" + padID + ":revs:" + i); - } - - callback(); - }, - - // remove pad from all authors who contributed - function(callback) { - var authorIDs = _this.getAllAuthors(); - - authorIDs.forEach(function (authorID) { - authorManager.removePad(authorID, padID); - }); - - callback(); - } - ], callback); - }, - - // delete the pad entry and delete pad from padManager - function(callback) { - padManager.removePad(padID); - hooks.callAll("padRemove", { 'padID': padID }); - callback(); - } - ], - function(err) { - if (ERR(err, callback)) return; - callback(); + // remove pad from all authors who contributed + this.getAllAuthors().forEach(authorID => { + authorManager.removePad(authorID, padID); }); -}); + + // delete the pad entry and delete pad from padManager + padManager.removePad(padID); + hooks.callAll("padRemove", { padID }); +} // set in db Pad.prototype.setPublicStatus = function setPublicStatus(publicStatus) { diff --git a/src/node/db/PadManager.js b/src/node/db/PadManager.js index 858ce5b45..23164a7a9 100644 --- a/src/node/db/PadManager.js +++ b/src/node/db/PadManager.js @@ -18,11 +18,9 @@ * limitations under the License. */ -var ERR = require("async-stacktrace"); var customError = require("../utils/customError"); var Pad = require("../db/Pad").Pad; -var db = require("./DB").db; -const thenify = require("thenify").withCallback; +var db = require("./DB"); /** * A cache of all loaded Pads. @@ -114,59 +112,43 @@ let padList = { * @param id A String with the id of the pad * @param {Function} callback */ -exports.getPad = thenify(function(id, text, callback) +exports.getPad = async function(id, text) { // check if this is a valid padId if (!exports.isValidPadId(id)) { - callback(new customError(id + " is not a valid padId", "apierror")); - - return; - } - - // make text an optional parameter - if (typeof text == "function") { - callback = text; - text = null; + throw new customError(id + " is not a valid padId", "apierror"); } // check if this is a valid text if (text != null) { // check if text is a string if (typeof text != "string") { - callback(new customError("text is not a string", "apierror")); - - return; + throw new customError("text is not a string", "apierror"); } // check if text is less than 100k chars if (text.length > 100000) { - callback(new customError("text must be less than 100k chars", "apierror")); - - return; + throw new customError("text must be less than 100k chars", "apierror"); } } - var pad = globalPads.get(id); + let pad = globalPads.get(id); // return pad if it's already loaded if (pad != null) { - callback(null, pad); - - return; + return pad; } // try to load pad pad = new Pad(id); // initalize the pad - pad.init(text, function(err) { - if (ERR(err, callback)) return; + await pad.init(text); + globalPads.set(id, pad); + padList.addPad(id); - globalPads.set(id, pad); - padList.addPad(id); - callback(null, pad); - }); -}); + return pad; +} exports.listAllPads = async function() { @@ -176,18 +158,12 @@ exports.listAllPads = async function() } // checks if a pad exists -exports.doesPadExist = thenify(function(padId, callback) +exports.doesPadExist = async function(padId) { - db.get("pad:" + padId, function(err, value) { - if (ERR(err, callback)) return; + let value = await db.get("pad:" + padId); - if (value != null && value.atext) { - callback(null, true); - } else { - callback(null, false); - } - }); -}); + return (value != null && value.atext); +} // alias for backwards compatibility exports.doesPadExists = exports.doesPadExist; From 4622309dc247ef7fcefc8dcca6d73ab07c44b3b6 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 13:42:41 +0000 Subject: [PATCH 52/58] TidyHtml.js: convert to promises test case uses "nodeify" to convert the calls to TidyHtml back into nodeback because it integrates better with the test framework --- src/node/utils/TidyHtml.js | 59 +++++++++++++++++---------------- tests/backend/specs/api/tidy.js | 10 ++++-- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/node/utils/TidyHtml.js b/src/node/utils/TidyHtml.js index 0f7119894..26d48a62f 100644 --- a/src/node/utils/TidyHtml.js +++ b/src/node/utils/TidyHtml.js @@ -5,38 +5,39 @@ var log4js = require('log4js'); var settings = require('./Settings'); var spawn = require('child_process').spawn; -const thenify = require("thenify").withCallback; -exports.tidy = thenify(function(srcFile, callback) { +exports.tidy = function(srcFile) { var logger = log4js.getLogger('TidyHtml'); - // Don't do anything if Tidy hasn't been enabled - if (!settings.tidyHtml) { - logger.debug('tidyHtml has not been configured yet, ignoring tidy request'); - return callback(null); - } + return new Promise((resolve, reject) => { - var errMessage = ''; - - // Spawn a new tidy instance that cleans up the file inline - logger.debug('Tidying ' + srcFile); - var tidy = spawn(settings.tidyHtml, ['-modify', srcFile]); - - // Keep track of any error messages - tidy.stderr.on('data', function (data) { - errMessage += data.toString(); - }); - - // Wait until Tidy is done - tidy.on('close', function(code) { - // Tidy returns a 0 when no errors occur and a 1 exit code when - // the file could be tidied but a few warnings were generated - if (code === 0 || code === 1) { - logger.debug('Tidied ' + srcFile + ' successfully'); - return callback(null); - } else { - logger.error('Failed to tidy ' + srcFile + '\n' + errMessage); - return callback('Tidy died with exit code ' + code); + // Don't do anything if Tidy hasn't been enabled + if (!settings.tidyHtml) { + logger.debug('tidyHtml has not been configured yet, ignoring tidy request'); + return resolve(null); } + + var errMessage = ''; + + // Spawn a new tidy instance that cleans up the file inline + logger.debug('Tidying ' + srcFile); + var tidy = spawn(settings.tidyHtml, ['-modify', srcFile]); + + // Keep track of any error messages + tidy.stderr.on('data', function (data) { + errMessage += data.toString(); + }); + + tidy.on('close', function(code) { + // Tidy returns a 0 when no errors occur and a 1 exit code when + // the file could be tidied but a few warnings were generated + if (code === 0 || code === 1) { + logger.debug('Tidied ' + srcFile + ' successfully'); + resolve(null); + } else { + logger.error('Failed to tidy ' + srcFile + '\n' + errMessage); + reject('Tidy died with exit code ' + code); + } + }); }); -}); +} diff --git a/tests/backend/specs/api/tidy.js b/tests/backend/specs/api/tidy.js index 6f38ac7b0..3ef61931b 100644 --- a/tests/backend/specs/api/tidy.js +++ b/tests/backend/specs/api/tidy.js @@ -1,10 +1,12 @@ var assert = require('assert') + os = require('os'), fs = require('fs'), path = require('path'), TidyHtml = null, Settings = null; var npm = require("../../../../src/node_modules/npm/lib/npm.js"); +var nodeify = require('../../../../src/node_modules/nodeify'); describe('tidyHtml', function() { before(function(done) { @@ -16,6 +18,10 @@ describe('tidyHtml', function() { }); }); + function tidy(file, callback) { + return nodeify(TidyHtml.tidy(file), callback); + } + it('Tidies HTML', function(done) { // If the user hasn't configured Tidy, we skip this tests as it's required for this test if (!Settings.tidyHtml) { @@ -27,7 +33,7 @@ describe('tidyHtml', function() { var tmpFile = path.join(tmpDir, 'tmp_' + (Math.floor(Math.random() * 1000000)) + '.html') fs.writeFileSync(tmpFile, '

    a paragraph

  2. List without outer UL
    3. trailing closing p

    '); - TidyHtml.tidy(tmpFile, function(err){ + tidy(tmpFile, function(err){ assert.ok(!err); // Read the file again @@ -56,7 +62,7 @@ describe('tidyHtml', function() { this.skip(); } - TidyHtml.tidy('/some/none/existing/file.html', function(err) { + tidy('/some/none/existing/file.html', function(err) { assert.ok(err); return done(); }); From ccb49dcdc1e5fea2bbd61f110de85af3c6213600 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 14:38:56 +0000 Subject: [PATCH 53/58] padDiff.js: convert to Promises/async --- src/node/utils/padDiff.js | 268 +++++++++++++------------------------- 1 file changed, 93 insertions(+), 175 deletions(-) diff --git a/src/node/utils/padDiff.js b/src/node/utils/padDiff.js index a801fdc84..7cf29aba4 100644 --- a/src/node/utils/padDiff.js +++ b/src/node/utils/padDiff.js @@ -1,7 +1,5 @@ var Changeset = require("../../static/js/Changeset"); -var async = require("async"); var exportHtml = require('./ExportHtml'); -const thenify = require("thenify").withCallback; function PadDiff (pad, fromRev, toRev) { // check parameters @@ -79,79 +77,52 @@ PadDiff.prototype._isClearAuthorship = function(changeset) { return true; }; -PadDiff.prototype._createClearAuthorship = thenify(function(rev, callback) { - var self = this; - this._pad.getInternalRevisionAText(rev, function(err, atext) { - if (err) { - return callback(err); - } +PadDiff.prototype._createClearAuthorship = async function(rev) { - // build clearAuthorship changeset - var builder = Changeset.builder(atext.text.length); - builder.keepText(atext.text, [['author','']], self._pad.pool); - var changeset = builder.toString(); + let atext = await this._pad.getInternalRevisionAText(rev); - callback(null, changeset); - }); -}); + // build clearAuthorship changeset + var builder = Changeset.builder(atext.text.length); + builder.keepText(atext.text, [['author','']], this._pad.pool); + var changeset = builder.toString(); -PadDiff.prototype._createClearStartAtext = thenify(function(rev, callback) { - var self = this; + return changeset; +} + +PadDiff.prototype._createClearStartAtext = async function(rev) { // get the atext of this revision - this._pad.getInternalRevisionAText(rev, function(err, atext) { - if (err) { - return callback(err); - } + let atext = this._pad.getInternalRevisionAText(rev); - // create the clearAuthorship changeset - self._createClearAuthorship(rev, function(err, changeset) { - if (err) { - return callback(err); - } + // create the clearAuthorship changeset + let changeset = await this._createClearAuthorship(rev); - try { - // apply the clearAuthorship changeset - var newAText = Changeset.applyToAText(changeset, atext, self._pad.pool); - } catch(err) { - return callback(err) - } + // apply the clearAuthorship changeset + let newAText = Changeset.applyToAText(changeset, atext, this._pad.pool); - callback(null, newAText); - }); - }); -}); + return newAText; +} -PadDiff.prototype._getChangesetsInBulk = thenify(function(startRev, count, callback) { - var self = this; +PadDiff.prototype._getChangesetsInBulk = async function(startRev, count) { // find out which revisions we need - var revisions = []; - for (var i = startRev; i < (startRev + count) && i <= this._pad.head; i++) { + let revisions = []; + for (let i = startRev; i < (startRev + count) && i <= this._pad.head; i++) { revisions.push(i); } - var changesets = [], authors = []; - - // get all needed revisions - async.forEach(revisions, function(rev, callback) { - self._pad.getRevision(rev, function(err, revision) { - if (err) { - return callback(err); - } - - var arrayNum = rev-startRev; - + // get all needed revisions (in parallel) + let changesets = [], authors = []; + await Promise.all(revisions.map(rev => { + return this._pad.getRevision(rev).then(revision => { + let arrayNum = rev - startRev; changesets[arrayNum] = revision.changeset; authors[arrayNum] = revision.meta.author; - - callback(); }); - }, - function(err) { - callback(err, changesets, authors); - }); -}); + })); + + return { changesets, authors }; +} PadDiff.prototype._addAuthors = function(authors) { var self = this; @@ -164,144 +135,91 @@ PadDiff.prototype._addAuthors = function(authors) { }); }; -PadDiff.prototype._createDiffAtext = thenify(function(callback) { - var self = this; - var bulkSize = 100; +PadDiff.prototype._createDiffAtext = async function() { + + let bulkSize = 100; // get the cleaned startAText - self._createClearStartAtext(self._fromRev, function(err, atext) { - if (err) { return callback(err); } + let atext = await this._createClearStartAtext(this._fromRev); - var superChangeset = null; + let superChangeset = null; + let rev = this._fromRev + 1; - var rev = self._fromRev + 1; + for (let rev = this._fromRev + 1; rev <= this._toRev; rev += bulkSize) { - // async while loop - async.whilst( - // loop condition - function () { return rev <= self._toRev; }, + // get the bulk + let { changesets, authors } = await this._getChangesetsInBulk(rev, bulkSize); - // loop body - function (callback) { - // get the bulk - self._getChangesetsInBulk(rev,bulkSize,function(err, changesets, authors) { - var addedAuthors = []; + let addedAuthors = []; - // run trough all changesets - for (var i = 0; i < changesets.length && (rev + i) <= self._toRev; i++) { - var changeset = changesets[i]; + // run through all changesets + for (let i = 0; i < changesets.length && (rev + i) <= this._toRev; ++i) { + let changeset = changesets[i]; - // skip clearAuthorship Changesets - if (self._isClearAuthorship(changeset)) { - continue; - } - - changeset = self._extendChangesetWithAuthor(changeset, authors[i], self._pad.pool); - - // add this author to the authorarray - addedAuthors.push(authors[i]); - - // compose it with the superChangset - if (superChangeset === null) { - superChangeset = changeset; - } else { - superChangeset = Changeset.composeWithDeletions(superChangeset, changeset, self._pad.pool); - } - } - - // add the authors to the PadDiff authorArray - self._addAuthors(addedAuthors); - - // lets continue with the next bulk - rev += bulkSize; - callback(); - }); - }, - - // after the loop has ended - function (err) { - // if there are only clearAuthorship changesets, we don't get a superChangeset, so we can skip this step - if (superChangeset) { - var deletionChangeset = self._createDeletionChangeset(superChangeset,atext,self._pad.pool); - - try { - // apply the superChangeset, which includes all addings - atext = Changeset.applyToAText(superChangeset, atext, self._pad.pool); - // apply the deletionChangeset, which adds a deletions - atext = Changeset.applyToAText(deletionChangeset, atext, self._pad.pool); - } catch(err) { - return callback(err) - } - } - - callback(err, atext); + // skip clearAuthorship Changesets + if (this._isClearAuthorship(changeset)) { + continue; } - ); - }); -}); -PadDiff.prototype.getHtml = thenify(function(callback) { + changeset = this._extendChangesetWithAuthor(changeset, authors[i], this._pad.pool); + + // add this author to the authorarray + addedAuthors.push(authors[i]); + + // compose it with the superChangset + if (superChangeset === null) { + superChangeset = changeset; + } else { + superChangeset = Changeset.composeWithDeletions(superChangeset, changeset, this._pad.pool); + } + } + + // add the authors to the PadDiff authorArray + this._addAuthors(addedAuthors); + } + + // if there are only clearAuthorship changesets, we don't get a superChangeset, so we can skip this step + if (superChangeset) { + let deletionChangeset = this._createDeletionChangeset(superChangeset, atext, this._pad.pool); + + // apply the superChangeset, which includes all addings + atext = Changeset.applyToAText(superChangeset, atext, this._pad.pool); + + // apply the deletionChangeset, which adds a deletions + atext = Changeset.applyToAText(deletionChangeset, atext, this._pad.pool); + } + + return atext; +} + +PadDiff.prototype.getHtml = async function() { + // cache the html if (this._html != null) { - return callback(null, this._html); + return this._html; } - var self = this; - var atext, html, authorColors; + // get the diff atext + let atext = await this._createDiffAtext(); - async.series([ - // get the diff atext - function(callback) { - self._createDiffAtext(function(err, _atext) { - if (err) { - return callback(err); - } + // get the authorColor table + let authorColors = await this._pad.getAllAuthorColors(); - atext = _atext; - callback(); - }); - }, + // convert the atext to html + this._html = exportHtml.getHTMLFromAtext(this._pad, atext, authorColors); - // get the authorColor table - function(callback) { - self._pad.getAllAuthorColors(function(err, _authorColors) { - if (err) { - return callback(err); - } + return this._html; +} - authorColors = _authorColors; - callback(); - }); - }, - - // convert the atext to html - function(callback) { - html = exportHtml.getHTMLFromAtext(self._pad, atext, authorColors); - self._html = html; - callback(); - } - ], - function(err) { - callback(err, html); - }); -}); - -PadDiff.prototype.getAuthors = thenify(function(callback) { - var self = this; +PadDiff.prototype.getAuthors = async function() { // check if html was already produced, if not produce it, this generates the author array at the same time - if (self._html == null) { - self.getHtml(function(err) { - if (err) { - return callback(err); - } - - callback(null, self._authors); - }); - } else { - callback(null, self._authors); + if (this._html == null) { + await this.getHtml(); } -}); + + return self._authors; +} PadDiff.prototype._extendChangesetWithAuthor = function(changeset, author, apool) { // unpack From b1c5024bcf008b816b7b80ac5bdb8a857567726a Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 14:48:22 +0000 Subject: [PATCH 54/58] remove thenify use - no longer required --- src/node/db/AuthorManager.js | 1 - src/node/db/DB.js | 40 +++++++++++++++++----------------- src/node/db/SecurityManager.js | 1 - src/package.json | 1 - 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index 0c6e3f750..a17952248 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -21,7 +21,6 @@ var db = require("./DB"); var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; -const thenify = require("thenify").withCallback; exports.getColorPalette = function() { return [ diff --git a/src/node/db/DB.js b/src/node/db/DB.js index c462ac303..17cf3080a 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -22,11 +22,10 @@ var ueberDB = require("ueberdb2"); var settings = require("../utils/Settings"); var log4js = require('log4js'); -const thenify = require("thenify").withCallback; const util = require("util"); // set database settings -var db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB")); +let db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB")); /** * The UeberDB Object that provides the database functions @@ -37,24 +36,25 @@ exports.db = null; * Initalizes the database with the settings provided by the settings module * @param {Function} callback */ -exports.init = thenify(function (callback) { +exports.init = function() { // initalize the database async - db.init(function(err) { - if (err) { - // there was an error while initializing the database, output it and stop - console.error("ERROR: Problem while initalizing the database"); - console.error(err.stack ? err.stack : err); - process.exit(1); - } else { - // everything ok - exports.db = db; + return new Promise((resolve, reject) => { + db.init(function(err) { + if (err) { + // there was an error while initializing the database, output it and stop + console.error("ERROR: Problem while initalizing the database"); + console.error(err.stack ? err.stack : err); + process.exit(1); + } else { + // everything ok, set up Promise-based methods + ['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove', 'doShutdown'].forEach(fn => { + exports[fn] = util.promisify(db[fn].bind(db)); + }); - // set up Promise-based methods - ['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove', 'doShutdown'].forEach(fn => { - exports[fn] = util.promisify(db[fn].bind(db)); - }); - - callback(null); - } + // exposed for those callers that need the underlying raw API + exports.db = db; + resolve(); + } + }); }); -}); +} diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index dcdbe6a62..e50a0b9c2 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -25,7 +25,6 @@ var sessionManager = require("./SessionManager"); var settings = require("../utils/Settings"); var log4js = require('log4js'); var authLogger = log4js.getLogger("auth"); -const thenify = require("thenify").withCallback; /** * This function controlls the access to a pad, it checks if the user can access a pad. diff --git a/src/package.json b/src/package.json index 11259e76d..2b5eb5c18 100644 --- a/src/package.json +++ b/src/package.json @@ -58,7 +58,6 @@ "slide": "1.1.6", "socket.io": "2.1.1", "swagger-node-express": "2.1.3", - "thenify": "^3.3.0", "tinycon": "0.0.1", "ueberdb2": "0.4.0", "uglify-js": "2.6.2", From 07ae44ddf4f2fd77ce04ab9a9e3cb405fe032806 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Thu, 31 Jan 2019 15:46:25 +0000 Subject: [PATCH 55/58] PadMessageHandler.js: cope better with session disconnects --- src/node/handler/PadMessageHandler.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index b41af86b9..cccb9e8a2 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -268,6 +268,12 @@ exports.handleMessage = async function(client, message) // FIXME: Use a hook instead // FIXME: Allow to override readwrite access with readonly + // the session may have been dropped during earlier processing + if (!sessioninfos[client.id]) { + messageLogger.warn("Dropping message from a connection that has gone away.") + return; + } + // Simulate using the load testing tool if (!sessioninfos[client.id].auth) { console.error("Auth was never applied to a session. If you are using the stress-test tool then restart Etherpad and the Stress test tool.") From e7c2fad7b0418eb891506e6890f5df380edda7ea Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 1 Feb 2019 00:07:06 +0000 Subject: [PATCH 56/58] convert some async loops into parallel loops If you use `await` inside a loop it makes the loop inherently serial. If you omit the `await` however, the tasks will all start but the loop will finish while the tasks are still being scheduled. So, to make a set of tasks run in parallel but then have the code block after the loop once all the tasks have been completed you have to get an array of Promises (one for each iteration) and then use `Promise.all()` to wait for those promises to be resolved. Using `Array#map` is a convenient way to go from an array of inputs to the require array of Promises. --- src/node/db/GroupManager.js | 19 +++---- src/node/handler/PadMessageHandler.js | 80 ++++++++++++++------------- 2 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/node/db/GroupManager.js b/src/node/db/GroupManager.js index 238c423b1..5df034ef6 100644 --- a/src/node/db/GroupManager.js +++ b/src/node/db/GroupManager.js @@ -43,20 +43,19 @@ exports.deleteGroup = async function(groupID) throw new customError("groupID does not exist", "apierror"); } - // iterate through all pads of this group and delete them - for (let padID in group.pads) { - let pad = await padManager.getPad(padID); - await pad.remove(); - } + // iterate through all pads of this group and delete them (in parallel) + await Promise.all(Object.keys(group.pads).map(padID => { + return padManager.getPad(padID).then(pad => pad.remove()); + })); // iterate through group2sessions and delete all sessions let group2sessions = await db.get("group2sessions:" + groupID); - let sessions = group2sessions ? group2sessions.sessionsIDs : []; + let sessions = group2sessions ? group2sessions.sessionsIDs : {}; - // loop through all sessions and delete them - for (let session in sessions) { - await sessionManager.deleteSession(session); - } + // loop through all sessions and delete them (in parallel) + await Promise.all(Object.keys(sessions).map(session => { + return sessionManager.deleteSession(session); + })); // remove group and group2sessions entry await db.remove("group2sessions:" + groupID); diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index cccb9e8a2..1c7b44cfc 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -722,6 +722,7 @@ exports.updatePadClients = async function(pad) // since all clients usually get the same set of changesets, store them in local cache // to remove unnecessary roundtrip to the datalayer + // NB: note below possibly now accommodated via the change to promises/async // TODO: in REAL world, if we're working without datalayer cache, all requests to revisions will be fired // BEFORE first result will be landed to our cache object. The solution is to replace parallel processing // via async.forEach with sequential for() loop. There is no real benefits of running this in parallel, @@ -928,15 +929,16 @@ async function handleClientReady(client, message) // get timestamp of latest revision needed for timeslider let currentTime = await pad.getRevisionDate(pad.getHeadRevisionNumber()); - // get all author data out of the database - for (let authorId of authors) { - try { - let author = await authorManager.getAuthor(authorId); - historicalAuthorData[authorId] = { name: author.name, colorId: author.colorId }; // Filter author attribs (e.g. don't send author's pads to all clients) - } catch (err) { - messageLogger.error("There is no author for authorId:", authorId); - } - } + // get all author data out of the database (in parallel) + await Promise.all(authors.map(authorId => { + return authorManager.getAuthor(authorId).then(author => { + if (!author) { + messageLogger.error("There is no author for authorId:", authorId); + } else { + historicalAuthorData[authorId] = { name: author.name, colorId: author.colorId }; // Filter author attribs (e.g. don't send author's pads to all clients) + } + }); + })); // glue the clientVars together, send them and tell the other clients that a new one is there @@ -1162,45 +1164,47 @@ async function handleClientReady(client, message) // notify all existing users about new user client.broadcast.to(padIds.padId).json.send(messageToTheOtherUsers); - // Get sessions for this pad + // Get sessions for this pad and update them (in parallel) roomClients = _getRoomClients(pad.id); - for (let roomClient of roomClients) { + await Promise.all(_getRoomClients(pad.id).map(async roomClient => { // Jump over, if this session is the connection session if (roomClient.id == client.id) { - continue; + return; } // Since sessioninfos might change while being enumerated, check if the // sessionID is still assigned to a valid session if (sessioninfos[roomClient.id] === undefined) { - continue; + return; } - let author = sessioninfos[roomClient.id].author; - // get the authorname & colorId + let author = sessioninfos[roomClient.id].author; + let cached = historicalAuthorData[author]; // reuse previously created cache of author's data - let authorInfo = historicalAuthorData[author] || await authorManager.getAuthor(author); + let p = cached ? Promise.resolve(cached) : authorManager.getAuthor(author); - // Send the new User a Notification about this other user - let msg = { - "type": "COLLABROOM", - "data": { - type: "USER_NEWINFO", - userInfo: { - "ip": "127.0.0.1", - "colorId": authorInfo.colorId, - "name": authorInfo.name, - "userAgent": "Anonymous", - "userId": author + return p.then(authorInfo => { + // Send the new User a Notification about this other user + let msg = { + "type": "COLLABROOM", + "data": { + type: "USER_NEWINFO", + userInfo: { + "ip": "127.0.0.1", + "colorId": authorInfo.colorId, + "name": authorInfo.name, + "userAgent": "Anonymous", + "userId": author + } } - } - }; + }; - client.json.send(msg); - } + client.json.send(msg); + }); + })); } } @@ -1448,16 +1452,16 @@ exports.padUsers = async function(padID) { let padUsers = []; let roomClients = _getRoomClients(padID); - for (let i = 0, n = roomClients.length; i < n; ++i) { - let roomClient = roomClients[i]; - + // iterate over all clients (in parallel) + await Promise.all(roomClients.map(async roomClient => { let s = sessioninfos[roomClient.id]; if (s) { - let author = await authorManager.getAuthor(s.author); - author.id = s.author; - padUsers.push(author); + return authorManager.getAuthor(s.author).then(author => { + author.id = s.author; + padUsers.push(author); + }); } - } + })); return { padUsers }; } From 769933786ceabf6aac87f788fef5560270e4db93 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Fri, 1 Feb 2019 09:57:50 +0000 Subject: [PATCH 57/58] allow some operations to proceed in parallel some code chunks previously used `async.parallel` but if you use `await` that forces them to be run serially. Instead, you can initiate the operation (getting a Promise) and then _later_ `await` the result of that Promise. --- src/node/db/Pad.js | 14 ++++++++++---- src/node/db/SecurityManager.js | 26 +++++++++++++++++--------- src/node/handler/PadMessageHandler.js | 18 ++++++++++-------- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index fb36bdf58..6c97fee8d 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -169,9 +169,8 @@ Pad.prototype.getInternalRevisionAText = async function getInternalRevisionAText // get all needed data out of the database - // get the atext of the key revision - let _atext = await db.getSub("pad:" + this.id + ":revs:" + keyRev, ["meta", "atext"]); - let atext = Changeset.cloneAText(_atext); + // start to get the atext of the key revision + let p_atext = db.getSub("pad:" + this.id + ":revs:" + keyRev, ["meta", "atext"]); // get all needed changesets let changesets = []; @@ -181,6 +180,10 @@ Pad.prototype.getInternalRevisionAText = async function getInternalRevisionAText }); })); + // we should have the atext by now + let atext = await p_atext; + atext = Changeset.cloneAText(atext); + // apply all changesets to the key changeset let apool = this.apool(); for (let curRev = keyRev; curRev < targetRev; ) { @@ -455,7 +458,10 @@ Pad.prototype.remove = async function remove() { // kick everyone from this pad padMessageHandler.kickSessionsFromPad(padID); - // delete all relations + // delete all relations - the original code used async.parallel but + // none of the operations except getting the group depended on callbacks + // so the database operations here are just started and then left to + // run to completion // is it a group pad? -> delete the entry of this pad in the group if (padID.indexOf("$") >= 0) { diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index e50a0b9c2..23af82836 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -49,11 +49,11 @@ exports.checkAccess = async function(padID, sessionCookie, token, password) return deny; } - // get author for this token - let tokenAuthor = await authorManager.getAuthor4Token(token); + // start to get author for this token + let p_tokenAuthor = authorManager.getAuthor4Token(token); - // check if pad exists - let padExists = await padManager.doesPadExist(padID); + // start to check if pad exists + let p_padExists = padManager.doesPadExist(padID); if (settings.requireSession) { // a valid session is required (api-only mode) @@ -67,11 +67,14 @@ exports.checkAccess = async function(padID, sessionCookie, token, password) // it's not a group pad, means we can grant access // assume user has access - let statusObject = { accessStatus: "grant", authorID: tokenAuthor }; + let authorID = await p_tokenAuthor; + let statusObject = { accessStatus: "grant", authorID }; if (settings.editOnly) { // user can't create pads + let padExists = await p_padExists; + if (!padExists) { // pad doesn't exist - user can't have access statusObject.accessStatus = "deny"; @@ -96,10 +99,13 @@ exports.checkAccess = async function(padID, sessionCookie, token, password) let sessionIDs = sessionCookie.split(','); // was previously iterated in parallel using async.forEach - for (let sessionID of sessionIDs) { - try { - let sessionInfo = await sessionManager.getSessionInfo(sessionID); + let sessionInfos = await Promise.all(sessionIDs.map(sessionID => { + return sessionManager.getSessionInfo(sessionID); + })); + // seperated out the iteration of sessioninfos from the (parallel) fetches from the DB + for (let sessionInfo of sessionInfos) { + try { // is it for this group? if (sessionInfo.groupID != groupID) { authLogger.debug("Auth failed: wrong group"); @@ -128,6 +134,8 @@ exports.checkAccess = async function(padID, sessionCookie, token, password) } } + let padExists = await p_padExists; + if (padExists) { let pad = await padManager.getPad(padID); @@ -205,7 +213,7 @@ exports.checkAccess = async function(padID, sessionCookie, token, password) if (!validSession && padExists) { // there is no valid session avaiable AND pad exists - let authorID = tokenAuthor; + let authorID = await p_tokenAuthor; let grant = Object.freeze({ accessStatus: "grant", authorID }); if (isPublic && !isPasswordProtected) { diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index 1c7b44cfc..18b08af5b 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -890,8 +890,6 @@ async function handleClientReady(client, message) return; } - var historicalAuthorData = {}; - hooks.callAll("clientReady", message); // Get ro/rw id:s @@ -915,12 +913,12 @@ async function handleClientReady(client, message) let author = statusObject.authorID; - // get all authordata of this new user, and load the pad-object from the database + // get all authordata of this new user let value = await authorManager.getAuthor(author); let authorColorId = value.colorId; let authorName = value.name; - // get pad + // load the pad-object from the database let pad = await padManager.getPad(padIds.padId); // these db requests all need the pad object (timestamp of latest revision, author data) @@ -930,6 +928,7 @@ async function handleClientReady(client, message) let currentTime = await pad.getRevisionDate(pad.getHeadRevisionNumber()); // get all author data out of the database (in parallel) + let historicalAuthorData = {}; await Promise.all(authors.map(authorId => { return authorManager.getAuthor(authorId).then(author => { if (!author) { @@ -1010,12 +1009,15 @@ async function handleClientReady(client, message) changesets[r] = {}; } - // get changesets, author and timestamp needed for pending revisions + // get changesets, author and timestamp needed for pending revisions (in parallel) + let promises = []; for (let revNum of revisionsNeeded) { - changesets[revNum]['changeset'] = await pad.getRevisionChangeset(revNum); - changesets[revNum]['author'] = await pad.getRevisionAuthor(revNum); - changesets[revNum]['timestamp'] = await pad.getRevisionDate(revNum); + let cs = changesets[revNum]; + promises.push( pad.getRevisionChangeset(revNum).then(result => cs.changeset = result )); + promises.push( pad.getRevisionAuthor(revNum).then(result => cs.author = result )); + promises.push( pad.getRevisionDate(revNum).then(result => cs.timestamp = result )); } + await Promise.all(promises); // return pending changesets for (let r of revisionsNeeded) { From ac7663c3372853f1c79d0bf783b3d929ad6a4828 Mon Sep 17 00:00:00 2001 From: Ray Bellis Date: Tue, 5 Mar 2019 10:46:57 +0000 Subject: [PATCH 58/58] db/DB.js: prevent DB layer from returning undefined ueberDB2 can return either undefined or null for a missing key, depending on which DB driver is used. This patch changes the promise version of the API so that it will always return null. --- src/node/db/DB.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/node/db/DB.js b/src/node/db/DB.js index 17cf3080a..573563665 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -51,6 +51,19 @@ exports.init = function() { exports[fn] = util.promisify(db[fn].bind(db)); }); + // set up wrappers for get and getSub that can't return "undefined" + let get = exports.get; + exports.get = async function(key) { + let result = await get(key); + return (result === undefined) ? null : result; + }; + + let getSub = exports.getSub; + exports.getSub = async function(key, sub) { + let result = await getSub(key, sub); + return (result === undefined) ? null : result; + }; + // exposed for those callers that need the underlying raw API exports.db = db; resolve();