mirrors/etherpad-lite
1
0
Fork 0
mirror of https://github.com/ether/etherpad-lite.git synced 2025-04-22 00:16:15 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 26

ImportEtherpad: Limit in-flight DB queries

Browse source
This commit is contained in:
Richard Hansen 2021-11-26 01:48:42 -05:00
parent 5b3575acf0
commit 77bcb507b3
1 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/node/utils/ImportEtherpad.js
View file

@ -18,6 +18,7 @@
const AttributePool = require('../../static/js/AttributePool'); const AttributePool = require('../../static/js/AttributePool');
const {Pad} = require('../db/Pad'); const {Pad} = require('../db/Pad');
const async = require('async');
const authorManager = require('../db/AuthorManager'); const authorManager = require('../db/AuthorManager');
const db = require('../db/DB'); const db = require('../db/DB');
const hooks = require('../../static/js/pluginfw/hooks'); const hooks = require('../../static/js/pluginfw/hooks');
@ -47,12 +48,16 @@ exports.setPadRaw = async (padId, r) => {
if (originalPadId !== padId) throw new Error('unexpected pad ID in record'); if (originalPadId !== padId) throw new Error('unexpected pad ID in record');
}; };
// Limit the number of in-flight database queries so that the queries do not time out when
// importing really large files.
const q = async.queue(async (task) => await task(), 100);
// First validate and transform values. Do not commit any records to the database yet in case // First validate and transform values. Do not commit any records to the database yet in case
// there is a problem with the data. // there is a problem with the data.
const dbRecords = new Map(); const dbRecords = new Map();
const existingAuthors = new Set(); const existingAuthors = new Set();
await Promise.all(Object.entries(records).map(async ([key, value]) => { await Promise.all(Object.entries(records).map(([key, value]) => q.pushAsync(async () => {
if (!value) { if (!value) {
return; return;
} }
@ -91,7 +96,7 @@ exports.setPadRaw = async (padId, r) => {
return; return;
} }
dbRecords.set(key, value); dbRecords.set(key, value);
})); })));
const pad = new Pad(padId, { const pad = new Pad(padId, {
// Only fetchers are needed to check the pad's integrity. // Only fetchers are needed to check the pad's integrity.
@ -109,7 +114,7 @@ exports.setPadRaw = async (padId, r) => {
await pad.check(); await pad.check();
await Promise.all([ await Promise.all([
...[...dbRecords].map(async ([k, v]) => await db.set(k, v)), ...[...dbRecords].map(([k, v]) => q.pushAsync(() => db.set(k, v))),
...[...existingAuthors].map(async (authorId) => await authorManager.addPad(authorId, padId)), ...[...existingAuthors].map((a) => q.pushAsync(() => authorManager.addPad(a, padId))),
]); ]);
}; };