libre-service.eu/pad.libre-service.eu-etherpad
libre-service.eu/pad.libre-service.eu-etherpad
1
0
Fork 0
mirror of https://github.com/ether/etherpad-lite.git synced 2025-01-31 19:02:59 +01:00
Code Issues Projects Releases Wiki Activity

ImportEtherpad: Limit in-flight DB queries

Browse source
This commit is contained in:
Richard Hansen 2021-11-26 01:48:42 -05:00
parent 5b3575acf0
commit 77bcb507b3
1 changed files with 9 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/node/utils/ImportEtherpad.js
View file

@ -18,6 +18,7 @@
const AttributePool = require('../../static/js/AttributePool');
const {Pad} = require('../db/Pad');
const async = require('async');
const authorManager = require('../db/AuthorManager');
const db = require('../db/DB');
const hooks = require('../../static/js/pluginfw/hooks');
@ -47,12 +48,16 @@ exports.setPadRaw = async (padId, r) => {
if (originalPadId !== padId) throw new Error('unexpected pad ID in record');
};
// Limit the number of in-flight database queries so that the queries do not time out when
// importing really large files.
const q = async.queue(async (task) => await task(), 100);
// First validate and transform values. Do not commit any records to the database yet in case
// there is a problem with the data.
const dbRecords = new Map();
const existingAuthors = new Set();
await Promise.all(Object.entries(records).map(async ([key, value]) => {
await Promise.all(Object.entries(records).map(([key, value]) => q.pushAsync(async () => {
if (!value) {
return;
}
@ -91,7 +96,7 @@ exports.setPadRaw = async (padId, r) => {
return;
}
dbRecords.set(key, value);
}));
})));
const pad = new Pad(padId, {
// Only fetchers are needed to check the pad's integrity.
@ -109,7 +114,7 @@ exports.setPadRaw = async (padId, r) => {
await pad.check();
await Promise.all([
...[...dbRecords].map(async ([k, v]) => await db.set(k, v)),
...[...existingAuthors].map(async (authorId) => await authorManager.addPad(authorId, padId)),
...[...dbRecords].map(([k, v]) => q.pushAsync(() => db.set(k, v))),
...[...existingAuthors].map((a) => q.pushAsync(() => authorManager.addPad(a, padId))),
]);
};