From 77bcb507b30e762e9375b0511b3763e0162aae53 Mon Sep 17 00:00:00 2001
From: Richard Hansen <rhansen@rhansen.org>
Date: Fri, 26 Nov 2021 01:48:42 -0500
Subject: [PATCH] ImportEtherpad: Limit in-flight DB queries

---
 src/node/utils/ImportEtherpad.js | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/node/utils/ImportEtherpad.js b/src/node/utils/ImportEtherpad.js
index e129a6995..60a3b8492 100644
--- a/src/node/utils/ImportEtherpad.js
+++ b/src/node/utils/ImportEtherpad.js
@@ -18,6 +18,7 @@
 
 const AttributePool = require('../../static/js/AttributePool');
 const {Pad} = require('../db/Pad');
+const async = require('async');
 const authorManager = require('../db/AuthorManager');
 const db = require('../db/DB');
 const hooks = require('../../static/js/pluginfw/hooks');
@@ -47,12 +48,16 @@ exports.setPadRaw = async (padId, r) => {
     if (originalPadId !== padId) throw new Error('unexpected pad ID in record');
   };
 
+  // Limit the number of in-flight database queries so that the queries do not time out when
+  // importing really large files.
+  const q = async.queue(async (task) => await task(), 100);
+
   // First validate and transform values. Do not commit any records to the database yet in case
   // there is a problem with the data.
 
   const dbRecords = new Map();
   const existingAuthors = new Set();
-  await Promise.all(Object.entries(records).map(async ([key, value]) => {
+  await Promise.all(Object.entries(records).map(([key, value]) => q.pushAsync(async () => {
     if (!value) {
       return;
     }
@@ -91,7 +96,7 @@ exports.setPadRaw = async (padId, r) => {
       return;
     }
     dbRecords.set(key, value);
-  }));
+  })));
 
   const pad = new Pad(padId, {
     // Only fetchers are needed to check the pad's integrity.
@@ -109,7 +114,7 @@ exports.setPadRaw = async (padId, r) => {
   await pad.check();
 
   await Promise.all([
-    ...[...dbRecords].map(async ([k, v]) => await db.set(k, v)),
-    ...[...existingAuthors].map(async (authorId) => await authorManager.addPad(authorId, padId)),
+    ...[...dbRecords].map(([k, v]) => q.pushAsync(() => db.set(k, v))),
+    ...[...existingAuthors].map((a) => q.pushAsync(() => authorManager.addPad(a, padId))),
   ]);
 };