mirror of
https://github.com/ether/etherpad-lite.git
synced 2025-01-19 22:23:33 +01:00
import/export: conversion to Promises/async
NB1: needs additional review and testing - no abiword available on my test bed
NB2: in ImportHandler.js, directly delete the file, and handle the eventual
error later: checking before for existence is prone to race conditions,
and does not handle any errors anyway.
This commit is contained in:
Ray Bellis
parent
5192a0c498
commit
62345ac8f7
8 changed files with 379 additions and 570 deletions
|
|
@ -287,7 +287,7 @@ exports.setHTML = async function(padID, html)
|
|||
|
||||
// add a new changeset with the new html to the pad
|
||||
try {
|
||||
await importHtml.setPadHTML(pad, cleanText(html));
|
||||
importHtml.setPadHTML(pad, cleanText(html));
|
||||
} catch (e) {
|
||||
throw new customError("HTML is malformed", "apierror");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,18 +19,20 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
var ERR = require("async-stacktrace");
|
||||
var exporthtml = require("../utils/ExportHtml");
|
||||
var exporttxt = require("../utils/ExportTxt");
|
||||
var exportEtherpad = require("../utils/ExportEtherpad");
|
||||
var async = require("async");
|
||||
var fs = require("fs");
|
||||
var settings = require('../utils/Settings');
|
||||
var os = require('os');
|
||||
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
|
||||
var TidyHtml = require('../utils/TidyHtml');
|
||||
const util = require("util");
|
||||
|
||||
var convertor = null;
|
||||
const fsp_writeFile = util.promisify(fs.writeFile);
|
||||
const fsp_unlink = util.promisify(fs.unlink);
|
||||
|
||||
let convertor = null;
|
||||
|
||||
// load abiword only if it is enabled
|
||||
if (settings.abiword != null) {
|
||||
|
|
@ -47,122 +49,92 @@ const tempDirectory = os.tmpdir();
|
|||
/**
|
||||
* do a requested export
|
||||
*/
|
||||
exports.doExport = function(req, res, padId, type)
|
||||
async function doExport(req, res, padId, type)
|
||||
{
|
||||
var fileName = padId;
|
||||
|
||||
// allow fileName to be overwritten by a hook, the type type is kept static for security reasons
|
||||
hooks.aCallFirst("exportFileName", padId,
|
||||
function(err, hookFileName){
|
||||
// if fileName is set then set it to the padId, note that fileName is returned as an array.
|
||||
if (hookFileName.length) {
|
||||
fileName = hookFileName;
|
||||
}
|
||||
let hookFileName = await hooks.aCallFirst("exportFileName", padId);
|
||||
|
||||
// tell the browser that this is a downloadable file
|
||||
res.attachment(fileName + "." + type);
|
||||
// if fileName is set then set it to the padId, note that fileName is returned as an array.
|
||||
if (hookFileName.length) {
|
||||
fileName = hookFileName;
|
||||
}
|
||||
|
||||
// if this is a plain text export, we can do this directly
|
||||
// We have to over engineer this because tabs are stored as attributes and not plain text
|
||||
if (type == "etherpad") {
|
||||
exportEtherpad.getPadRaw(padId, function(err, pad) {
|
||||
if (!err) {
|
||||
res.send(pad);
|
||||
// return;
|
||||
}
|
||||
});
|
||||
} else if (type == "txt") {
|
||||
exporttxt.getPadTXTDocument(padId, req.params.rev, function(err, txt) {
|
||||
if (!err) {
|
||||
res.send(txt);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
var html;
|
||||
var randNum;
|
||||
var srcFile, destFile;
|
||||
// tell the browser that this is a downloadable file
|
||||
res.attachment(fileName + "." + type);
|
||||
|
||||
async.series([
|
||||
// render the html document
|
||||
function(callback) {
|
||||
exporthtml.getPadHTMLDocument(padId, req.params.rev, function(err, _html) {
|
||||
if (ERR(err, callback)) return;
|
||||
html = _html;
|
||||
callback();
|
||||
});
|
||||
},
|
||||
// if this is a plain text export, we can do this directly
|
||||
// We have to over engineer this because tabs are stored as attributes and not plain text
|
||||
if (type === "etherpad") {
|
||||
let pad = await exportEtherpad.getPadRaw(padId);
|
||||
res.send(pad);
|
||||
} else if (type === "txt") {
|
||||
let txt = await exporttxt.getPadTXTDocument(padId, req.params.rev);
|
||||
res.send(txt);
|
||||
} else {
|
||||
// render the html document
|
||||
let html = await exporthtml.getPadHTMLDocument(padId, req.params.rev);
|
||||
|
||||
// decide what to do with the html export
|
||||
function(callback) {
|
||||
// if this is a html export, we can send this from here directly
|
||||
if (type == "html") {
|
||||
// do any final changes the plugin might want to make
|
||||
hooks.aCallFirst("exportHTMLSend", html, function(err, newHTML) {
|
||||
if (newHTML.length) html = newHTML;
|
||||
res.send(html);
|
||||
callback("stop");
|
||||
});
|
||||
} else {
|
||||
// write the html export to a file
|
||||
randNum = Math.floor(Math.random()*0xFFFFFFFF);
|
||||
srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html";
|
||||
fs.writeFile(srcFile, html, callback);
|
||||
}
|
||||
},
|
||||
// decide what to do with the html export
|
||||
|
||||
// Tidy up the exported HTML
|
||||
function(callback) {
|
||||
// ensure html can be collected by the garbage collector
|
||||
html = null;
|
||||
|
||||
TidyHtml.tidy(srcFile, callback);
|
||||
},
|
||||
|
||||
// send the convert job to the convertor (abiword, libreoffice, ..)
|
||||
function(callback) {
|
||||
destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type;
|
||||
|
||||
// Allow plugins to overwrite the convert in export process
|
||||
hooks.aCallAll("exportConvert", { srcFile: srcFile, destFile: destFile, req: req, res: res }, function(err, result) {
|
||||
if (!err && result.length > 0) {
|
||||
// console.log("export handled by plugin", destFile);
|
||||
handledByPlugin = true;
|
||||
callback();
|
||||
} else {
|
||||
convertor.convertFile(srcFile, destFile, type, callback);
|
||||
}
|
||||
});
|
||||
|
||||
},
|
||||
|
||||
// send the file
|
||||
function(callback) {
|
||||
res.sendFile(destFile, null, callback);
|
||||
},
|
||||
|
||||
// clean up temporary files
|
||||
function(callback) {
|
||||
async.parallel([
|
||||
function(callback) {
|
||||
fs.unlink(srcFile, callback);
|
||||
},
|
||||
function(callback) {
|
||||
// 100ms delay to accommodate for slow windows fs
|
||||
if (os.type().indexOf("Windows") > -1) {
|
||||
setTimeout(function() {
|
||||
fs.unlink(destFile, callback);
|
||||
}, 100);
|
||||
} else {
|
||||
fs.unlink(destFile, callback);
|
||||
}
|
||||
}
|
||||
], callback);
|
||||
}
|
||||
],
|
||||
function(err) {
|
||||
if (err && err != "stop") ERR(err);
|
||||
})
|
||||
}
|
||||
// if this is a html export, we can send this from here directly
|
||||
if (type === "html") {
|
||||
// do any final changes the plugin might want to make
|
||||
let newHTML = await hooks.aCallFirst("exportHTMLSend", html);
|
||||
if (newHTML.length) html = newHTML;
|
||||
res.send(html);
|
||||
throw "stop";
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
// else write the html export to a file
|
||||
let randNum = Math.floor(Math.random()*0xFFFFFFFF);
|
||||
let srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html";
|
||||
await fsp_writeFile(srcFile, html);
|
||||
|
||||
// Tidy up the exported HTML
|
||||
// ensure html can be collected by the garbage collector
|
||||
html = null;
|
||||
await TidyHtml.tidy(srcFile);
|
||||
|
||||
// send the convert job to the convertor (abiword, libreoffice, ..)
|
||||
let destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type;
|
||||
|
||||
// Allow plugins to overwrite the convert in export process
|
||||
let result = await hooks.aCallAll("exportConvert", { srcFile, destFile, req, res });
|
||||
if (result.length > 0) {
|
||||
// console.log("export handled by plugin", destFile);
|
||||
handledByPlugin = true;
|
||||
} else {
|
||||
// @TODO no Promise interface for convertors (yet)
|
||||
await new Promise((resolve, reject) => {
|
||||
convertor.convertFile(srcFile, destFile, type, function(err) {
|
||||
err ? reject("convertFailed") : resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// send the file
|
||||
let sendFile = util.promisify(res.sendFile);
|
||||
await res.sendFile(destFile, null);
|
||||
|
||||
// clean up temporary files
|
||||
await fsp_unlink(srcFile);
|
||||
|
||||
// 100ms delay to accommodate for slow windows fs
|
||||
if (os.type().indexOf("Windows") > -1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
await fsp_unlink(destFile);
|
||||
}
|
||||
}
|
||||
|
||||
exports.doExport = function(req, res, padId, type)
|
||||
{
|
||||
doExport(req, res, padId, type).catch(err => {
|
||||
if (err !== "stop") {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,10 +20,8 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
var ERR = require("async-stacktrace")
|
||||
, padManager = require("../db/PadManager")
|
||||
var padManager = require("../db/PadManager")
|
||||
, padMessageHandler = require("./PadMessageHandler")
|
||||
, async = require("async")
|
||||
, fs = require("fs")
|
||||
, path = require("path")
|
||||
, settings = require('../utils/Settings')
|
||||
|
|
@ -32,10 +30,16 @@ var ERR = require("async-stacktrace")
|
|||
, importHtml = require("../utils/ImportHtml")
|
||||
, importEtherpad = require("../utils/ImportEtherpad")
|
||||
, log4js = require("log4js")
|
||||
, hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js");
|
||||
, hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js")
|
||||
, util = require("util");
|
||||
|
||||
var convertor = null;
|
||||
var exportExtension = "htm";
|
||||
let fsp_exists = util.promisify(fs.exists);
|
||||
let fsp_rename = util.promisify(fs.rename);
|
||||
let fsp_readFile = util.promisify(fs.readFile);
|
||||
let fsp_unlink = util.promisify(fs.unlink)
|
||||
|
||||
let convertor = null;
|
||||
let exportExtension = "htm";
|
||||
|
||||
// load abiword only if it is enabled and if soffice is disabled
|
||||
if (settings.abiword != null && settings.soffice === null) {
|
||||
|
|
@ -53,292 +57,213 @@ const tmpDirectory = os.tmpdir();
|
|||
/**
|
||||
* do a requested import
|
||||
*/
|
||||
exports.doImport = function(req, res, padId)
|
||||
async function doImport(req, res, padId)
|
||||
{
|
||||
var apiLogger = log4js.getLogger("ImportHandler");
|
||||
|
||||
// pipe to a file
|
||||
// convert file to html via abiword or soffice
|
||||
// set html in the pad
|
||||
|
||||
var srcFile, destFile
|
||||
, pad
|
||||
, text
|
||||
, importHandledByPlugin
|
||||
, directDatabaseAccess
|
||||
, useConvertor;
|
||||
|
||||
var randNum = Math.floor(Math.random()*0xFFFFFFFF);
|
||||
|
||||
// setting flag for whether to use convertor or not
|
||||
useConvertor = (convertor != null);
|
||||
let useConvertor = (convertor != null);
|
||||
|
||||
async.series([
|
||||
// save the uploaded file to /tmp
|
||||
function(callback) {
|
||||
var form = new formidable.IncomingForm();
|
||||
form.keepExtensions = true;
|
||||
form.uploadDir = tmpDirectory;
|
||||
let form = new formidable.IncomingForm();
|
||||
form.keepExtensions = true;
|
||||
form.uploadDir = tmpDirectory;
|
||||
|
||||
form.parse(req, function(err, fields, files) {
|
||||
if (err || files.file === undefined) {
|
||||
// the upload failed, stop at this point
|
||||
if (err) {
|
||||
console.warn("Uploading Error: " + err.stack);
|
||||
}
|
||||
callback("uploadFailed");
|
||||
|
||||
return;
|
||||
// locally wrapped Promise, since form.parse requires a callback
|
||||
let srcFile = await new Promise((resolve, reject) => {
|
||||
form.parse(req, function(err, fields, files) {
|
||||
if (err || files.file === undefined) {
|
||||
// the upload failed, stop at this point
|
||||
if (err) {
|
||||
console.warn("Uploading Error: " + err.stack);
|
||||
}
|
||||
|
||||
// everything ok, continue
|
||||
// save the path of the uploaded file
|
||||
srcFile = files.file.path;
|
||||
callback();
|
||||
});
|
||||
},
|
||||
|
||||
// ensure this is a file ending we know, else we change the file ending to .txt
|
||||
// this allows us to accept source code files like .c or .java
|
||||
function(callback) {
|
||||
var fileEnding = path.extname(srcFile).toLowerCase()
|
||||
, knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"]
|
||||
, fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1);
|
||||
|
||||
// if the file ending is known, continue as normal
|
||||
if (fileEndingKnown) {
|
||||
callback();
|
||||
|
||||
return;
|
||||
reject("uploadFailed");
|
||||
}
|
||||
resolve(files.file.path);
|
||||
});
|
||||
});
|
||||
|
||||
// ensure this is a file ending we know, else we change the file ending to .txt
|
||||
// this allows us to accept source code files like .c or .java
|
||||
let fileEnding = path.extname(srcFile).toLowerCase()
|
||||
, knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"]
|
||||
, fileEndingUnknown = (knownFileEndings.indexOf(fileEnding) < 0);
|
||||
|
||||
if (fileEndingUnknown) {
|
||||
// the file ending is not known
|
||||
|
||||
if (settings.allowUnknownFileEnds === true) {
|
||||
// we need to rename this file with a .txt ending
|
||||
if (settings.allowUnknownFileEnds === true) {
|
||||
var oldSrcFile = srcFile;
|
||||
srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt");
|
||||
fs.rename(oldSrcFile, srcFile, callback);
|
||||
} else {
|
||||
console.warn("Not allowing unknown file type to be imported", fileEnding);
|
||||
callback("uploadFailed");
|
||||
}
|
||||
},
|
||||
let oldSrcFile = srcFile;
|
||||
|
||||
function(callback) {
|
||||
destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension);
|
||||
srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt");
|
||||
await fs.rename(oldSrcFile, srcFile);
|
||||
} else {
|
||||
console.warn("Not allowing unknown file type to be imported", fileEnding);
|
||||
throw "uploadFailed";
|
||||
}
|
||||
}
|
||||
|
||||
// Logic for allowing external Import Plugins
|
||||
hooks.aCallAll("import", { srcFile: srcFile, destFile: destFile }, function(err, result) {
|
||||
if (ERR(err, callback)) return callback();
|
||||
let destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension);
|
||||
|
||||
if (result.length > 0) { // This feels hacky and wrong..
|
||||
importHandledByPlugin = true;
|
||||
}
|
||||
callback();
|
||||
});
|
||||
},
|
||||
// Logic for allowing external Import Plugins
|
||||
let result = await hooks.aCallAll("import", { srcFile, destFile });
|
||||
let importHandledByPlugin = (result.length > 0); // This feels hacky and wrong..
|
||||
|
||||
function(callback) {
|
||||
var fileEnding = path.extname(srcFile).toLowerCase()
|
||||
var fileIsNotEtherpad = (fileEnding !== ".etherpad");
|
||||
let fileIsEtherpad = (fileEnding === ".etherpad");
|
||||
let fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
|
||||
let fileIsTXT = (fileEnding === ".txt");
|
||||
|
||||
if (fileIsNotEtherpad) {
|
||||
callback();
|
||||
let directDatabaseAccess = false;
|
||||
|
||||
return;
|
||||
}
|
||||
if (fileIsEtherpad) {
|
||||
// we do this here so we can see if the pad has quite a few edits
|
||||
let _pad = await padManager.getPad(padId);
|
||||
let headCount = _pad.head;
|
||||
|
||||
// we do this here so we can see if the pad has quite a few edits
|
||||
padManager.getPad(padId, function(err, _pad) {
|
||||
var headCount = _pad.head;
|
||||
if (headCount >= 10) {
|
||||
apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this");
|
||||
return callback("padHasData");
|
||||
}
|
||||
if (headCount >= 10) {
|
||||
apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this");
|
||||
throw "padHasData";
|
||||
}
|
||||
|
||||
fs.readFile(srcFile, "utf8", function(err, _text) {
|
||||
directDatabaseAccess = true;
|
||||
importEtherpad.setPadRaw(padId, _text, function(err) {
|
||||
callback();
|
||||
});
|
||||
const fsp_readFile = util.promisify(fs.readFile);
|
||||
let _text = await fsp_readFile(srcFile, "utf8");
|
||||
directDatabaseAccess = true;
|
||||
await importEtherpad.setPadRaw(padId, _text);
|
||||
}
|
||||
|
||||
// convert file to html if necessary
|
||||
if (!importHandledByPlugin && !directDatabaseAccess) {
|
||||
if (fileIsTXT) {
|
||||
// Don't use convertor for text files
|
||||
useConvertor = false;
|
||||
}
|
||||
|
||||
// See https://github.com/ether/etherpad-lite/issues/2572
|
||||
if (fileIsHTML || !useConvertor) {
|
||||
// if no convertor only rename
|
||||
fs.renameSync(srcFile, destFile);
|
||||
} else {
|
||||
// @TODO - no Promise interface for convertors (yet)
|
||||
await new Promise((resolve, reject) => {
|
||||
convertor.convertFile(srcFile, destFile, exportExtension, function(err) {
|
||||
// catch convert errors
|
||||
if (err) {
|
||||
console.warn("Converting Error:", err);
|
||||
reject("convertFailed");
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
// convert file to html if necessary
|
||||
function(callback) {
|
||||
if (importHandledByPlugin || directDatabaseAccess) {
|
||||
callback();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var fileEnding = path.extname(srcFile).toLowerCase();
|
||||
var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
|
||||
var fileIsTXT = (fileEnding === ".txt");
|
||||
|
||||
if (fileIsTXT) useConvertor = false; // Don't use convertor for text files
|
||||
|
||||
// See https://github.com/ether/etherpad-lite/issues/2572
|
||||
if (fileIsHTML || (useConvertor === false)) {
|
||||
// if no convertor only rename
|
||||
fs.rename(srcFile, destFile, callback);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
convertor.convertFile(srcFile, destFile, exportExtension, function(err) {
|
||||
// catch convert errors
|
||||
if (err) {
|
||||
console.warn("Converting Error:", err);
|
||||
return callback("convertFailed");
|
||||
}
|
||||
|
||||
callback();
|
||||
});
|
||||
},
|
||||
|
||||
function(callback) {
|
||||
if (useConvertor || directDatabaseAccess) {
|
||||
callback();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Read the file with no encoding for raw buffer access.
|
||||
fs.readFile(destFile, function(err, buf) {
|
||||
if (err) throw err;
|
||||
var isAscii = true;
|
||||
// Check if there are only ascii chars in the uploaded file
|
||||
for (var i=0, len=buf.length; i<len; i++) {
|
||||
if (buf[i] > 240) {
|
||||
isAscii=false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isAscii) {
|
||||
callback("uploadFailed");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
callback();
|
||||
});
|
||||
},
|
||||
|
||||
// get the pad object
|
||||
function(callback) {
|
||||
padManager.getPad(padId, function(err, _pad) {
|
||||
if (ERR(err, callback)) return;
|
||||
pad = _pad;
|
||||
callback();
|
||||
});
|
||||
},
|
||||
|
||||
// read the text
|
||||
function(callback) {
|
||||
if (directDatabaseAccess) {
|
||||
callback();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
fs.readFile(destFile, "utf8", function(err, _text) {
|
||||
if (ERR(err, callback)) return;
|
||||
text = _text;
|
||||
// Title needs to be stripped out else it appends it to the pad..
|
||||
text = text.replace("<title>", "<!-- <title>");
|
||||
text = text.replace("</title>","</title>-->");
|
||||
|
||||
// node on windows has a delay on releasing of the file lock.
|
||||
// We add a 100ms delay to work around this
|
||||
if (os.type().indexOf("Windows") > -1) {
|
||||
setTimeout(function() {callback();}, 100);
|
||||
} else {
|
||||
callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
// change text of the pad and broadcast the changeset
|
||||
function(callback) {
|
||||
if (!directDatabaseAccess) {
|
||||
var fileEnding = path.extname(srcFile).toLowerCase();
|
||||
if (importHandledByPlugin || useConvertor || fileEnding == ".htm" || fileEnding == ".html") {
|
||||
importHtml.setPadHTML(pad, text, function(e){
|
||||
if (e) {
|
||||
apiLogger.warn("Error importing, possibly caused by malformed HTML");
|
||||
}
|
||||
});
|
||||
} else {
|
||||
pad.setText(text);
|
||||
}
|
||||
}
|
||||
|
||||
// Load the Pad into memory then broadcast updates to all clients
|
||||
padManager.unloadPad(padId);
|
||||
padManager.getPad(padId, function(err, _pad) {
|
||||
var pad = _pad;
|
||||
padManager.unloadPad(padId);
|
||||
|
||||
// direct Database Access means a pad user should perform a switchToPad
|
||||
// and not attempt to receive updated pad data
|
||||
if (directDatabaseAccess) {
|
||||
callback();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// @TODO: not waiting for updatePadClients to finish
|
||||
padMessageHandler.updatePadClients(pad);
|
||||
callback();
|
||||
});
|
||||
|
||||
},
|
||||
|
||||
// clean up temporary files
|
||||
function(callback) {
|
||||
if (directDatabaseAccess) {
|
||||
callback();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
fs.unlinkSync(srcFile);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
|
||||
try {
|
||||
fs.unlinkSync(destFile);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
|
||||
callback();
|
||||
}
|
||||
], function(err) {
|
||||
var status = "ok";
|
||||
}
|
||||
|
||||
if (!useConvertor && !directDatabaseAccess) {
|
||||
// Read the file with no encoding for raw buffer access.
|
||||
let buf = await fsp_readFile(destFile);
|
||||
|
||||
// Check if there are only ascii chars in the uploaded file
|
||||
let isAscii = ! Array.prototype.some.call(buf, c => (c > 240));
|
||||
|
||||
if (!isAscii) {
|
||||
throw "uploadFailed";
|
||||
}
|
||||
}
|
||||
|
||||
// get the pad object
|
||||
let pad = await padManager.getPad(padId);
|
||||
|
||||
// read the text
|
||||
let text;
|
||||
|
||||
if (!directDatabaseAccess) {
|
||||
text = await fsp_readFile(destFile, "utf8");
|
||||
|
||||
// Title needs to be stripped out else it appends it to the pad..
|
||||
text = text.replace("<title>", "<!-- <title>");
|
||||
text = text.replace("</title>","</title>-->");
|
||||
|
||||
// node on windows has a delay on releasing of the file lock.
|
||||
// We add a 100ms delay to work around this
|
||||
if (os.type().indexOf("Windows") > -1){
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
|
||||
// change text of the pad and broadcast the changeset
|
||||
if (!directDatabaseAccess) {
|
||||
if (importHandledByPlugin || useConvertor || fileIsHTML) {
|
||||
try {
|
||||
importHtml.setPadHTML(pad, text);
|
||||
} catch (e) {
|
||||
apiLogger.warn("Error importing, possibly caused by malformed HTML");
|
||||
}
|
||||
} else {
|
||||
pad.setText(text);
|
||||
}
|
||||
}
|
||||
|
||||
// Load the Pad into memory then broadcast updates to all clients
|
||||
padManager.unloadPad(padId);
|
||||
pad = await padManager.getPad(padId);
|
||||
padManager.unloadPad(padId);
|
||||
|
||||
// direct Database Access means a pad user should perform a switchToPad
|
||||
// and not attempt to receive updated pad data
|
||||
if (!directDatabaseAccess) {
|
||||
// tell clients to update
|
||||
await padMessageHandler.updatePadClients(pad);
|
||||
}
|
||||
|
||||
if (!directDatabaseAccess) {
|
||||
// clean up temporary files
|
||||
|
||||
/*
|
||||
* TODO: directly delete the file and handle the eventual error. Checking
|
||||
* before for existence is prone to race conditions, and does not handle any
|
||||
* errors anyway.
|
||||
*/
|
||||
if (await fsp_exists(srcFile)) {
|
||||
fsp_unlink(srcFile);
|
||||
}
|
||||
|
||||
if (await fsp_exists(destFile)) {
|
||||
fsp_unlink(destFile);
|
||||
}
|
||||
}
|
||||
|
||||
return directDatabaseAccess;
|
||||
}
|
||||
|
||||
exports.doImport = function (req, res, padId)
|
||||
{
|
||||
let status = "ok";
|
||||
let directDatabaseAccess;
|
||||
|
||||
doImport(req, res, padId).then(result => {
|
||||
directDatabaseAccess = result;
|
||||
}).catch(err => {
|
||||
// check for known errors and replace the status
|
||||
if (err == "uploadFailed" || err == "convertFailed" || err == "padHasData") {
|
||||
status = err;
|
||||
err = null;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
|
||||
ERR(err);
|
||||
|
||||
// close the connection
|
||||
res.send(
|
||||
"<head> \
|
||||
<script type='text/javascript' src='../../static/js/jquery.js'></script> \
|
||||
</head> \
|
||||
<script> \
|
||||
$(window).load(function(){ \
|
||||
var impexp = window.parent.padimpexp.handleFrameCall('" + directDatabaseAccess +"', '" + status + "'); \
|
||||
}) \
|
||||
</script>"
|
||||
);
|
||||
});
|
||||
|
||||
// close the connection
|
||||
res.send(
|
||||
"<head> \
|
||||
<script type='text/javascript' src='../../static/js/jquery.js'></script> \
|
||||
</head> \
|
||||
<script> \
|
||||
$(window).load(function(){ \
|
||||
var impexp = window.parent.padimpexp.handleFrameCall('" + directDatabaseAccess +"', '" + status + "'); \
|
||||
}) \
|
||||
</script>"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,59 +15,48 @@
|
|||
*/
|
||||
|
||||
|
||||
var async = require("async");
|
||||
var db = require("../db/DB").db;
|
||||
var ERR = require("async-stacktrace");
|
||||
const thenify = require("thenify").withCallback;
|
||||
let db = require("../db/DB");
|
||||
|
||||
exports.getPadRaw = thenify(function(padId, callback){
|
||||
async.waterfall([
|
||||
function(cb){
|
||||
db.get("pad:"+padId, cb);
|
||||
},
|
||||
function(padcontent,cb){
|
||||
var records = ["pad:"+padId];
|
||||
for (var i = 0; i <= padcontent.head; i++) {
|
||||
records.push("pad:"+padId+":revs:" + i);
|
||||
}
|
||||
exports.getPadRaw = async function(padId) {
|
||||
|
||||
for (var i = 0; i <= padcontent.chatHead; i++) {
|
||||
records.push("pad:"+padId+":chat:" + i);
|
||||
}
|
||||
let padKey = "pad:" + padId;
|
||||
let padcontent = await db.get(padKey);
|
||||
|
||||
var data = {};
|
||||
|
||||
async.forEachSeries(Object.keys(records), function(key, r){
|
||||
|
||||
// For each piece of info about a pad.
|
||||
db.get(records[key], function(err, entry){
|
||||
data[records[key]] = entry;
|
||||
|
||||
// Get the Pad Authors
|
||||
if(entry.pool && entry.pool.numToAttrib){
|
||||
var authors = entry.pool.numToAttrib;
|
||||
async.forEachSeries(Object.keys(authors), function(k, c){
|
||||
if(authors[k][0] === "author"){
|
||||
var authorId = authors[k][1];
|
||||
|
||||
// Get the author info
|
||||
db.get("globalAuthor:"+authorId, function(e, authorEntry){
|
||||
if(authorEntry && authorEntry.padIDs) authorEntry.padIDs = padId;
|
||||
if(!e) data["globalAuthor:"+authorId] = authorEntry;
|
||||
});
|
||||
|
||||
}
|
||||
// console.log("authorsK", authors[k]);
|
||||
c(null);
|
||||
});
|
||||
}
|
||||
r(null); // callback;
|
||||
});
|
||||
}, function(err){
|
||||
cb(err, data);
|
||||
})
|
||||
let records = [ padKey ];
|
||||
for (let i = 0; i <= padcontent.head; i++) {
|
||||
records.push(padKey + ":revs:" + i);
|
||||
}
|
||||
], function(err, data){
|
||||
callback(null, data);
|
||||
});
|
||||
});
|
||||
|
||||
for (let i = 0; i <= padcontent.chatHead; i++) {
|
||||
records.push(padKey + ":chat:" + i);
|
||||
}
|
||||
|
||||
let data = {};
|
||||
for (let key of records) {
|
||||
|
||||
// For each piece of info about a pad.
|
||||
let entry = data[key] = await db.get(key);
|
||||
|
||||
// Get the Pad Authors
|
||||
if (entry.pool && entry.pool.numToAttrib) {
|
||||
let authors = entry.pool.numToAttrib;
|
||||
|
||||
for (let k of Object.keys(authors)) {
|
||||
if (authors[k][0] === "author") {
|
||||
let authorId = authors[k][1];
|
||||
|
||||
// Get the author info
|
||||
let authorEntry = await db.get("globalAuthor:" + authorId);
|
||||
if (authorEntry) {
|
||||
data["globalAuthor:" + authorId] = authorEntry;
|
||||
if (authorEntry.padIDs) {
|
||||
authorEntry.padIDs = padId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,61 +14,29 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
var async = require("async");
|
||||
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
|
||||
var padManager = require("../db/PadManager");
|
||||
var ERR = require("async-stacktrace");
|
||||
var _ = require('underscore');
|
||||
var Security = require('ep_etherpad-lite/static/js/security');
|
||||
var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
|
||||
var eejs = require('ep_etherpad-lite/node/eejs');
|
||||
var _analyzeLine = require('./ExportHelper')._analyzeLine;
|
||||
var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace;
|
||||
const thenify = require("thenify").withCallback;
|
||||
|
||||
function getPadHTML(pad, revNum, callback)
|
||||
async function getPadHTML(pad, revNum)
|
||||
{
|
||||
var atext = pad.atext;
|
||||
var html;
|
||||
async.waterfall([
|
||||
let atext = pad.atext;
|
||||
|
||||
// fetch revision atext
|
||||
function (callback)
|
||||
{
|
||||
if (revNum != undefined)
|
||||
{
|
||||
pad.getInternalRevisionAText(revNum, function (err, revisionAtext)
|
||||
{
|
||||
if(ERR(err, callback)) return;
|
||||
atext = revisionAtext;
|
||||
callback();
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
callback(null);
|
||||
}
|
||||
},
|
||||
if (revNum != undefined) {
|
||||
atext = await pad.getInternalRevisionAText(revNum);
|
||||
}
|
||||
|
||||
// convert atext to html
|
||||
|
||||
|
||||
function (callback)
|
||||
{
|
||||
html = getHTMLFromAtext(pad, atext);
|
||||
callback(null);
|
||||
}],
|
||||
// run final callback
|
||||
|
||||
|
||||
function (err)
|
||||
{
|
||||
if(ERR(err, callback)) return;
|
||||
callback(null, html);
|
||||
});
|
||||
return getHTMLFromAtext(pad, atext);
|
||||
}
|
||||
|
||||
exports.getPadHTML = thenify(getPadHTML);
|
||||
exports.getPadHTML = getPadHTML;
|
||||
exports.getHTMLFromAtext = getHTMLFromAtext;
|
||||
|
||||
function getHTMLFromAtext(pad, atext, authorColors)
|
||||
|
|
@ -82,15 +50,16 @@ function getHTMLFromAtext(pad, atext, authorColors)
|
|||
|
||||
// prepare tags stored as ['tag', true] to be exported
|
||||
hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){
|
||||
newProps.forEach(function (propName, i){
|
||||
newProps.forEach(function (propName, i) {
|
||||
tags.push(propName);
|
||||
props.push(propName);
|
||||
});
|
||||
});
|
||||
|
||||
// prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML
|
||||
// with tags like <span data-tag="value">
|
||||
hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){
|
||||
newProps.forEach(function (propName, i){
|
||||
newProps.forEach(function (propName, i) {
|
||||
tags.push('span data-' + propName[0] + '="' + propName[1] + '"');
|
||||
props.push(propName);
|
||||
});
|
||||
|
|
@ -454,38 +423,31 @@ function getHTMLFromAtext(pad, atext, authorColors)
|
|||
|
||||
hooks.aCallAll("getLineHTMLForExport", context);
|
||||
pieces.push(context.lineContent, "<br>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pieces.join('');
|
||||
}
|
||||
|
||||
exports.getPadHTMLDocument = thenify(function (padId, revNum, callback)
|
||||
exports.getPadHTMLDocument = async function (padId, revNum)
|
||||
{
|
||||
padManager.getPad(padId, function (err, pad)
|
||||
{
|
||||
if(ERR(err, callback)) return;
|
||||
let pad = await padManager.getPad(padId);
|
||||
|
||||
var stylesForExportCSS = "";
|
||||
// Include some Styles into the Head for Export
|
||||
hooks.aCallAll("stylesForExport", padId, function(err, stylesForExport){
|
||||
stylesForExport.forEach(function(css){
|
||||
stylesForExportCSS += css;
|
||||
});
|
||||
|
||||
getPadHTML(pad, revNum, function (err, html)
|
||||
{
|
||||
if(ERR(err, callback)) return;
|
||||
var exportedDoc = eejs.require("ep_etherpad-lite/templates/export_html.html", {
|
||||
body: html,
|
||||
padId: Security.escapeHTML(padId),
|
||||
extraCSS: stylesForExportCSS
|
||||
});
|
||||
callback(null, exportedDoc);
|
||||
});
|
||||
});
|
||||
// Include some Styles into the Head for Export
|
||||
let stylesForExportCSS = "";
|
||||
let stylesForExport = await hooks.aCallAll("stylesForExport", padId);
|
||||
stylesForExport.forEach(function(css){
|
||||
stylesForExportCSS += css;
|
||||
});
|
||||
});
|
||||
|
||||
let html = await getPadHTML(pad, revNum);
|
||||
|
||||
return eejs.require("ep_etherpad-lite/templates/export_html.html", {
|
||||
body: html,
|
||||
padId: Security.escapeHTML(padId),
|
||||
extraCSS: stylesForExportCSS
|
||||
});
|
||||
}
|
||||
|
||||
// copied from ACE
|
||||
var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/;
|
||||
|
|
|
|||
|
|
@ -18,46 +18,22 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
var async = require("async");
|
||||
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
|
||||
var padManager = require("../db/PadManager");
|
||||
var ERR = require("async-stacktrace");
|
||||
var _analyzeLine = require('./ExportHelper')._analyzeLine;
|
||||
|
||||
// This is slightly different than the HTML method as it passes the output to getTXTFromAText
|
||||
function getPadTXT(pad, revNum, callback)
|
||||
var getPadTXT = async function(pad, revNum)
|
||||
{
|
||||
var atext = pad.atext;
|
||||
var html;
|
||||
async.waterfall([
|
||||
let atext = pad.atext;
|
||||
|
||||
// fetch revision atext
|
||||
function(callback) {
|
||||
if (revNum != undefined) {
|
||||
pad.getInternalRevisionAText(revNum, function(err, revisionAtext) {
|
||||
if (ERR(err, callback)) return;
|
||||
|
||||
atext = revisionAtext;
|
||||
callback();
|
||||
});
|
||||
} else {
|
||||
callback(null);
|
||||
}
|
||||
},
|
||||
if (revNum != undefined) {
|
||||
// fetch revision atext
|
||||
atext = await pad.getInternalRevisionAText(revNum);
|
||||
}
|
||||
|
||||
// convert atext to html
|
||||
function(callback) {
|
||||
// only this line is different to the HTML function
|
||||
html = getTXTFromAtext(pad, atext);
|
||||
callback(null);
|
||||
}],
|
||||
|
||||
// run final callback
|
||||
function(err) {
|
||||
if (ERR(err, callback)) return;
|
||||
|
||||
callback(null, html);
|
||||
});
|
||||
return getTXTFromAtext(pad, atext);
|
||||
}
|
||||
|
||||
// This is different than the functionality provided in ExportHtml as it provides formatting
|
||||
|
|
@ -244,15 +220,8 @@ function getTXTFromAtext(pad, atext, authorColors)
|
|||
|
||||
exports.getTXTFromAtext = getTXTFromAtext;
|
||||
|
||||
exports.getPadTXTDocument = function(padId, revNum, callback)
|
||||
exports.getPadTXTDocument = async function(padId, revNum)
|
||||
{
|
||||
padManager.getPad(padId, function(err, pad) {
|
||||
if (ERR(err, callback)) return;
|
||||
|
||||
getPadTXT(pad, revNum, function(err, html) {
|
||||
if (ERR(err, callback)) return;
|
||||
|
||||
callback(null, html);
|
||||
});
|
||||
});
|
||||
};
|
||||
let pad = await padManager.getPad(padId);
|
||||
return getPadTXT(pad, revNum);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,43 +15,44 @@
|
|||
*/
|
||||
|
||||
var log4js = require('log4js');
|
||||
var async = require("async");
|
||||
var db = require("../db/DB").db;
|
||||
const thenify = require("thenify").withCallback;
|
||||
const db = require("../db/DB");
|
||||
|
||||
exports.setPadRaw = thenify(function(padId, records, callback)
|
||||
exports.setPadRaw = function(padId, records)
|
||||
{
|
||||
records = JSON.parse(records);
|
||||
|
||||
async.eachSeries(Object.keys(records), function(key, cb) {
|
||||
var value = records[key];
|
||||
Object.keys(records).forEach(async function(key) {
|
||||
let value = records[key];
|
||||
|
||||
if (!value) {
|
||||
return setImmediate(cb);
|
||||
return;
|
||||
}
|
||||
|
||||
let newKey;
|
||||
|
||||
if (value.padIDs) {
|
||||
// Author data - rewrite author pad ids
|
||||
value.padIDs[padId] = 1;
|
||||
var newKey = key;
|
||||
newKey = key;
|
||||
|
||||
// Does this author already exist?
|
||||
db.get(key, function(err, author) {
|
||||
if (author) {
|
||||
// Yes, add the padID to the author
|
||||
if (Object.prototype.toString.call(author) === '[object Array]') {
|
||||
author.padIDs.push(padId);
|
||||
}
|
||||
value = author;
|
||||
} else {
|
||||
// No, create a new array with the author info in
|
||||
value.padIDs = [padId];
|
||||
let author = await db.get(key);
|
||||
|
||||
if (author) {
|
||||
// Yes, add the padID to the author
|
||||
if (Object.prototype.toString.call(author) === '[object Array]') {
|
||||
author.padIDs.push(padId);
|
||||
}
|
||||
});
|
||||
|
||||
value = author;
|
||||
} else {
|
||||
// No, create a new array with the author info in
|
||||
value.padIDs = [ padId ];
|
||||
}
|
||||
} else {
|
||||
// Not author data, probably pad data
|
||||
// we can split it to look to see if it's pad data
|
||||
var oldPadId = key.split(":");
|
||||
let oldPadId = key.split(":");
|
||||
|
||||
// we know it's pad data
|
||||
if (oldPadId[0] === "pad") {
|
||||
|
|
@ -59,16 +60,11 @@ exports.setPadRaw = thenify(function(padId, records, callback)
|
|||
oldPadId[1] = padId;
|
||||
|
||||
// and create the value
|
||||
var newKey = oldPadId.join(":"); // create the new key
|
||||
newKey = oldPadId.join(":"); // create the new key
|
||||
}
|
||||
}
|
||||
|
||||
// Write the value to the server
|
||||
db.set(newKey, value);
|
||||
|
||||
setImmediate(cb);
|
||||
},
|
||||
function() {
|
||||
callback(null, true);
|
||||
await db.set(newKey, value);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,9 +18,8 @@ var log4js = require('log4js');
|
|||
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
|
||||
var contentcollector = require("ep_etherpad-lite/static/js/contentcollector");
|
||||
var cheerio = require("cheerio");
|
||||
const thenify = require("thenify").withCallback;
|
||||
|
||||
function setPadHTML(pad, html, callback)
|
||||
exports.setPadHTML = function(pad, html)
|
||||
{
|
||||
var apiLogger = log4js.getLogger("ImportHtml");
|
||||
|
||||
|
|
@ -44,7 +43,7 @@ function setPadHTML(pad, html, callback)
|
|||
apiLogger.warn("HTML was not properly formed", e);
|
||||
|
||||
// don't process the HTML because it was bad
|
||||
return callback(e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
var result = cc.finish();
|
||||
|
|
@ -52,7 +51,7 @@ function setPadHTML(pad, html, callback)
|
|||
apiLogger.debug('Lines:');
|
||||
|
||||
var i;
|
||||
for (i = 0; i < result.lines.length; i += 1) {
|
||||
for (i = 0; i < result.lines.length; i++) {
|
||||
apiLogger.debug('Line ' + (i + 1) + ' text: ' + result.lines[i]);
|
||||
apiLogger.debug('Line ' + (i + 1) + ' attributes: ' + result.lineAttribs[i]);
|
||||
}
|
||||
|
|
@ -92,7 +91,4 @@ function setPadHTML(pad, html, callback)
|
|||
apiLogger.debug('The changeset: ' + theChangeset);
|
||||
pad.setText("\n");
|
||||
pad.appendRevision(theChangeset);
|
||||
callback(null);
|
||||
}
|
||||
|
||||
exports.setPadHTML = thenify(setPadHTML);
|
||||
|
|
|
|||
Loading…
Reference in a new issue