mirror of
https://github.com/ether/etherpad-lite.git
synced 2025-01-31 19:02:59 +01:00
performance: Use worker threads to minify JS/CSS files (#3823)
This commit is contained in:
Chocobozzz
GitHub
parent
02af7d0c2d
commit
c854cced65
4 changed files with 175 additions and 82 deletions
|
|
@ -23,19 +23,23 @@ var ERR = require("async-stacktrace");
|
|||
var settings = require('./Settings');
|
||||
var async = require('async');
|
||||
var fs = require('fs');
|
||||
var StringDecoder = require('string_decoder').StringDecoder;
|
||||
var CleanCSS = require('clean-css');
|
||||
var uglifyJS = require("uglify-js");
|
||||
var path = require('path');
|
||||
var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
|
||||
var RequireKernel = require('etherpad-require-kernel');
|
||||
var urlutil = require('url');
|
||||
var mime = require('mime-types')
|
||||
var Threads = require('threads')
|
||||
var log4js = require('log4js');
|
||||
|
||||
var logger = log4js.getLogger("Minify");
|
||||
|
||||
var ROOT_DIR = path.normalize(__dirname + "/../../static/");
|
||||
var TAR_PATH = path.join(__dirname, 'tar.json');
|
||||
var tar = JSON.parse(fs.readFileSync(TAR_PATH, 'utf8'));
|
||||
|
||||
var threadsPool = Threads.Pool(function () {
|
||||
return Threads.spawn(new Threads.Worker("./MinifyWorker"))
|
||||
}, 2)
|
||||
|
||||
var LIBRARY_WHITELIST = [
|
||||
'async'
|
||||
|
|
@ -178,7 +182,7 @@ function minify(req, res)
|
|||
}
|
||||
}
|
||||
|
||||
var contentType = mime.lookup(filename);
|
||||
var contentType = mime.lookup(filename);
|
||||
|
||||
statFile(filename, function (error, date, exists) {
|
||||
if (date) {
|
||||
|
|
@ -362,20 +366,36 @@ function getFileCompressed(filename, contentType, callback) {
|
|||
if (error || !content || !settings.minify) {
|
||||
callback(error, content);
|
||||
} else if (contentType == 'text/javascript') {
|
||||
try {
|
||||
content = compressJS(content);
|
||||
if (content.error) {
|
||||
console.error(`Error compressing JS (${filename}) using UglifyJS`, content.error);
|
||||
callback('compressionError', content.error);
|
||||
} else {
|
||||
content = content.code.toString(); // Convert content obj code to string
|
||||
threadsPool.queue(async ({ compressJS }) => {
|
||||
try {
|
||||
logger.info('Compress JS file %s.', filename)
|
||||
|
||||
content = content.toString();
|
||||
const compressResult = await compressJS(content);
|
||||
|
||||
if (compressResult.error) {
|
||||
console.error(`Error compressing JS (${filename}) using UglifyJS`, compressResult.error);
|
||||
} else {
|
||||
content = compressResult.code.toString(); // Convert content obj code to string
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`getFile() returned an error in getFileCompressed(${filename}, ${contentType}): ${error}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`getFile() returned an error in getFileCompressed(${filename}, ${contentType}): ${error}`);
|
||||
}
|
||||
callback(null, content);
|
||||
|
||||
callback(null, content);
|
||||
})
|
||||
} else if (contentType == 'text/css') {
|
||||
compressCSS(filename, content, callback);
|
||||
threadsPool.queue(async ({ compressCSS }) => {
|
||||
try {
|
||||
logger.info('Compress CSS file %s.', filename)
|
||||
|
||||
content = await compressCSS(filename, ROOT_DIR);
|
||||
} catch (error) {
|
||||
console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`);
|
||||
}
|
||||
|
||||
callback(null, content);
|
||||
})
|
||||
} else {
|
||||
callback(null, content);
|
||||
}
|
||||
|
|
@ -392,65 +412,6 @@ function getFile(filename, callback) {
|
|||
}
|
||||
}
|
||||
|
||||
function compressJS(content)
|
||||
{
|
||||
const contentAsString = content.toString();
|
||||
const codeObj = uglifyJS.minify(contentAsString);
|
||||
|
||||
return codeObj;
|
||||
}
|
||||
|
||||
function compressCSS(filename, content, callback)
|
||||
{
|
||||
try {
|
||||
const absPath = path.join(ROOT_DIR, filename);
|
||||
|
||||
/*
|
||||
* Changes done to migrate CleanCSS 3.x -> 4.x:
|
||||
*
|
||||
* 1. Rework the rebase logic, because the API was simplified (but we have
|
||||
* less control now). See:
|
||||
* https://github.com/jakubpawlowicz/clean-css/blob/08f3a74925524d30bbe7ac450979de0a8a9e54b2/README.md#important-40-breaking-changes
|
||||
*
|
||||
* EXAMPLE:
|
||||
* The URLs contained in a CSS file (including all the stylesheets
|
||||
* imported by it) residing on disk at:
|
||||
* /home/muxator/etherpad/src/static/css/pad.css
|
||||
*
|
||||
* Will be rewritten rebasing them to:
|
||||
* /home/muxator/etherpad/src/static/css
|
||||
*
|
||||
* 2. CleanCSS.minify() can either receive a string containing the CSS, or
|
||||
* an array of strings. In that case each array element is interpreted as
|
||||
* an absolute local path from which the CSS file is read.
|
||||
*
|
||||
* In version 4.x, CleanCSS API was simplified, eliminating the
|
||||
* relativeTo parameter, and thus we cannot use our already loaded
|
||||
* "content" argument, but we have to wrap the absolute path to the CSS
|
||||
* in an array and ask the library to read it by itself.
|
||||
*/
|
||||
|
||||
const basePath = path.dirname(absPath);
|
||||
|
||||
new CleanCSS({
|
||||
rebase: true,
|
||||
rebaseTo: basePath,
|
||||
}).minify([absPath], function (errors, minified) {
|
||||
if (errors) {
|
||||
// on error, just yield the un-minified original, but write a log message
|
||||
console.error(`CleanCSS.minify() returned an error on ${filename} (${absPath}): ${errors}`);
|
||||
callback(null, content);
|
||||
} else {
|
||||
callback(null, minified.styles);
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
// on error, just yield the un-minified original, but write a log message
|
||||
console.error(`Unexpected error minifying ${filename} (${absPath}): ${error}`);
|
||||
callback(null, content);
|
||||
}
|
||||
}
|
||||
|
||||
exports.minify = minify;
|
||||
|
||||
exports.requestURI = requestURI;
|
||||
|
|
|
|||
67
src/node/utils/MinifyWorker.js
Normal file
67
src/node/utils/MinifyWorker.js
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Worker thread to minify JS & CSS files out of the main NodeJS thread
|
||||
*/
|
||||
|
||||
var CleanCSS = require('clean-css');
|
||||
var uglifyJS = require("uglify-js");
|
||||
var path = require('path');
|
||||
var Threads = require('threads')
|
||||
|
||||
function compressJS(content)
|
||||
{
|
||||
return uglifyJS.minify(content);
|
||||
}
|
||||
|
||||
function compressCSS(filename, ROOT_DIR)
|
||||
{
|
||||
return new Promise((res, rej) => {
|
||||
try {
|
||||
const absPath = path.join(ROOT_DIR, filename);
|
||||
|
||||
/*
|
||||
* Changes done to migrate CleanCSS 3.x -> 4.x:
|
||||
*
|
||||
* 1. Rework the rebase logic, because the API was simplified (but we have
|
||||
* less control now). See:
|
||||
* https://github.com/jakubpawlowicz/clean-css/blob/08f3a74925524d30bbe7ac450979de0a8a9e54b2/README.md#important-40-breaking-changes
|
||||
*
|
||||
* EXAMPLE:
|
||||
* The URLs contained in a CSS file (including all the stylesheets
|
||||
* imported by it) residing on disk at:
|
||||
* /home/muxator/etherpad/src/static/css/pad.css
|
||||
*
|
||||
* Will be rewritten rebasing them to:
|
||||
* /home/muxator/etherpad/src/static/css
|
||||
*
|
||||
* 2. CleanCSS.minify() can either receive a string containing the CSS, or
|
||||
* an array of strings. In that case each array element is interpreted as
|
||||
* an absolute local path from which the CSS file is read.
|
||||
*
|
||||
* In version 4.x, CleanCSS API was simplified, eliminating the
|
||||
* relativeTo parameter, and thus we cannot use our already loaded
|
||||
* "content" argument, but we have to wrap the absolute path to the CSS
|
||||
* in an array and ask the library to read it by itself.
|
||||
*/
|
||||
|
||||
const basePath = path.dirname(absPath);
|
||||
|
||||
new CleanCSS({
|
||||
rebase: true,
|
||||
rebaseTo: basePath,
|
||||
}).minify([absPath], function (errors, minified) {
|
||||
if (errors) return rej(errors)
|
||||
|
||||
return res(minified.styles)
|
||||
});
|
||||
} catch (error) {
|
||||
// on error, just yield the un-minified original, but write a log message
|
||||
console.error(`Unexpected error minifying ${filename} (${absPath}): ${error}`);
|
||||
callback(null, content);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Threads.expose({
|
||||
compressJS,
|
||||
compressCSS
|
||||
})
|
||||
77
src/package-lock.json
generated
77
src/package-lock.json
generated
|
|
@ -909,6 +909,11 @@
|
|||
"resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz",
|
||||
"integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA="
|
||||
},
|
||||
"callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="
|
||||
},
|
||||
"camelcase": {
|
||||
"version": "5.3.1",
|
||||
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
|
||||
|
|
@ -1287,7 +1292,7 @@
|
|||
},
|
||||
"css-select": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
|
||||
"integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
|
||||
"requires": {
|
||||
"boolbase": "~1.0.0",
|
||||
|
|
@ -1477,7 +1482,7 @@
|
|||
},
|
||||
"engine.io-client": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.2.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/engine.io-client/-/engine.io-client-3.2.1.tgz",
|
||||
"integrity": "sha512-y5AbkytWeM4jQr7m/koQLc5AxpRKC1hEVUb/s1FUAWEJq5AzJJ4NLvzuKPuxtDi5Mq755WuDvZ6Iv2rXj4PTzw==",
|
||||
"requires": {
|
||||
"component-emitter": "1.2.1",
|
||||
|
|
@ -1571,6 +1576,11 @@
|
|||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
|
||||
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
|
||||
},
|
||||
"esm": {
|
||||
"version": "3.2.25",
|
||||
"resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
|
||||
"integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA=="
|
||||
},
|
||||
"esprima": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
|
||||
|
|
@ -2113,6 +2123,14 @@
|
|||
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
|
||||
"dev": true
|
||||
},
|
||||
"is-observable": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/is-observable/-/is-observable-1.1.0.tgz",
|
||||
"integrity": "sha512-NqCa4Sa2d+u7BWc6CukaObG3Fh+CU9bvixbpcXYhy2VvYS7vVGIdAgnIS5Ks3A/cqk4rebLJ9s8zBstT2aKnIA==",
|
||||
"requires": {
|
||||
"symbol-observable": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"is-promise": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-1.0.1.tgz",
|
||||
|
|
@ -2569,7 +2587,7 @@
|
|||
},
|
||||
"log4js": {
|
||||
"version": "0.6.35",
|
||||
"resolved": "https://registry.npmjs.org/log4js/-/log4js-0.6.35.tgz",
|
||||
"resolved": "http://registry.npmjs.org/log4js/-/log4js-0.6.35.tgz",
|
||||
"integrity": "sha1-OrHafLFII7dO04ZcSFk6zfEfG1k=",
|
||||
"requires": {
|
||||
"readable-stream": "~1.0.2",
|
||||
|
|
@ -2578,7 +2596,7 @@
|
|||
"dependencies": {
|
||||
"readable-stream": {
|
||||
"version": "1.0.34",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
|
|
@ -2589,7 +2607,7 @@
|
|||
},
|
||||
"semver": {
|
||||
"version": "4.3.6",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
|
||||
"resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
|
||||
"integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto="
|
||||
},
|
||||
"string_decoder": {
|
||||
|
|
@ -6683,6 +6701,11 @@
|
|||
"es-abstract": "^1.17.0-next.1"
|
||||
}
|
||||
},
|
||||
"observable-fns": {
|
||||
"version": "0.5.1",
|
||||
"resolved": "https://registry.npmjs.org/observable-fns/-/observable-fns-0.5.1.tgz",
|
||||
"integrity": "sha512-wf7g4Jpo1Wt2KIqZKLGeiuLOEMqpaOZ5gJn7DmSdqXgTdxRwSdBhWegQQpPteQ2gZvzCKqNNpwb853wcpA0j7A=="
|
||||
},
|
||||
"on-finished": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
|
||||
|
|
@ -6827,7 +6850,7 @@
|
|||
},
|
||||
"path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
|
||||
"dev": true
|
||||
},
|
||||
|
|
@ -7368,7 +7391,7 @@
|
|||
},
|
||||
"socket.io-parser": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.2.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.2.0.tgz",
|
||||
"integrity": "sha512-FYiBx7rc/KORMJlgsXysflWx/RIvtqZbyGLlHZvjfmPTPeuD/I8MaW7cfFrj5tRltICJdgwflhfZ3NVVbVLFQA==",
|
||||
"requires": {
|
||||
"component-emitter": "1.2.1",
|
||||
|
|
@ -7656,6 +7679,11 @@
|
|||
"resolved": "https://registry.npmjs.org/swagger-schema-official/-/swagger-schema-official-2.0.0-bab6bed.tgz",
|
||||
"integrity": "sha1-cAcEaNbSl3ylI3suUZyn0Gouo/0="
|
||||
},
|
||||
"symbol-observable": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz",
|
||||
"integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ=="
|
||||
},
|
||||
"tar-stream": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz",
|
||||
|
|
@ -7696,11 +7724,46 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"threads": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/threads/-/threads-1.4.0.tgz",
|
||||
"integrity": "sha512-vaKhZODDnciJn4Bjmkd1GbJ2dlzFbzxwcQNM1IZV1bsCXmlJpirSAKsYG7MT7MHgO+qQxTaIn6CMstmlYnGNWw==",
|
||||
"requires": {
|
||||
"callsites": "^3.1.0",
|
||||
"debug": "^4.1.1",
|
||||
"is-observable": "^1.1.0",
|
||||
"observable-fns": "^0.5.1",
|
||||
"tiny-worker": ">= 2"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
|
||||
},
|
||||
"tiny-worker": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/tiny-worker/-/tiny-worker-2.3.0.tgz",
|
||||
"integrity": "sha512-pJ70wq5EAqTAEl9IkGzA+fN0836rycEuz2Cn6yeZ6FRzlVS5IDOkFHpIoEsksPRQV34GDqXm65+OlnZqUSyK2g==",
|
||||
"requires": {
|
||||
"esm": "^3.2.25"
|
||||
}
|
||||
},
|
||||
"tinycon": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/tinycon/-/tinycon-0.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@
|
|||
"semver": "5.6.0",
|
||||
"slide": "1.1.6",
|
||||
"socket.io": "2.1.1",
|
||||
"threads": "^1.4.0",
|
||||
"tiny-worker": "^2.3.0",
|
||||
"tinycon": "0.0.1",
|
||||
"ueberdb2": "0.4.9",
|
||||
"uglify-js": "3.8.1",
|
||||
|
|
|
|||
Loading…
Reference in a new issue