From c854cced659d9fe9a4ba796c46984c5d67a2fa36 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 4 Jun 2020 15:00:50 +0200 Subject: [PATCH] performance: Use worker threads to minify JS/CSS files (#3823) --- src/node/utils/Minify.js | 111 +++++++++++---------------------- src/node/utils/MinifyWorker.js | 67 ++++++++++++++++++++ src/package-lock.json | 77 ++++++++++++++++++++--- src/package.json | 2 + 4 files changed, 175 insertions(+), 82 deletions(-) create mode 100644 src/node/utils/MinifyWorker.js diff --git a/src/node/utils/Minify.js b/src/node/utils/Minify.js index ef1b89a07..190871b1c 100644 --- a/src/node/utils/Minify.js +++ b/src/node/utils/Minify.js @@ -23,19 +23,23 @@ var ERR = require("async-stacktrace"); var settings = require('./Settings'); var async = require('async'); var fs = require('fs'); -var StringDecoder = require('string_decoder').StringDecoder; -var CleanCSS = require('clean-css'); -var uglifyJS = require("uglify-js"); var path = require('path'); var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins"); var RequireKernel = require('etherpad-require-kernel'); var urlutil = require('url'); var mime = require('mime-types') +var Threads = require('threads') +var log4js = require('log4js'); + +var logger = log4js.getLogger("Minify"); var ROOT_DIR = path.normalize(__dirname + "/../../static/"); var TAR_PATH = path.join(__dirname, 'tar.json'); var tar = JSON.parse(fs.readFileSync(TAR_PATH, 'utf8')); +var threadsPool = Threads.Pool(function () { + return Threads.spawn(new Threads.Worker("./MinifyWorker")) +}, 2) var LIBRARY_WHITELIST = [ 'async' @@ -178,7 +182,7 @@ function minify(req, res) } } - var contentType = mime.lookup(filename); + var contentType = mime.lookup(filename); statFile(filename, function (error, date, exists) { if (date) { @@ -362,20 +366,36 @@ function getFileCompressed(filename, contentType, callback) { if (error || !content || !settings.minify) { callback(error, content); } else if (contentType == 'text/javascript') { - try { - content = compressJS(content); - if (content.error) { - console.error(`Error compressing JS (${filename}) using UglifyJS`, content.error); - callback('compressionError', content.error); - } else { - content = content.code.toString(); // Convert content obj code to string + threadsPool.queue(async ({ compressJS }) => { + try { + logger.info('Compress JS file %s.', filename) + + content = content.toString(); + const compressResult = await compressJS(content); + + if (compressResult.error) { + console.error(`Error compressing JS (${filename}) using UglifyJS`, compressResult.error); + } else { + content = compressResult.code.toString(); // Convert content obj code to string + } + } catch (error) { + console.error(`getFile() returned an error in getFileCompressed(${filename}, ${contentType}): ${error}`); } - } catch (error) { - console.error(`getFile() returned an error in getFileCompressed(${filename}, ${contentType}): ${error}`); - } - callback(null, content); + + callback(null, content); + }) } else if (contentType == 'text/css') { - compressCSS(filename, content, callback); + threadsPool.queue(async ({ compressCSS }) => { + try { + logger.info('Compress CSS file %s.', filename) + + content = await compressCSS(filename, ROOT_DIR); + } catch (error) { + console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`); + } + + callback(null, content); + }) } else { callback(null, content); } @@ -392,65 +412,6 @@ function getFile(filename, callback) { } } -function compressJS(content) -{ - const contentAsString = content.toString(); - const codeObj = uglifyJS.minify(contentAsString); - - return codeObj; -} - -function compressCSS(filename, content, callback) -{ - try { - const absPath = path.join(ROOT_DIR, filename); - - /* - * Changes done to migrate CleanCSS 3.x -> 4.x: - * - * 1. Rework the rebase logic, because the API was simplified (but we have - * less control now). See: - * https://github.com/jakubpawlowicz/clean-css/blob/08f3a74925524d30bbe7ac450979de0a8a9e54b2/README.md#important-40-breaking-changes - * - * EXAMPLE: - * The URLs contained in a CSS file (including all the stylesheets - * imported by it) residing on disk at: - * /home/muxator/etherpad/src/static/css/pad.css - * - * Will be rewritten rebasing them to: - * /home/muxator/etherpad/src/static/css - * - * 2. CleanCSS.minify() can either receive a string containing the CSS, or - * an array of strings. In that case each array element is interpreted as - * an absolute local path from which the CSS file is read. - * - * In version 4.x, CleanCSS API was simplified, eliminating the - * relativeTo parameter, and thus we cannot use our already loaded - * "content" argument, but we have to wrap the absolute path to the CSS - * in an array and ask the library to read it by itself. - */ - - const basePath = path.dirname(absPath); - - new CleanCSS({ - rebase: true, - rebaseTo: basePath, - }).minify([absPath], function (errors, minified) { - if (errors) { - // on error, just yield the un-minified original, but write a log message - console.error(`CleanCSS.minify() returned an error on ${filename} (${absPath}): ${errors}`); - callback(null, content); - } else { - callback(null, minified.styles); - } - }); - } catch (error) { - // on error, just yield the un-minified original, but write a log message - console.error(`Unexpected error minifying ${filename} (${absPath}): ${error}`); - callback(null, content); - } -} - exports.minify = minify; exports.requestURI = requestURI; diff --git a/src/node/utils/MinifyWorker.js b/src/node/utils/MinifyWorker.js new file mode 100644 index 000000000..624efa4a3 --- /dev/null +++ b/src/node/utils/MinifyWorker.js @@ -0,0 +1,67 @@ +/** + * Worker thread to minify JS & CSS files out of the main NodeJS thread + */ + +var CleanCSS = require('clean-css'); +var uglifyJS = require("uglify-js"); +var path = require('path'); +var Threads = require('threads') + +function compressJS(content) +{ + return uglifyJS.minify(content); +} + +function compressCSS(filename, ROOT_DIR) +{ + return new Promise((res, rej) => { + try { + const absPath = path.join(ROOT_DIR, filename); + + /* + * Changes done to migrate CleanCSS 3.x -> 4.x: + * + * 1. Rework the rebase logic, because the API was simplified (but we have + * less control now). See: + * https://github.com/jakubpawlowicz/clean-css/blob/08f3a74925524d30bbe7ac450979de0a8a9e54b2/README.md#important-40-breaking-changes + * + * EXAMPLE: + * The URLs contained in a CSS file (including all the stylesheets + * imported by it) residing on disk at: + * /home/muxator/etherpad/src/static/css/pad.css + * + * Will be rewritten rebasing them to: + * /home/muxator/etherpad/src/static/css + * + * 2. CleanCSS.minify() can either receive a string containing the CSS, or + * an array of strings. In that case each array element is interpreted as + * an absolute local path from which the CSS file is read. + * + * In version 4.x, CleanCSS API was simplified, eliminating the + * relativeTo parameter, and thus we cannot use our already loaded + * "content" argument, but we have to wrap the absolute path to the CSS + * in an array and ask the library to read it by itself. + */ + + const basePath = path.dirname(absPath); + + new CleanCSS({ + rebase: true, + rebaseTo: basePath, + }).minify([absPath], function (errors, minified) { + if (errors) return rej(errors) + + return res(minified.styles) + }); + } catch (error) { + // on error, just yield the un-minified original, but write a log message + console.error(`Unexpected error minifying ${filename} (${absPath}): ${error}`); + callback(null, content); + } + }) +} + +Threads.expose({ + compressJS, + compressCSS +}) diff --git a/src/package-lock.json b/src/package-lock.json index dbbcc7f70..f485ac27a 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -909,6 +909,11 @@ "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", "integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA=" }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==" + }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -1287,7 +1292,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "requires": { "boolbase": "~1.0.0", @@ -1477,7 +1482,7 @@ }, "engine.io-client": { "version": "3.2.1", - "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.2.1.tgz", + "resolved": "http://registry.npmjs.org/engine.io-client/-/engine.io-client-3.2.1.tgz", "integrity": "sha512-y5AbkytWeM4jQr7m/koQLc5AxpRKC1hEVUb/s1FUAWEJq5AzJJ4NLvzuKPuxtDi5Mq755WuDvZ6Iv2rXj4PTzw==", "requires": { "component-emitter": "1.2.1", @@ -1571,6 +1576,11 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -2113,6 +2123,14 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "is-observable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-1.1.0.tgz", + "integrity": "sha512-NqCa4Sa2d+u7BWc6CukaObG3Fh+CU9bvixbpcXYhy2VvYS7vVGIdAgnIS5Ks3A/cqk4rebLJ9s8zBstT2aKnIA==", + "requires": { + "symbol-observable": "^1.1.0" + } + }, "is-promise": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-1.0.1.tgz", @@ -2569,7 +2587,7 @@ }, "log4js": { "version": "0.6.35", - "resolved": "https://registry.npmjs.org/log4js/-/log4js-0.6.35.tgz", + "resolved": "http://registry.npmjs.org/log4js/-/log4js-0.6.35.tgz", "integrity": "sha1-OrHafLFII7dO04ZcSFk6zfEfG1k=", "requires": { "readable-stream": "~1.0.2", @@ -2578,7 +2596,7 @@ "dependencies": { "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "requires": { "core-util-is": "~1.0.0", @@ -2589,7 +2607,7 @@ }, "semver": { "version": "4.3.6", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz", "integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=" }, "string_decoder": { @@ -6683,6 +6701,11 @@ "es-abstract": "^1.17.0-next.1" } }, + "observable-fns": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/observable-fns/-/observable-fns-0.5.1.tgz", + "integrity": "sha512-wf7g4Jpo1Wt2KIqZKLGeiuLOEMqpaOZ5gJn7DmSdqXgTdxRwSdBhWegQQpPteQ2gZvzCKqNNpwb853wcpA0j7A==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -6827,7 +6850,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -7368,7 +7391,7 @@ }, "socket.io-parser": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.2.0.tgz", "integrity": "sha512-FYiBx7rc/KORMJlgsXysflWx/RIvtqZbyGLlHZvjfmPTPeuD/I8MaW7cfFrj5tRltICJdgwflhfZ3NVVbVLFQA==", "requires": { "component-emitter": "1.2.1", @@ -7656,6 +7679,11 @@ "resolved": "https://registry.npmjs.org/swagger-schema-official/-/swagger-schema-official-2.0.0-bab6bed.tgz", "integrity": "sha1-cAcEaNbSl3ylI3suUZyn0Gouo/0=" }, + "symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" + }, "tar-stream": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz", @@ -7696,11 +7724,46 @@ } } }, + "threads": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/threads/-/threads-1.4.0.tgz", + "integrity": "sha512-vaKhZODDnciJn4Bjmkd1GbJ2dlzFbzxwcQNM1IZV1bsCXmlJpirSAKsYG7MT7MHgO+qQxTaIn6CMstmlYnGNWw==", + "requires": { + "callsites": "^3.1.0", + "debug": "^4.1.1", + "is-observable": "^1.1.0", + "observable-fns": "^0.5.1", + "tiny-worker": ">= 2" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "tiny-worker": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tiny-worker/-/tiny-worker-2.3.0.tgz", + "integrity": "sha512-pJ70wq5EAqTAEl9IkGzA+fN0836rycEuz2Cn6yeZ6FRzlVS5IDOkFHpIoEsksPRQV34GDqXm65+OlnZqUSyK2g==", + "requires": { + "esm": "^3.2.25" + } + }, "tinycon": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tinycon/-/tinycon-0.0.1.tgz", diff --git a/src/package.json b/src/package.json index dbd7289a4..330f37a2f 100644 --- a/src/package.json +++ b/src/package.json @@ -60,6 +60,8 @@ "semver": "5.6.0", "slide": "1.1.6", "socket.io": "2.1.1", + "threads": "^1.4.0", + "tiny-worker": "^2.3.0", "tinycon": "0.0.1", "ueberdb2": "0.4.9", "uglify-js": "3.8.1",