From eec1dd18d22c237bd43e170312f94908e1c378b7 Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+samtv12345@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:04:08 +0100 Subject: [PATCH] Added more performant minifiers for css and js. --- src/ep.json | 6 -- src/node/hooks/express.ts | 5 +- src/node/hooks/express/static.ts | 8 +- src/node/server.ts | 4 +- src/node/utils/{Minify.js => Minify.ts} | 109 ++++++++++-------------- src/node/utils/MinifyWorker.js | 33 ------- src/node/utils/MinifyWorker.ts | 29 +++++++ src/package.json | 5 +- 8 files changed, 87 insertions(+), 112 deletions(-) rename src/node/utils/{Minify.js => Minify.ts} (79%) delete mode 100644 src/node/utils/MinifyWorker.js create mode 100644 src/node/utils/MinifyWorker.ts diff --git a/src/ep.json b/src/ep.json index a6d65a08f..084d7b49c 100644 --- a/src/ep.json +++ b/src/ep.json @@ -6,12 +6,6 @@ "shutdown": "ep_etherpad-lite/node/db/DB" } }, - { - "name": "Minify", - "hooks": { - "shutdown": "ep_etherpad-lite/node/utils/Minify" - } - }, { "name": "express", "hooks": { diff --git a/src/node/hooks/express.ts b/src/node/hooks/express.ts index 29da71ac3..e5d34a14c 100644 --- a/src/node/hooks/express.ts +++ b/src/node/hooks/express.ts @@ -28,6 +28,8 @@ let sessionStore: { shutdown: () => void; } | null; const sockets:Set = new Set(); const socketsEvents = new events.EventEmitter(); const startTime = stats.settableGauge('httpStartTime'); +import https from 'https'; +import http from 'http'; exports.server = null; @@ -119,11 +121,8 @@ exports.restartServer = async () => { options.ca.push(fs.readFileSync(caFileName)); } } - - const https = require('https'); exports.server = https.createServer(options, app); } else { - const http = require('http'); exports.server = http.createServer(app); } diff --git a/src/node/hooks/express/static.ts b/src/node/hooks/express/static.ts index 1d9ba01e9..e7480101d 100644 --- a/src/node/hooks/express/static.ts +++ b/src/node/hooks/express/static.ts @@ -4,8 +4,8 @@ import {MapArrayType} from "../../types/MapType"; import {PartType} from "../../types/PartType"; const fs = require('fs').promises; -const minify = require('../../utils/Minify'); -const path = require('path'); +import {requestURIs, minify} from '../../utils/Minify'; +import path from 'path'; const plugins = require('../../../static/js/pluginfw/plugin_defs'); const settings = require('../../utils/Settings'); import CachingMiddleware from '../../utils/caching_middleware'; @@ -41,7 +41,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => { // Minify will serve static files compressed (minify enabled). It also has // file-specific hacks for ace/require-kernel/etc. - app.all('/static/:filename(*)', minify.minify); + app.all('/static/:filename(*)', minify); // Setup middleware that will package JavaScript files served by minify for // CommonJS loader on the client-side. @@ -51,7 +51,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => { rootURI: 'http://invalid.invalid/static/js/', libraryPath: 'javascripts/lib/', libraryURI: 'http://invalid.invalid/static/plugins/', - requestURIs: minify.requestURIs, // Loop-back is causing problems, this is a workaround. + requestURIs: requestURIs, // Loop-back is causing problems, this is a workaround. }); const StaticAssociator = Yajsml.associators.StaticAssociator; diff --git a/src/node/server.ts b/src/node/server.ts index f96db3ab1..13a004664 100755 --- a/src/node/server.ts +++ b/src/node/server.ts @@ -31,11 +31,13 @@ import axios from "axios"; const settings = require('./utils/Settings'); +import wtfnodeMod from 'wtfnode'; + let wtfnode: any; if (settings.dumpOnUncleanExit) { // wtfnode should be loaded after log4js.replaceConsole() so that it uses log4js for logging, and // it should be above everything else so that it can hook in before resources are used. - wtfnode = require('wtfnode'); + wtfnode = wtfnodeMod; } diff --git a/src/node/utils/Minify.js b/src/node/utils/Minify.ts similarity index 79% rename from src/node/utils/Minify.js rename to src/node/utils/Minify.ts index 2e8a2d960..ad4514a23 100644 --- a/src/node/utils/Minify.js +++ b/src/node/utils/Minify.ts @@ -21,21 +21,22 @@ * limitations under the License. */ +import {compressCSS, compressJS} from "./MinifyWorker"; +import {MapArrayType} from "../types/MapType"; + const settings = require('./Settings'); -const fs = require('fs').promises; -const path = require('path'); +import {promises as fs} from 'fs' +import path from 'path'; const plugins = require('../../static/js/pluginfw/plugin_defs'); const RequireKernel = require('etherpad-require-kernel'); -const mime = require('mime-types'); -const Threads = require('threads'); -const log4js = require('log4js'); +import mime from 'mime-types'; +import log4js from 'log4js'; const sanitizePathname = require('./sanitizePathname'); const logger = log4js.getLogger('Minify'); const ROOT_DIR = path.join(settings.root, 'src/static/'); -const threadsPool = new Threads.Pool(() => Threads.spawn(new Threads.Worker('./MinifyWorker')), 2); const LIBRARY_WHITELIST = [ 'async', @@ -49,10 +50,10 @@ const LIBRARY_WHITELIST = [ // What follows is a terrible hack to avoid loop-back within the server. // TODO: Serve files from another service, or directly from the file system. -const requestURI = async (url, method, headers) => { +export const requestURI = async (url: string, method: string, headers: MapArrayType) => { const parsedUrl = new URL(url); let status = 500; - const content = []; + const content: string[] = []; const mockRequest = { url, method, @@ -62,7 +63,7 @@ const requestURI = async (url, method, headers) => { let mockResponse; const p = new Promise((resolve) => { mockResponse = { - writeHead: (_status, _headers) => { + writeHead: (_status: number, _headers: MapArrayType) => { status = _status; for (const header in _headers) { if (Object.prototype.hasOwnProperty.call(_headers, header)) { @@ -70,16 +71,16 @@ const requestURI = async (url, method, headers) => { } } }, - setHeader: (header, value) => { + setHeader: (header: string, value: string) => { headers[header.toLowerCase()] = value.toString(); }, - header: (header, value) => { + header: (header: string, value: string) => { headers[header.toLowerCase()] = value.toString(); }, - write: (_content) => { + write: (_content: string) => { _content && content.push(_content); }, - end: (_content) => { + end: (_content: string) => { _content && content.push(_content); resolve([status, headers, content.join('')]); }, @@ -89,16 +90,17 @@ const requestURI = async (url, method, headers) => { return await p; }; -const requestURIs = (locations, method, headers, callback) => { +export const requestURIs = (locations: string[], method: string, headers: string[], callback:Function) => { Promise.all(locations.map(async (loc) => { try { return await requestURI(loc, method, headers); - } catch (err) { + } catch (err:any) { logger.debug(`requestURI(${JSON.stringify(loc)}, ${JSON.stringify(method)}, ` + `${JSON.stringify(headers)}) failed: ${err.stack || err}`); return [500, headers, '']; } - })).then((responses) => { + // @ts-ignore + })).then((responses:[[number,number,number]]) => { const statuss = responses.map((x) => x[0]); const headerss = responses.map((x) => x[1]); const contentss = responses.map((x) => x[2]); @@ -120,11 +122,11 @@ const compatPaths = { * @param req the Express request * @param res the Express response */ -const minify = async (req, res) => { +export const minify = async (req: any, res:any) => { let filename = req.params.filename; try { filename = sanitizePathname(filename); - } catch (err) { + } catch (err:any) { logger.error(`sanitization of pathname "${filename}" failed: ${err.stack || err}`); res.writeHead(404, {}); res.end(); @@ -132,6 +134,7 @@ const minify = async (req, res) => { } // Backward compatibility for plugins that require() files from old paths. + // @ts-ignore const newLocation = compatPaths[filename.replace(/^plugins\/ep_etherpad-lite\/static\//, '')]; if (newLocation != null) { logger.warn(`request for deprecated path "${filename}", replacing with "${newLocation}"`); @@ -169,7 +172,7 @@ const minify = async (req, res) => { const [, testf] = /^plugins\/ep_etherpad-lite\/(tests\/frontend\/.*)/.exec(filename) || []; if (testf != null) filename = `../${testf}`; - const contentType = mime.lookup(filename); + const contentType = mime.lookup(filename) as string; const [date, exists] = await statFile(filename, 3); if (date) { @@ -186,7 +189,7 @@ const minify = async (req, res) => { if (!exists) { res.writeHead(404, {}); res.end(); - } else if (new Date(req.headers['if-modified-since']) >= date) { + } else if (new Date(req.headers['if-modified-since']) >= date!) { res.writeHead(304, {}); res.end(); } else if (req.method === 'HEAD') { @@ -206,7 +209,7 @@ const minify = async (req, res) => { }; // Check for the existance of the file and get the last modification date. -const statFile = async (filename, dirStatLimit) => { +const statFile = async (filename: string, dirStatLimit: number):Promise<[Date|null, boolean]> => { /* * The only external call to this function provides an explicit value for * dirStatLimit: this check could be removed. @@ -227,9 +230,10 @@ const statFile = async (filename, dirStatLimit) => { let stats; try { stats = await fs.stat(path.resolve(ROOT_DIR, filename)); - } catch (err) { + } catch (err:any) { if (['ENOENT', 'ENOTDIR'].includes(err.code)) { // Stat the directory instead. + // @ts-ignore const [date] = await statFile(path.dirname(filename), dirStatLimit - 1); return [date, false]; } @@ -241,7 +245,7 @@ const statFile = async (filename, dirStatLimit) => { const lastModifiedDateOfEverything = async () => { const folders2check = [path.join(ROOT_DIR, 'js/'), path.join(ROOT_DIR, 'css/')]; - let latestModification = null; + let latestModification:Date|null = null; // go through this two folders await Promise.all(folders2check.map(async (dir) => { // read the files in the folder @@ -251,7 +255,7 @@ const lastModifiedDateOfEverything = async () => { files.push('.'); // go through all files in this folder - await Promise.all(files.map(async (filename) => { + await Promise.all(files.map(async (filename: string) => { // get the stat data of this file const stats = await fs.stat(path.join(dir, filename)); @@ -269,58 +273,39 @@ const lastModifiedDateOfEverything = async () => { const _requireLastModified = new Date(); const requireDefinition = () => `var require = ${RequireKernel.kernelSource};\n`; -const getFileCompressed = async (filename, contentType) => { +const getFileCompressed = async (filename: string, contentType: string) => { let content = await getFile(filename); if (!content || !settings.minify) { return content; } else if (contentType === 'application/javascript') { - return await new Promise((resolve) => { - threadsPool.queue(async ({compressJS}) => { - try { - logger.info('Compress JS file %s.', filename); + try { + logger.info('Compress JS file %s.', filename); - content = content.toString(); - const compressResult = await compressJS(content); + content = content.toString(); + const compressResult = compressJS(content); - if (compressResult.error) { - console.error(`Error compressing JS (${filename}) using terser`, compressResult.error); - } else { - content = compressResult.code.toString(); // Convert content obj code to string - } - } catch (error) { - console.error('getFile() returned an error in ' + + content = compressResult.code.toString(); // Convert content obj code to string + } catch (error) { + console.error('getFile() returned an error in ' + `getFileCompressed(${filename}, ${contentType}): ${error}`); - } - resolve(content); - }); - }); + } + return content } else if (contentType === 'text/css') { - return await new Promise((resolve) => { - threadsPool.queue(async ({compressCSS}) => { - try { - logger.info('Compress CSS file %s.', filename); - - content = await compressCSS(filename, ROOT_DIR); - } catch (error) { - console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`); - } - resolve(content); - }); - }); + try { + logger.info('Compress CSS file %s.', filename); + content = compressCSS(filename, ROOT_DIR, Buffer.from(content)).toString(); + return content + } catch (error) { + console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`); + } } else { return content; } }; -const getFile = async (filename) => { +const getFile = async (filename: string) => { if (filename === 'js/require-kernel.js') return requireDefinition(); return await fs.readFile(path.resolve(ROOT_DIR, filename)); }; -exports.minify = (req, res, next) => minify(req, res).catch((err) => next(err || new Error(err))); - -exports.requestURIs = requestURIs; - -exports.shutdown = async (hookName, context) => { - await threadsPool.terminate(); -}; +exports.minify = (req: Request, res: Response, next: Function) => minify(req, res).catch((err) => next(err || new Error(err))); diff --git a/src/node/utils/MinifyWorker.js b/src/node/utils/MinifyWorker.js deleted file mode 100644 index 364ecc96c..000000000 --- a/src/node/utils/MinifyWorker.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; -/** - * Worker thread to minify JS & CSS files out of the main NodeJS thread - */ - -const CleanCSS = require('clean-css'); -const Terser = require('terser'); -const fsp = require('fs').promises; -const path = require('path'); -const Threads = require('threads'); - -const compressJS = (content) => Terser.minify(content); - -const compressCSS = async (filename, ROOT_DIR) => { - const absPath = path.resolve(ROOT_DIR, filename); - try { - const basePath = path.dirname(absPath); - const output = await new CleanCSS({ - rebase: true, - rebaseTo: basePath, - }).minify([absPath]); - return output.styles; - } catch (error) { - // on error, just yield the un-minified original, but write a log message - console.error(`Unexpected error minifying ${filename} (${absPath}): ${error}`); - return await fsp.readFile(absPath, 'utf8'); - } -}; - -Threads.expose({ - compressJS, - compressCSS, -}); diff --git a/src/node/utils/MinifyWorker.ts b/src/node/utils/MinifyWorker.ts new file mode 100644 index 000000000..08a365996 --- /dev/null +++ b/src/node/utils/MinifyWorker.ts @@ -0,0 +1,29 @@ +'use strict'; +/** + * Worker thread to minify JS & CSS files out of the main NodeJS thread + */ + + +import {promises as fsp} from "fs"; +import path from 'path'; +import {minifySync} from "@swc/core"; +import lightminify from 'lightningcss'; + +export const compressJS = (content: string) => minifySync(content); + +export const compressCSS = (filename: string, ROOT_DIR: string, content: Buffer) => { + const absPath = path.resolve(ROOT_DIR, filename); + try { + const basePath = path.dirname(absPath); + let { code } = lightminify.transform({ + filename: absPath, + minify: true, + code: content + }); + return code; + } catch (error) { + // on error, just yield the un-minified original, but write a log message + console.error(`Unexpected error minifying ${filename} (${absPath}): ${error}`); + return content.toString() + } +}; diff --git a/src/package.json b/src/package.json index 4d915758c..409643e1a 100644 --- a/src/package.json +++ b/src/package.json @@ -30,9 +30,9 @@ } ], "dependencies": { + "@swc/core": "^1.4.8", "async": "^3.2.5", "axios": "^1.7.2", - "clean-css": "^5.3.3", "cookie-parser": "^1.4.6", "cross-spawn": "^7.0.3", "ejs": "^3.1.10", @@ -51,6 +51,7 @@ "jsonminify": "0.4.2", "jsonwebtoken": "^9.0.2", "languages4translatewiki": "0.1.3", + "lightningcss": "^1.24.1", "live-plugin-manager": "^1.0.0", "lodash.clonedeep": "4.5.0", "log4js": "^6.9.1", @@ -68,8 +69,6 @@ "socket.io": "^4.7.5", "socket.io-client": "^4.7.5", "superagent": "^9.0.2", - "terser": "^5.30.3", - "threads": "^1.7.0", "tinycon": "0.6.8", "tsx": "^4.10.5", "ueberdb2": "^4.2.63",