Fixed backend

This commit is contained in:
SamTv12345 2024-07-22 16:20:10 +02:00
parent f8175a6433
commit 1d977679dd
28 changed files with 244 additions and 211 deletions

View file

@ -315,6 +315,12 @@ importers:
'@types/jsonwebtoken':
specifier: ^9.0.6
version: 9.0.6
'@types/lodash.clonedeep':
specifier: ^4.5.9
version: 4.5.9
'@types/mime-types':
specifier: ^2.1.4
version: 2.1.4
'@types/mocha':
specifier: ^10.0.7
version: 10.0.7
@ -324,6 +330,9 @@ importers:
'@types/oidc-provider':
specifier: ^8.5.1
version: 8.5.1
'@types/proxy-addr':
specifier: ^2.0.3
version: 2.0.3
'@types/resolve':
specifier: ^1.20.6
version: 1.20.6
@ -1567,6 +1576,12 @@ packages:
'@types/lockfile@1.0.4':
resolution: {integrity: sha512-Q8oFIHJHr+htLrTXN2FuZfg+WXVHQRwU/hC2GpUu+Q8e3FUM9EDkS2pE3R2AO1ZGu56f479ybdMCNF1DAu8cAQ==}
'@types/lodash.clonedeep@4.5.9':
resolution: {integrity: sha512-19429mWC+FyaAhOLzsS8kZUsI+/GmBAQ0HFiCPsKGU+7pBXOQWhyrY6xNNDwUSX8SMZMJvuFVMF9O5dQOlQK9Q==}
'@types/lodash@4.17.7':
resolution: {integrity: sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA==}
'@types/markdown-it@14.1.1':
resolution: {integrity: sha512-4NpsnpYl2Gt1ljyBGrKMxFYAYvpqbnnkgP/i/g+NLpjEUa3obn1XJCur9YbEXKDAkaXqsR1LbDnGEJ0MmKFxfg==}
@ -1579,6 +1594,9 @@ packages:
'@types/methods@1.1.4':
resolution: {integrity: sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==}
'@types/mime-types@2.1.4':
resolution: {integrity: sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==}
'@types/mime@1.3.5':
resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
@ -1600,6 +1618,9 @@ packages:
'@types/prop-types@15.7.12':
resolution: {integrity: sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==}
'@types/proxy-addr@2.0.3':
resolution: {integrity: sha512-TgAHHO4tNG3HgLTUhB+hM4iwW6JUNeQHCLnF1DjaDA9c69PN+IasoFu2MYDhubFc+ZIw5c5t9DMtjvrD6R3Egg==}
'@types/qs@6.9.15':
resolution: {integrity: sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==}
@ -5725,6 +5746,12 @@ snapshots:
'@types/lockfile@1.0.4': {}
'@types/lodash.clonedeep@4.5.9':
dependencies:
'@types/lodash': 4.17.7
'@types/lodash@4.17.7': {}
'@types/markdown-it@14.1.1':
dependencies:
'@types/linkify-it': 5.0.0
@ -5738,6 +5765,8 @@ snapshots:
'@types/methods@1.1.4': {}
'@types/mime-types@2.1.4': {}
'@types/mime@1.3.5': {}
'@types/mocha@10.0.7': {}
@ -5760,6 +5789,10 @@ snapshots:
'@types/prop-types@15.7.12': {}
'@types/proxy-addr@2.0.3':
dependencies:
'@types/node': 20.14.11
'@types/qs@6.9.15': {}
'@types/range-parser@1.2.7': {}

View file

@ -22,7 +22,7 @@
*/
import ueberDB from 'ueberdb2';
const settings = require('../utils/Settings');
import settings from '../utils/Settings';
import log4js from 'log4js';
import {measuredCollection} from '../stats';

View file

@ -14,7 +14,7 @@ import AttributePool from '../../static/js/AttributePool';
import Stream from '../utils/Stream';
const assert = require('assert').strict;
import {get, set, setSub, remove} from './DB';
const settings = require('../utils/Settings');
import settings from '../utils/Settings';
import {addPad, getAuthorColorId, getAuthorName, getColorPalette, removePad} from './AuthorManager';
import {doesPadExist, getPad} from './PadManager';
import {kickSessionsFromPad} from '../handler/PadMessageHandler';

View file

@ -26,9 +26,9 @@ import {callAll} from '../../static/js/pluginfw/hooks.js';
import {doesPadExist, getPad} from './PadManager';
import {getPadId, isReadOnlyId} from './ReadOnlyManager';
import {findAuthorID} from './SessionManager';
const settings = require('../utils/Settings');
import settings from '../utils/Settings';
import {normalizeAuthzLevel} from '../hooks/express/webaccess';
const log4js = require('log4js');
import log4js from 'log4js';
const authLogger = log4js.getLogger('auth');
import {padUtils as padutils} from '../../static/js/pad_utils';

View file

@ -21,15 +21,15 @@
import {MapArrayType} from "../types/MapType";
const api = require('../db/API');
const padManager = require('../db/PadManager');
import * as api from '../db/API';
import {sanitizePadId} from '../db/PadManager';
import createHTTPError from 'http-errors';
import {Http2ServerRequest, Http2ServerResponse} from "node:http2";
import {Http2ServerRequest} from "node:http2";
import {publicKeyExported} from "../security/OAuth2Provider";
import {jwtVerify} from "jose";
import {apikey} from './APIKeyHandler'
// a list of all functions
const version:MapArrayType<any> = {};
export const version:MapArrayType<any> = {};
version['1'] = {
createGroup: [],
@ -142,10 +142,9 @@ version['1.3.0'] = {
};
// set the latest available API version here
exports.latestApiVersion = '1.3.0';
export const latestApiVersion = '1.3.0';
// exports the versions so it can be used by the new Swagger endpoint
exports.version = version;
type APIFields = {
@ -163,7 +162,7 @@ type APIFields = {
* @param fields the params of the called function
* @param req express request object
*/
exports.handle = async function (apiVersion: string, functionName: string, fields: APIFields,
export const handle = async function (apiVersion: string, functionName: string, fields: APIFields,
req: Http2ServerRequest) {
// say goodbye if this is an unknown API version
if (!(apiVersion in version)) {
@ -197,19 +196,20 @@ exports.handle = async function (apiVersion: string, functionName: string, field
// sanitize any padIDs before continuing
if (fields.padID) {
fields.padID = await padManager.sanitizePadId(fields.padID);
fields.padID = await sanitizePadId(fields.padID);
}
// there was an 'else' here before - removed it to ensure
// that this sanitize step can't be circumvented by forcing
// the first branch to be taken
if (fields.padName) {
fields.padName = await padManager.sanitizePadId(fields.padName);
fields.padName = await sanitizePadId(fields.padName);
}
// put the function parameters in an array
// @ts-ignore
const functionParams = version[apiVersion][functionName].map((field) => fields[field]);
const functionParams = version[apiVersion][functionName].map((field: string) => fields[field]);
// call the api function
// @ts-ignore
return api[functionName].apply(this, functionParams);
};

View file

@ -1,15 +1,15 @@
const absolutePaths = require('../utils/AbsolutePaths');
import {makeAbsolute} from '../utils/AbsolutePaths';
import fs from 'fs';
import log4js from 'log4js';
const randomString = require('../utils/randomstring');
const argv = require('../utils/Cli').argv;
const settings = require('../utils/Settings');
import {randomString} from '../utils/randomstring';
import {argvP} from "../utils/Cli";
import settings from '../utils/Settings';
const apiHandlerLogger = log4js.getLogger('APIHandler');
// ensure we have an apikey
export let apikey:string|null = null;
const apikeyFilename = absolutePaths.makeAbsolute(argv.apikey || './APIKEY.txt');
const apikeyFilename = makeAbsolute(argvP.apikey || './APIKEY.txt');
if(settings.authenticationMethod === 'apikey') {

View file

@ -22,9 +22,9 @@
import {MapArrayType} from "../types/MapType";
import {SocketModule} from "../types/SocketModule";
const log4js = require('log4js');
const settings = require('../utils/Settings');
const stats = require('../../node/stats')
import log4js from 'log4js';
import settings from '../utils/Settings';
import {measuredCollection} from '../stats';
const logger = log4js.getLogger('socket.io');
@ -41,8 +41,8 @@ let io:any;
* @param {string} moduleName
* @param {Module} module
*/
exports.addComponent = (moduleName: string, module: SocketModule) => {
if (module == null) return exports.deleteComponent(moduleName);
export const addComponent = (moduleName: string, module: SocketModule) => {
if (module == null) return deleteComponent(moduleName);
components[moduleName] = module;
module.setSocketIO(io);
};
@ -51,13 +51,13 @@ exports.addComponent = (moduleName: string, module: SocketModule) => {
* removes a component
* @param {Module} moduleName
*/
exports.deleteComponent = (moduleName: string) => { delete components[moduleName]; };
export const deleteComponent = (moduleName: string) => { delete components[moduleName]; };
/**
* sets the socket.io and adds event functions for routing
* @param {Object} _io the socket.io instance
*/
exports.setSocketIO = (_io:any) => {
export const setSocketIO = (_io:any) => {
io = _io;
io.sockets.on('connection', (socket:any) => {
@ -96,7 +96,7 @@ exports.setSocketIO = (_io:any) => {
// when the last user disconnected. If your activePads is 0 and totalUsers is 0
// you can say, if there has been no active pads or active users for 10 minutes
// this instance can be brought out of a scaling cluster.
stats.gauge('lastDisconnect', () => Date.now());
measuredCollection.gauge('lastDisconnect', () => Date.now());
// tell all components about this disconnect
for (const i of Object.keys(components)) {
components[i].handleDisconnect(socket);

View file

@ -182,8 +182,7 @@ export const restartServer = async () => {
// starts listening to requests as reported in issue #158. Not installing the log4js connect
// logger when the log level has a higher severity than INFO since it would not log at that level
// anyway.
// @ts-ignore
if (!(loglevel === 'WARN' && loglevel === 'ERROR')) {
if (!(settings.loglevel === 'WARN' || settings.loglevel === 'ERROR')) {
app.use(log4js.connectLogger(logger, {
level: log4js.levels.DEBUG.levelStr,
format: ':status, :method :url',
@ -263,6 +262,6 @@ export const restartServer = async () => {
logger.info('HTTP server listening for connections');
};
export const shutdown = async (hookName:string, context: any) => {
export const shutdown = async () => {
await closeServer();
};

View file

@ -5,7 +5,7 @@ import fs from "fs";
import * as url from "node:url";
import {MapArrayType} from "../../types/MapType";
const settings = require('ep_etherpad-lite/node/utils/Settings');
import settings from 'ep_etherpad-lite/node/utils/Settings';
const ADMIN_PATH = path.join(settings.root, 'src', 'templates');
const PROXY_HEADER = "x-proxy-path"

View file

@ -20,13 +20,13 @@ import {ErrorCaused} from "../../types/ErrorCaused";
const OpenAPIBackend = require('openapi-backend').default;
const IncomingForm = require('formidable').IncomingForm;
const cloneDeep = require('lodash.clonedeep');
const createHTTPError = require('http-errors');
import cloneDeep from 'lodash.clonedeep';
import createHTTPError from 'http-errors';
const apiHandler = require('../../handler/APIHandler');
const settings = require('../../utils/Settings');
import {handle, latestApiVersion, version as apiVersion} from '../../handler/APIHandler';
import settings from '../../utils/Settings';
const log4js = require('log4js');
import log4js from 'log4js';
const logger = log4js.getLogger('API');
// https://github.com/OAI/OpenAPI-Specification/tree/master/schemas/v3.0
@ -48,7 +48,7 @@ const info = {
name: 'Apache 2.0',
url: 'https://www.apache.org/licenses/LICENSE-2.0.html',
},
version: apiHandler.latestApiVersion,
version: latestApiVersion,
};
const APIPathStyle = {
@ -401,6 +401,7 @@ for (const [resource, actions] of Object.entries(resources)) {
// add response objects
const responses:OpenAPISuccessResponse = {...defaultResponseRefs};
if (responseSchema) {
// @ts-ignore
responses[200] = cloneDeep(defaultResponses.Success);
responses[200].content!['application/json'].schema.properties.data = {
type: 'object',
@ -504,7 +505,7 @@ const generateDefinitionForVersion = (version:string, style = APIPathStyle.FLAT)
};
// build operations
for (const funcName of Object.keys(apiHandler.version[version])) {
for (const funcName of Object.keys(apiVersion[version])) {
let operation:OpenAPIOperations = {};
if (operations[funcName]) {
operation = {...operations[funcName]};
@ -518,7 +519,7 @@ const generateDefinitionForVersion = (version:string, style = APIPathStyle.FLAT)
// set parameters
operation.parameters = operation.parameters || [];
for (const paramName of apiHandler.version[version][funcName]) {
for (const paramName of apiVersion[version][funcName]) {
operation.parameters.push({$ref: `#/components/parameters/${paramName}`});
// @ts-ignore
if (!definition.components.parameters[paramName]) {
@ -559,7 +560,7 @@ const generateDefinitionForVersion = (version:string, style = APIPathStyle.FLAT)
exports.expressPreSession = async (hookName:string, {app}:any) => {
// create openapi-backend handlers for each api version under /api/{version}/*
for (const version of Object.keys(apiHandler.version)) {
for (const version of Object.keys(apiVersion)) {
// we support two different styles of api: flat + rest
// TODO: do we really want to support both?
@ -577,7 +578,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
});
// serve latest openapi definition file under /api/openapi.json
const isLatestAPIVersion = version === apiHandler.latestApiVersion;
const isLatestAPIVersion = version === latestApiVersion;
if (isLatestAPIVersion) {
app.get(`/${style}/openapi.json`, (req:any, res:any) => {
res.header('Access-Control-Allow-Origin', '*');
@ -605,7 +606,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
});
// register operation handlers
for (const funcName of Object.keys(apiHandler.version[version])) {
for (const funcName of Object.keys(apiVersion[version])) {
const handler = async (c: any, req:any, res:any) => {
// parse fields from request
const {headers, params, query} = c.request;
@ -630,7 +631,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
// pass to api handler
let data;
try {
data = await apiHandler.handle(version, funcName, fields, req, res);
data = await handle(version, funcName, fields, req);
} catch (err) {
const errCaused = err as ErrorCaused
// convert all errors to http errors
@ -645,7 +646,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
// an unknown error happened
// log it and throw internal error
logger.error(errCaused.stack || errCaused.toString());
throw new createHTTPError.InternalError('internal error');
throw new createHTTPError.InternalServerError('internal error');
}
}

View file

@ -3,14 +3,14 @@
import {ArgsExpressType} from "../../types/ArgsExpressType";
import events from 'events';
const express = require('../express');
import {sessionMiddleware} from '../express';
import log4js from 'log4js';
const proxyaddr = require('proxy-addr');
const settings = require('../../utils/Settings');
import proxyaddr from 'proxy-addr';
import settings from '../../utils/Settings';
import {Server, Socket} from 'socket.io'
const socketIORouter = require('../../handler/SocketIORouter');
const hooks = require('../../../static/js/pluginfw/hooks');
const padMessageHandler = require('../../handler/PadMessageHandler');
import {addComponent, setSocketIO} from '../../handler/SocketIORouter';
import {callAll} from '../../../static/js/pluginfw/hooks';
import * as padMessageHandler from '../../handler/PadMessageHandler';
let io:any;
const logger = log4js.getLogger('socket.io');
@ -62,7 +62,7 @@ const socketSessionMiddleware = (args: any) => (socket: any, next: Function) =>
// socketio.js-client on node.js doesn't support cookies, so pass them via a query parameter.
req.headers.cookie = socket.handshake.query.cookie;
}
express.sessionMiddleware(req, {}, next);
sessionMiddleware(req, {}, next);
};
export const expressCreateServer = (hookName:string, args:ArgsExpressType, cb:Function) => {
@ -71,6 +71,7 @@ export const expressCreateServer = (hookName:string, args:ArgsExpressType, cb:Fu
// transports in this list at once
// e.g. XHR is disabled in IE by default, so in IE it should use jsonp-polling
io = new Server(args.server,{
// @ts-ignore
transports: settings.socketTransportProtocols,
cookie: false,
maxHttpBufferSize: settings.socketIo.maxHttpBufferSize,
@ -133,10 +134,10 @@ export const expressCreateServer = (hookName:string, args:ArgsExpressType, cb:Fu
// if(settings.minify) io.enable('browser client minification');
// Initialize the Socket.IO Router
socketIORouter.setSocketIO(io);
socketIORouter.addComponent('pad', padMessageHandler);
setSocketIO(io);
addComponent('pad', padMessageHandler);
hooks.callAll('socketio', {app: args.app, io, server: args.server});
callAll('socketio', {app: args.app, io, server: args.server});
return cb();
};

View file

@ -1,15 +1,15 @@
'use strict';
import path from 'node:path';
const eejs = require('../../eejs')
import {requireP} from '../../eejs';
import fs from 'node:fs';
const fsp = fs.promises;
const toolbar = require('../../utils/toolbar');
const hooks = require('../../../static/js/pluginfw/hooks');
const settings = require('../../utils/Settings');
import toolbar from '../../utils/toolbar';
import {callAll} from '../../../static/js/pluginfw/hooks';
import settings from '../../utils/Settings';
import util from 'node:util';
const webaccess = require('./webaccess');
const plugins = require('../../../static/js/pluginfw/plugin_defs');
import {userCanModify} from './webaccess';
import {pluginDefs} from '../../../static/js/pluginfw/plugin_defs';
import {build, buildSync} from 'esbuild'
let ioI: { sockets: { sockets: any[]; }; } | null = null
@ -35,12 +35,12 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
});
app.get('/javascript', (req:any, res:any) => {
res.send(eejs.require('ep_etherpad-lite/templates/javascript.html', {req}));
res.send(requireP('ep_etherpad-lite/templates/javascript.html', {req}));
});
app.get('/robots.txt', (req:any, res:any) => {
let filePath =
path.join(settings.root, 'src', 'static', 'skins', settings.skinName, 'robots.txt');
path.join(settings.root, 'src', 'static', 'skins', settings.skinName!, 'robots.txt');
res.sendFile(filePath, (err:any) => {
// there is no custom robots.txt, send the default robots.txt which dissallows all
if (err) {
@ -64,7 +64,7 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
const fns = [
...(settings.favicon ? [path.resolve(settings.root, settings.favicon)] : []),
path.join(settings.root, 'src', 'static', 'skins', settings.skinName, 'favicon.ico'),
path.join(settings.root, 'src', 'static', 'skins', settings.skinName!, 'favicon.ico'),
path.join(settings.root, 'src', 'static', 'favicon.ico'),
];
for (const fn of fns) {
@ -147,14 +147,14 @@ const handleLiveReload = async (args: any, padString: string, timeSliderString:
setRouteHandler("/p/:pad", (req: any, res: any, next: Function) => {
// The below might break for pads being rewritten
const isReadOnly = !webaccess.userCanModify(req.params.pad, req);
const isReadOnly = !userCanModify(req.params.pad, req);
hooks.callAll('padInitToolbar', {
callAll('padInitToolbar', {
toolbar,
isReadOnly
});
const content = eejs.require('ep_etherpad-lite/templates/pad.html', {
const content = requireP('ep_etherpad-lite/templates/pad.html', {
req,
toolbar,
isReadOnly,
@ -176,14 +176,14 @@ const handleLiveReload = async (args: any, padString: string, timeSliderString:
setRouteHandler("/p/:pad/timeslider", (req: any, res: any, next: Function) => {
console.log("Reloading pad")
// The below might break for pads being rewritten
const isReadOnly = !webaccess.userCanModify(req.params.pad, req);
const isReadOnly = !userCanModify(req.params.pad, req);
hooks.callAll('padInitToolbar', {
callAll('padInitToolbar', {
toolbar,
isReadOnly
});
const content = eejs.require('ep_etherpad-lite/templates/timeslider.html', {
const content = requireP('ep_etherpad-lite/templates/timeslider.html', {
req,
toolbar,
isReadOnly,
@ -230,14 +230,14 @@ const convertTypescriptWatched = (content: string, cb: (output:string, hash: str
exports.expressCreateServer = async (hookName: string, args: any, cb: Function) => {
// serve index.html under /
args.app.get('/', (req: any, res: any) => {
res.send(eejs.require('ep_etherpad-lite/templates/index.html', {req}));
res.send(requireP('ep_etherpad-lite/templates/index.html', {req}));
});
const padString = eejs.require('ep_etherpad-lite/templates/padBootstrap.js', {
const padString = requireP('ep_etherpad-lite/templates/padBootstrap.js', {
pluginModules: (() => {
const pluginModules = new Set();
for (const part of plugins.parts) {
for (const part of pluginDefs.getParts()) {
for (const [, hookFnName] of Object.entries(part.client_hooks || {})) {
// @ts-ignore
pluginModules.add(hookFnName.split(':')[0]);
@ -248,10 +248,10 @@ exports.expressCreateServer = async (hookName: string, args: any, cb: Function)
settings,
})
const timeSliderString = eejs.require('ep_etherpad-lite/templates/timeSliderBootstrap.js', {
const timeSliderString = requireP('ep_etherpad-lite/templates/timeSliderBootstrap.js', {
pluginModules: (() => {
const pluginModules = new Set();
for (const part of plugins.parts) {
for (const part of pluginDefs.getParts()) {
for (const [, hookFnName] of Object.entries(part.client_hooks || {})) {
// @ts-ignore
pluginModules.add(hookFnName.split(':')[0]);
@ -297,14 +297,14 @@ exports.expressCreateServer = async (hookName: string, args: any, cb: Function)
// serve pad.html under /p
args.app.get('/p/:pad', (req: any, res: any, next: Function) => {
// The below might break for pads being rewritten
const isReadOnly = !webaccess.userCanModify(req.params.pad, req);
const isReadOnly = !userCanModify(req.params.pad, req);
hooks.callAll('padInitToolbar', {
callAll('padInitToolbar', {
toolbar,
isReadOnly
});
const content = eejs.require('ep_etherpad-lite/templates/pad.html', {
const content = requireP('ep_etherpad-lite/templates/pad.html', {
req,
toolbar,
isReadOnly,
@ -315,11 +315,11 @@ exports.expressCreateServer = async (hookName: string, args: any, cb: Function)
// serve timeslider.html under /p/$padname/timeslider
args.app.get('/p/:pad/timeslider', (req: any, res: any, next: Function) => {
hooks.callAll('padInitToolbar', {
callAll('padInitToolbar', {
toolbar,
});
res.send(eejs.require('ep_etherpad-lite/templates/timeslider.html', {
res.send(requireP('ep_etherpad-lite/templates/timeslider.html', {
req,
toolbar,
entrypoint: "/"+fileNameTimeSlider

View file

@ -4,10 +4,10 @@ import {MapArrayType} from "../../types/MapType";
import {PartType} from "../../types/PartType";
const fs = require('fs').promises;
const minify = require('../../utils/Minify');
const path = require('path');
const plugins = require('../../../static/js/pluginfw/plugin_defs');
const settings = require('../../utils/Settings');
import {minify} from '../../utils/Minify';
import path from 'path';
import {pluginDefs} from '../../../static/js/pluginfw/plugin_defs';
import settings from '../../utils/Settings';
import CachingMiddleware from '../../utils/caching_middleware';
// Rewrite tar to include modules with no extensions and proper rooted paths.
@ -40,13 +40,15 @@ exports.expressPreSession = async (hookName:string, {app}:any) => {
// Minify will serve static files compressed (minify enabled). It also has
// file-specific hacks for ace/require-kernel/etc.
app.all('/static/:filename(*)', minify.minify);
app.all('/static/:filename(*)', (req: Request, res: Response, next: Function)=>{
minify(req,res, next)
});
// serve plugin definitions
// not very static, but served here so that client can do
// require("pluginfw/static/js/plugin-definitions.js");
app.get('/pluginfw/plugin-definitions.json', (req: any, res:any, next:Function) => {
const clientParts = plugins.parts.filter((part: PartType) => part.client_hooks != null);
const clientParts = pluginDefs.getParts().filter((part: PartType) => part.client_hooks != null);
const clientPlugins:MapArrayType<string> = {};
for (const name of new Set(clientParts.map((part: PartType) => part.plugin))) {
// @ts-ignore

View file

@ -7,7 +7,7 @@ import path from 'path';
import {promises as fsp} from 'fs';
import {pluginDefs} from '../../../static/js/pluginfw/plugin_defs';
import sanitizePathname from '../../utils/sanitizePathname';
const settings = require('../../utils/Settings');
import settings from '../../utils/Settings';
// Returns all *.js files under specDir (recursively) as relative paths to specDir, using '/'
// instead of path.sep to separate pathname components.

View file

@ -4,12 +4,12 @@ import type {MapArrayType} from "../types/MapType";
import {I18nPluginDefs} from "../types/I18nPluginDefs";
const languages = require('languages4translatewiki');
const fs = require('fs');
const path = require('path');
const _ = require('underscore');
const pluginDefs = require('../../static/js/pluginfw/plugin_defs.js');
const existsSync = require('../utils/path_exists');
const settings = require('../utils/Settings');
import fs from 'fs';
import path from 'path';
import _ from 'underscore';
import {pluginDefs} from '../../static/js/pluginfw/plugin_defs.js';
import existsSync from '../utils/path_exists';
import settings from '../utils/Settings';
// returns all existing messages merged together and grouped by langcode
// {es: {"foo": "string"}, en:...}
@ -43,7 +43,7 @@ const getAllLocales = () => {
extractLangs(path.join(settings.root, 'src/locales'));
// add plugins languages (if any)
for (const {package: {path: pluginPath}} of Object.values<I18nPluginDefs>(pluginDefs.plugins)) {
for (const {package: {path: pluginPath}} of Object.values<I18nPluginDefs>(pluginDefs.getPlugins())) {
// plugin locales should overwrite etherpad's core locales
if (pluginPath.endsWith('/ep_etherpad-lite')) continue;
extractLangs(path.join(pluginPath, 'locales'));

View file

@ -1,6 +1,7 @@
'use strict';
const measured = require('measured-core');
// @ts-ignore
import measured from 'measured-core';
export const measuredCollection = measured.createCollection();

View file

@ -1,6 +1,6 @@
export type RunCMDOptions = {
cwd?: string,
stdio?: string[]|null[],
stdio?: (string|null)[]
env?: NodeJS.ProcessEnv
}

View file

@ -21,20 +21,20 @@
* limitations under the License.
*/
const settings = require('./Settings');
const fs = require('fs').promises;
const path = require('path');
const plugins = require('../../static/js/pluginfw/plugin_defs');
const mime = require('mime-types');
const Threads = require('threads');
const log4js = require('log4js');
const sanitizePathname = require('./sanitizePathname');
import settings from './Settings';
import {promises as fs} from 'fs'
import path from 'path';
import {pluginDefs} from '../../static/js/pluginfw/plugin_defs';
import mime from 'mime-types';
import log4js from 'log4js';
import sanitizePathname from './sanitizePathname';
import {MapArrayType} from "../types/MapType";
import {compressCSS, compressJS} from "./MinifyWorker";
const logger = log4js.getLogger('Minify');
const ROOT_DIR = path.join(settings.root, 'src/static/');
const threadsPool = new Threads.Pool(() => Threads.spawn(new Threads.Worker('./MinifyWorker')), 2);
const LIBRARY_WHITELIST = [
'async',
@ -48,10 +48,10 @@ const LIBRARY_WHITELIST = [
// What follows is a terrible hack to avoid loop-back within the server.
// TODO: Serve files from another service, or directly from the file system.
const requestURI = async (url, method, headers) => {
const requestURI = async (url: string, method: string, headers: MapArrayType<any>) => {
const parsedUrl = new URL(url);
let status = 500;
const content = [];
const content: string[] = [];
const mockRequest = {
url,
method,
@ -72,7 +72,7 @@ const requestURI = async (url, method, headers) => {
setHeader: (header, value) => {
headers[header.toLowerCase()] = value.toString();
},
header: (header, value) => {
header: (header: string, value: string) => {
headers[header.toLowerCase()] = value.toString();
},
write: (_content) => {
@ -84,20 +84,21 @@ const requestURI = async (url, method, headers) => {
},
};
});
await minify(mockRequest, mockResponse);
await _minify(mockRequest, mockResponse);
return await p;
};
const requestURIs = (locations, method, headers, callback) => {
Promise.all(locations.map(async (loc) => {
export const requestURIs = (locations: string[], method: string, headers: MapArrayType<any>, callback: (arg0: any[], arg1: any[], arg2: any[]) => void) => {
Promise.all<[number, MapArrayType<any>, string]>(locations.map(async (loc: string) => {
try {
return await requestURI(loc, method, headers);
return await requestURI(loc, method, headers) as [number, MapArrayType<any>, string];
} catch (err) {
logger.debug(`requestURI(${JSON.stringify(loc)}, ${JSON.stringify(method)}, ` +
`${JSON.stringify(headers)}) failed: ${err.stack || err}`);
return [500, headers, ''];
// @ts-ignore
`${JSON.stringify(headers)}) failed: ${err.stack || err}`);
return [500, headers, ''] as [number, MapArrayType<any>, string] ;
}
})).then((responses) => {
})).then((responses ) => {
const statuss = responses.map((x) => x[0]);
const headerss = responses.map((x) => x[1]);
const contentss = responses.map((x) => x[2]);
@ -119,11 +120,12 @@ const compatPaths = {
* @param req the Express request
* @param res the Express response
*/
const minify = async (req, res) => {
const _minify = async (req: any, res: any) => {
let filename = req.params.filename;
try {
filename = sanitizePathname(filename);
} catch (err) {
// @ts-ignore
logger.error(`sanitization of pathname "${filename}" failed: ${err.stack || err}`);
res.writeHead(404, {});
res.end();
@ -131,6 +133,7 @@ const minify = async (req, res) => {
}
// Backward compatibility for plugins that require() files from old paths.
// @ts-ignore
const newLocation = compatPaths[filename.replace(/^plugins\/ep_etherpad-lite\/static\//, '')];
if (newLocation != null) {
logger.warn(`request for deprecated path "${filename}", replacing with "${newLocation}"`);
@ -147,8 +150,8 @@ const minify = async (req, res) => {
const library = match[1];
const libraryPath = match[2] || '';
if (plugins.plugins[library] && match[3]) {
const plugin = plugins.plugins[library];
if (pluginDefs.getPlugins()[library] && match[3]) {
const plugin = pluginDefs.getPlugins()[library];
const pluginPath = plugin.package.realPath;
filename = path.join(pluginPath, libraryPath);
// On Windows, path.relative converts forward slashes to backslashes. Convert them back
@ -185,7 +188,7 @@ const minify = async (req, res) => {
if (!exists) {
res.writeHead(404, {});
res.end();
} else if (new Date(req.headers['if-modified-since']) >= date) {
} else if (new Date(req.headers['if-modified-since']) >= date!) {
res.writeHead(304, {});
res.end();
} else if (req.method === 'HEAD') {
@ -205,7 +208,7 @@ const minify = async (req, res) => {
};
// Check for the existance of the file and get the last modification date.
const statFile = async (filename, dirStatLimit) => {
const statFile = async (filename: string, dirStatLimit?: number): Promise<[Date|null,boolean]> => {
/*
* The only external call to this function provides an explicit value for
* dirStatLimit: this check could be removed.
@ -221,8 +224,10 @@ const statFile = async (filename, dirStatLimit) => {
try {
stats = await fs.stat(path.resolve(ROOT_DIR, filename));
} catch (err) {
// @ts-ignore
if (['ENOENT', 'ENOTDIR'].includes(err.code)) {
// Stat the directory instead.
// @ts-ignore
const [date] = await statFile(path.dirname(filename), dirStatLimit - 1);
return [date, false];
}
@ -232,63 +237,57 @@ const statFile = async (filename, dirStatLimit) => {
}
};
const getFileCompressed = async (filename, contentType) => {
const getFileCompressed = async (filename: string, contentType: string|false) => {
let content = await getFile(filename);
if (!content || !settings.minify) {
return content;
} else if (contentType === 'application/javascript') {
return await new Promise((resolve) => {
threadsPool.queue(async ({compressJS}) => {
try {
logger.info('Compress JS file %s.', filename);
let jsSources = ''
return await new Promise(async (resolve) => {
try {
logger.info('Compress JS file %s.', filename);
content = content.toString();
const compressResult = await compressJS(content);
const compressResult = await compressJS(content.toString());
if (compressResult.error) {
console.error(`Error compressing JS (${filename}) using terser`, compressResult.error);
} else {
content = compressResult.code.toString(); // Convert content obj code to string
}
} catch (error) {
console.error('getFile() returned an error in ' +
`getFileCompressed(${filename}, ${contentType}): ${error}`);
if (compressResult.warnings) {
console.error(`Error compressing JS (${filename}) using terser`, compressResult.warnings);
} else {
jsSources = compressResult.code.toString(); // Convert content obj code to string
}
resolve(content);
});
} catch (error) {
console.error('getFile() returned an error in ' +
`getFileCompressed(${filename}, ${contentType}): ${error}`);
}
resolve(content.toString());
});
} else if (contentType === 'text/css') {
return await new Promise((resolve) => {
threadsPool.queue(async ({compressCSS}) => {
try {
logger.info('Compress CSS file %s.', filename);
let contentString = ''
return await new Promise(async (resolve) => {
try {
logger.info('Compress CSS file %s.', filename);
const compressResult = await compressCSS(path.resolve(ROOT_DIR,filename));
const compressResult = await compressCSS(path.resolve(ROOT_DIR, filename));
if (compressResult.error) {
console.error(`Error compressing CSS (${filename}) using terser`, compressResult.error);
} else {
content = compressResult
}
} catch (error) {
console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`);
if (compressResult) {
console.error(`Error compressing CSS (${filename}) using terser`, compressResult);
} else {
contentString = compressResult
}
resolve(content);
});
} catch (error) {
console.error(`CleanCSS.minify() returned an error on ${filename}: ${error}`);
}
resolve(content.toString());
});
} else {
return content;
}
};
const getFile = async (filename) => {
const getFile = async (filename: string) => {
return await fs.readFile(path.resolve(ROOT_DIR, filename));
};
exports.minify = (req, res, next) => minify(req, res).catch((err) => next(err || new Error(err)));
export const minify = (req: any, res:any, next:Function) => _minify(req, res).catch((err) => next(err || new Error(err)));
exports.requestURIs = requestURIs;
exports.shutdown = async (hookName, context) => {
await threadsPool.terminate();
export const shutdown = async () => {
};

View file

@ -1,16 +1,14 @@
'use strict';
/**
* Worker thread to minify JS & CSS files out of the main NodeJS thread
*/
import {expose} from 'threads'
import {build, transform} from 'esbuild';
/*
* Minify JS content
* @param {string} content - JS content to minify
*/
const compressJS = async (content) => {
export const compressJS = async (content: string) => {
return await transform(content, {minify: true});
}
@ -19,7 +17,7 @@ const compressJS = async (content) => {
* @param {string} filename - name of the file
* @param {string} ROOT_DIR - the root dir of Etherpad
*/
const compressCSS = async (content) => {
export const compressCSS = async (content: string) => {
const transformedCSS = await build(
{
entryPoints: [content],
@ -39,7 +37,4 @@ const compressCSS = async (content) => {
return transformedCSS.outputFiles[0].text
};
expose({
compressJS: compressJS,
compressCSS,
});

View file

@ -66,14 +66,14 @@ class Settings {
private defaultLogLevel = 'INFO';
private logger = log4js.getLogger('settings');
/* Root path of the installation */
private root = findEtherpadRoot();
root = findEtherpadRoot();
/**
* Pathname of the favicon you want to use. If null, the skin's favicon is
* used if one is provided by the skin, otherwise the default Etherpad favicon
* is used. If this is a relative path it is interpreted as relative to the
* Etherpad root directory.
*/
private favicon: string|null = null;
favicon: string|null = null;
// Exported values that settings.json and credentials.json cannot override.
private nonSettings = [
'credentialsFilename',
@ -131,8 +131,8 @@ class Settings {
/**
* socket.io transport methods
**/
private socketTransportProtocols = ['websocket', 'polling'];
private socketIo = {
socketTransportProtocols: ("polling"| "websocket"|"webtransport")[] = ['websocket', 'polling'];
socketIo = {
/**
* Maximum permitted client message size (in bytes).
*
@ -149,20 +149,20 @@ class Settings {
The default value is sso
If you want to use the old authentication system, change this to apikey
*/
private authenticationMethod = 'sso'
authenticationMethod = 'sso'
/*
* The Type of the database
*/
private dbType = 'dirty';
dbType = 'dirty';
/**
* This setting is passed with dbType to ueberDB to set up the database
*/
private dbSettings = {filename: path.join(this.root, 'var/dirty.db')};
dbSettings = {filename: path.join(this.root, 'var/dirty.db')};
/**
* The default Text of a new pad
*/
private defaultPadText = [
defaultPadText = [
'Welcome to Etherpad!',
'',
'This pad text is synchronized as you type, so that everyone viewing this page sees the same ' +
@ -240,22 +240,22 @@ class Settings {
/**
* A flag that requires any user to have a valid session (via the api) before accessing a pad
*/
private requireSession = false;
requireSession = false;
/**
* A flag that prevents users from creating new pads
*/
private editOnly = false;
editOnly = false;
/**
* Max age that responses will have (affects caching layer).
*/
private maxAge = 1000 * 60 * 60 * 6; // 6 hours
maxAge = 1000 * 60 * 60 * 6; // 6 hours
/**
* A flag that shows if minification is enabled or not
*/
private minify = true;
minify = true;
/**
* The path of the abiword executable
@ -275,7 +275,7 @@ class Settings {
/**
* The log level of log4js
*/
private loglevel: string = this.defaultLogLevel;
loglevel: string = this.defaultLogLevel;
@ -292,7 +292,7 @@ class Settings {
/**
* Disable Load Testing
*/
private loadTest = false;
loadTest = false;
/**
* Disable dump of objects preventing a clean exit
@ -345,14 +345,14 @@ class Settings {
* authorization. Note: /admin always requires authentication, and
* either authorization by a module, or a user with is_admin set
*/
private requireAuthentication = false;
requireAuthentication = false;
private requireAuthorization = false;
users = {};
/*
* This setting is used for configuring sso
*/
private sso = {
sso = {
issuer: "http://localhost:9001"
}
@ -403,7 +403,7 @@ class Settings {
/*
* Override any strings found in locale directories
*/
private customLocaleStrings = {};
customLocaleStrings = {};
/*
* From Etherpad 1.8.3 onwards, import and export of pads is always rate
@ -816,7 +816,7 @@ class Settings {
}
}
private reloadSettings = () => {
reloadSettings = () => {
const settings = this.parseSettings(this.settingsFilename, true);
const credentials = this.parseSettings(this.credentialsFilename, false);
this.storeSettings(settings);

View file

@ -1,11 +1,11 @@
'use strict';
import semver from 'semver';
import {getEpVersion} from './Settings';
import settings from './Settings';
import axios from 'axios';
const headers = {
'User-Agent': 'Etherpad/' + getEpVersion(),
'User-Agent': 'Etherpad/' + settings.getEpVersion(),
}
type Infos = {
@ -47,7 +47,7 @@ export const getLatestVersion = () => {
export const needsUpdate = async (cb?: Function) => {
try {
const info = await loadEtherpadInformations()
if (semver.gt(info!.latestVersion, getEpVersion())) {
if (semver.gt(info!.latestVersion, settings.getEpVersion())) {
if (cb) return cb(true);
}
} catch (err) {

View file

@ -21,7 +21,7 @@ import fs from 'fs';
const fsp = fs.promises;
import path from 'path';
import zlib from 'zlib';
const settings = require('./Settings');
import settings from './Settings';
import existsSync from './path_exists';
import util from 'util';

View file

@ -8,7 +8,7 @@ import {Readable} from "node:stream";
import spawn from 'cross-spawn';
import log4js from 'log4js';
import path from 'path';
const settings = require('./Settings');
import settings from './Settings';
const logger = log4js.getLogger('runCmd');

View file

@ -94,9 +94,12 @@
"@types/jsdom": "^21.1.7",
"@types/jsonminify": "^0.4.3",
"@types/jsonwebtoken": "^9.0.6",
"@types/lodash.clonedeep": "^4.5.9",
"@types/mime-types": "^2.1.4",
"@types/mocha": "^10.0.7",
"@types/node": "^20.14.11",
"@types/oidc-provider": "^8.5.1",
"@types/proxy-addr": "^2.0.3",
"@types/resolve": "^1.20.6",
"@types/semver": "^7.5.8",
"@types/sinon": "^17.0.3",

View file

@ -4,7 +4,7 @@ import {node_modules, pluginInstallPath} from "./installer";
import {accessSync, constants, rmSync, symlinkSync, unlinkSync} from "node:fs";
import {dependencies, name} from '../../../package.json'
import {pathToFileURL} from 'node:url';
const settings = require('../../../node/utils/Settings');
import settings from '../../../node/utils/Settings';
import {readFileSync} from "fs";
import {IPluginInfoExtended} from "./IPluginInfoExtended";

View file

@ -10,13 +10,13 @@ import path from "path";
import {promises as fs} from "fs";
const plugins = require('./plugins');
const hooks = require('./hooks');
const runCmd = require('../../../node/utils/run_cmd');
const settings = require('../../../node/utils/Settings');
import {update, prefix, getPackages} from './plugins';
import {aCallAll} from './hooks';
import runCmd from '../../../node/utils/run_cmd';
import settings from '../../../node/utils/Settings';
import {LinkInstaller} from "./LinkInstaller";
const {findEtherpadRoot} = require('../../../node/utils/AbsolutePaths');
import {findEtherpadRoot} from '../../../node/utils/AbsolutePaths';
const logger = log4js.getLogger('plugins');
export const pluginInstallPath = path.join(settings.root, 'src','plugin_packages');
@ -25,11 +25,11 @@ export const node_modules = path.join(findEtherpadRoot(),'src', 'node_modules');
export const installedPluginsPath = path.join(settings.root, 'var/installed_plugins.json');
const onAllTasksFinished = async () => {
await plugins.update();
await update();
await persistInstalledPlugins();
settings.reloadSettings();
await hooks.aCallAll('loadSettings', {settings});
await hooks.aCallAll('restartServer');
await aCallAll('loadSettings', {settings});
await aCallAll('restartServer');
};
const headers = {
@ -62,7 +62,7 @@ const migratePluginsFromNodeModules = async () => {
{stdio: [null, 'string']}));
await Promise.all(Object.entries(dependencies)
.filter(([pkg, info]) => pkg.startsWith(plugins.prefix) && pkg !== 'ep_etherpad-lite')
.filter(([pkg, info]) => pkg.startsWith(prefix) && pkg !== 'ep_etherpad-lite')
.map(async ([pkg, info]) => {
const _info = info as PackageInfo
if (!_info.resolved) {
@ -120,7 +120,7 @@ export const checkForMigration = async () => {
const installedPlugins = JSON.parse(fileContent.toString());
for (const plugin of installedPlugins.plugins) {
if (plugin.name.startsWith(plugins.prefix) && plugin.name !== 'ep_etherpad-lite') {
if (plugin.name.startsWith(prefix) && plugin.name !== 'ep_etherpad-lite') {
await linkInstaller.installPlugin(plugin.name, plugin.version);
}
}
@ -130,7 +130,7 @@ const persistInstalledPlugins = async () => {
const installedPlugins:{
plugins: PackageData[]
} = {plugins: []};
for (const pkg of Object.values(await plugins.getPackages()) as PackageData[]) {
for (const pkg of Object.values(await getPackages()) as PackageData[]) {
installedPlugins.plugins.push({
name: pkg.name,
version: pkg.version,
@ -146,7 +146,7 @@ export const uninstall = async (pluginName: string, cb:Function|null = null) =>
await linkInstaller.uninstallPlugin(pluginName);
logger.info(`Successfully uninstalled plugin ${pluginName}`);
await hooks.aCallAll('pluginUninstall', {pluginName});
await aCallAll('pluginUninstall', {pluginName});
cb(null);
};
@ -155,7 +155,7 @@ export const install = async (pluginName: string, cb:Function|null = null) => {
logger.info(`Installing plugin ${pluginName}...`);
await linkInstaller.installPlugin(pluginName);
logger.info(`Successfully installed plugin ${pluginName}`);
await hooks.aCallAll('pluginInstall', {pluginName});
await aCallAll('pluginInstall', {pluginName});
cb(null);
};
@ -193,7 +193,7 @@ export const search = (searchTerm: string, maxCacheAge: number) => getAvailableP
for (const pluginName in results) {
// for every available plugin
// TODO: Also search in keywords here!
if (pluginName.indexOf(plugins.prefix) !== 0) continue;
if (pluginName.indexOf(prefix) !== 0) continue;
if (searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm) &&
(typeof results[pluginName].description !== 'undefined' &&

View file

@ -3,15 +3,14 @@
import {Part} from "./plugin_defs";
const fs = require('fs').promises;
const hooks = require('./hooks');
import {aCallAll} from './hooks';
import log4js from 'log4js';
import path from 'path';
const runCmd = require('../../../node/utils/run_cmd');
import runCmd from '../../../node/utils/run_cmd';
import {TSort} from './tsort';
const pluginUtils = require('./shared');
import {extractHooks} from './shared';
import {pluginDefs} from './plugin_defs';
import {IPluginInfo} from "live-plugin-manager";
const settings = require('../../../node/utils/Settings');
import settings from '../../../node/utils/Settings';
const logger = log4js.getLogger('plugins');
@ -116,12 +115,12 @@ export const update = async () => {
pluginDefs.setPlugins(plugins);
pluginDefs.setParts(sortParts(parts));
pluginDefs.setHooks(pluginUtils.extractHooks(pluginDefs.getParts(), 'hooks', pathNormalization))
pluginDefs.setHooks(extractHooks(pluginDefs.getParts(), 'hooks', pathNormalization)!)
pluginDefs.setLoaded(true);
await Promise.all(Object.keys(pluginDefs.getPlugins()).map(async (p) => {
const logger = log4js.getLogger(`plugin:${p}`);
await hooks.aCallAll(`init_${p}`, {logger});
await aCallAll(`init_${p}`, {logger});
}));
};

View file

@ -10,7 +10,7 @@ const disabledHookReasons: MapArrayType<any> = {
},
};
const loadFn = (path: string, hookName: string, modules: Function) => {
const loadFn = (path: string, hookName: string, modules?: Function) => {
let functionName;
const parts = path.split(':');
@ -41,7 +41,7 @@ const loadFn = (path: string, hookName: string, modules: Function) => {
return fn;
};
export const extractHooks = (parts: Part[], hookSetName: string, normalizer: Function|null, modules: Function) => {
export const extractHooks = (parts: Part[], hookSetName: string, normalizer: Function|null, modules?: Function) => {
const hooks: MapArrayType<PluginHook[]> = {};
for (const part of parts) {
// @ts-ignore