cleanup after workspace refactoring (#6174)

* fix bin folder and workflows as far its possible

cleanup of dockerfile

changed paths of scripts

add lock file

fix working directory for workflows

fix windows bin

fix travis (is travis used anyway?)

fix package refs

remove pnpm-lock file in root as these conflicts with the docker volume setup

optimize comments

use install again

refactor prod image call to run

fix --workspace can only be used inside a workspace

correct comment

try fix pipeline

try fix pipeline for upgrade-from-latest-release

install all deps

smaller adjustments

save

update dockerfile

remove workspace command

fix run test command

start repair latest release workflow

start repair latest release workflow

start repair latest release workflow

further repairs

* remove test plugin from docker compose

.github/workflows/backend-tests.yml

@@ -53,10 +53,10 @@ jobs:
           sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Run the backend tests
-        run: cd src && pnpm test
+        run: pnpm test
 
   withpluginsLinux:
     # run on pushes to any branch
@@ -102,10 +102,11 @@ jobs:
           sudo add-apt-repository -y ppa:libreoffice/ppa
           sudo apt update
           sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+      -
+        name: Install all dependencies and symlink for ep_etherpad-lite
+        run: bin/installDeps.sh
       -
         name: Install Etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm v7:
-        # https://github.com/npm/cli/issues/2199
         run: >
           pnpm install --workspace-root
           ep_align
@@ -121,21 +122,9 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
-      -
-        name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
       -
         name: Run the backend tests
-        run: cd src && pnpm test
+        run: pnpm test
 
   withoutpluginsWindows:
     # run on pushes to any branch
@@ -173,7 +162,7 @@ jobs:
         run: pnpm config set auto-install-peers false
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installOnWindows.bat
+        run: bin/installOnWindows.bat
       -
         name:  Fix up the settings.json
         run: |
@@ -248,7 +237,7 @@ jobs:
       # rules.
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installOnWindows.bat
+        run: bin/installOnWindows.bat
       -
         name:  Fix up the settings.json
         run: |

.github/workflows/docker.yml

@@ -60,7 +60,7 @@ jobs:
         name: Test
         run: |
           docker run --rm -d -p 9001:9001 --name test ${{ env.TEST_TAG }}
-          ./src/bin/installDeps.sh
+          ./bin/installDeps.sh
           docker logs -f test &
           while true; do
             echo "Waiting for Docker container to start..."

.github/workflows/frontend-admin-tests.yml

@@ -67,7 +67,7 @@ jobs:
       # rules.
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Install etherpad plugins
         run: rm -Rf node_modules/ep_align/static/tests/*

.github/workflows/frontend-tests.yml

@@ -46,7 +46,7 @@ jobs:
         run: pnpm config set auto-install-peers false
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: export GIT_HASH to env
         id: environment
@@ -142,7 +142,7 @@ jobs:
       # rules.
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: export GIT_HASH to env
         id: environment

.github/workflows/load-test.yml

@@ -43,7 +43,7 @@ jobs:
         run: pnpm config set auto-install-peers false
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Install etherpad-load-test
         run: sudo npm install -g etherpad-load-test-socket-io
@@ -117,7 +117,7 @@ jobs:
       # rules.
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Run load test
         run: src/tests/frontend/travis/runnerLoadTest.sh 25 50
@@ -158,7 +158,7 @@ jobs:
         run: pnpm config set auto-install-peers false
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Install etherpad-load-test
         run: sudo npm install -g etherpad-load-test-socket-io

.github/workflows/perform-type-check.yml

@@ -41,7 +41,6 @@ jobs:
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
         run: ./bin/installDeps.sh
-        working-directory: ./src
       - name: Perform type check
         working-directory: ./src
         run: npm run ts-check

.github/workflows/rate-limit.yml

@@ -57,7 +57,7 @@ jobs:
           docker run --rm --network ep_net --ip 172.23.42.3 --name anotherip -dt anotherip
       -
         name: install dependencies and create symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: run rate limit test
         run: |

.github/workflows/upgrade-from-latest-release.yml

@@ -29,30 +29,13 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
-      - uses: pnpm/action-setup@v3
-        name: Install pnpm
-        with:
-          version: 8
-          run_install: false
-      - name: Get pnpm store directory
-        shell: bash
-        run: |
-          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-      - uses: actions/cache@v4
-        name: Setup pnpm cache
-        with:
-          path: ${{ env.STORE_PATH }}
-          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-pnpm-store-
-      - name: Only install direct dependencies
-        run: pnpm config set auto-install-peers false
       -
         name: Install Etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm
+        # Important: Installer for old master which does not have pnpm right now
-        # v7: https://github.com/npm/cli/issues/2199
+        # The --legacy-peer-deps flag is required to work around a bug in npm v7:
+        # https://github.com/npm/cli/issues/2199
         run: >
-          pnpm install --workspace root
+          npm install --no-save --legacy-peer-deps
           ep_align
           ep_author_hover
           ep_cursortrace
@@ -66,21 +49,12 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
         run: src/bin/installDeps.sh
       -
         name: Run the backend tests
-        run: cd src && pnpm test
+        run: cd src && npm test
       # Because actions/checkout@v4 is called with "ref: master" and without
       # "fetch-depth: 0", the local clone does not have the ${GITHUB_SHA}
       # commit. Fetch ${GITHUB_REF} to get the ${GITHUB_SHA} commit. Note that a
@@ -96,13 +70,36 @@ jobs:
         # For pull requests, ${GITHUB_SHA} is the automatically generated merge
         # commit that merges the PR's source branch to its destination branch.
         run: git checkout "${GITHUB_SHA}"
+      - uses: pnpm/action-setup@v3
+        name: Install pnpm
+        with:
+          version: 8
+          run_install: false
+      - name: Only install direct dependencies
+        run: pnpm config set auto-install-peers false
+      -
+        name: Install libreoffice
+        run: |
+          sudo add-apt-repository -y ppa:libreoffice/ppa
+          sudo apt update
+          sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+      - name: Get pnpm store directory
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Run the backend tests
-        working-directory: ./src
+        run: pnpm test
-        run: npm test
       -
         name: Install Cypress
         working-directory: ./src
@@ -111,6 +108,6 @@ jobs:
         name: Run Etherpad & Test Frontend
         working-directory: ./src
         run: |
-          npm run dev &
+          pnpm run dev &
           curl --connect-timeout 10 --max-time 20 --retry 5 --retry-delay 10 --retry-max-time 60 --retry-connrefused http://127.0.0.1:9001/p/test
           ./node_modules/cypress/bin/cypress run --config-file tests/frontend/cypress/cypress.config.js

.github/workflows/windows.yml

@@ -50,7 +50,7 @@ jobs:
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
         shell: msys2 {0}
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Run the backend tests
         shell: msys2 {0}
@@ -58,7 +58,7 @@ jobs:
       -
         name: Build the .zip
         shell: msys2 {0}
-        run: src/bin/buildForWindows.sh
+        run: bin/buildForWindows.sh
       -
         name: Archive production artifacts
         uses: actions/upload-artifact@v4
@@ -91,7 +91,7 @@ jobs:
         name: Create installer
         uses: joncloud/makensis-action@v4.1
         with:
-          script-file: 'src/bin/nsis/etherpad.nsi'
+          script-file: 'bin/nsis/etherpad.nsi'
       -
         name: Archive production artifacts
         uses: actions/upload-artifact@v4
@@ -143,7 +143,7 @@ jobs:
         run: pnpm config set auto-install-peers false
       - name: Install all dependencies and symlink for ep_etherpad-lite
         run: .\bin\installOnWindows.bat
-        working-directory: etherpad/src
+        working-directory: etherpad
       -
         name: Run Etherpad
         working-directory: etherpad/src

.gitignore

@@ -22,5 +22,4 @@ out/
 /src/bin/convertSettings.json
 /src/bin/etherpad-1.deb
 /src/bin/node.exe
-plugin_packages
+plugin_packages
-/pnpm-lock.yaml

.travis.yml

@@ -55,7 +55,7 @@ jobs:
         - *set_loglevel_warn
         - *enable_admin_tests
         - "src/tests/frontend/travis/sauce_tunnel.sh"
-        - "src/bin/installDeps.sh"
+        - "bin/installDeps.sh"
         - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
       script:
         - "./src/tests/frontend/travis/runner.sh"
@@ -63,7 +63,7 @@ jobs:
       install:
         - *install_libreoffice
         - *set_loglevel_warn
-        - "src/bin/installDeps.sh"
+        - "bin/installDeps.sh"
         - "cd src && pnpm install && cd -"
       script:
         - "cd src && pnpm test"
@@ -77,7 +77,7 @@ jobs:
     - name: "Load test Etherpad without Plugins"
       install:
         - *set_loglevel_warn
-        - "src/bin/installDeps.sh"
+        - "bin/installDeps.sh"
         - "cd src && pnpm install && cd -"
         - "npm install -g etherpad-load-test"
       script:
@@ -90,7 +90,7 @@ jobs:
         - *set_loglevel_warn
         - *enable_admin_tests
         - "src/tests/frontend/travis/sauce_tunnel.sh"
-        - "src/bin/installDeps.sh"
+        - "bin/installDeps.sh"
         - "rm src/tests/frontend/specs/*"
         - *install_plugins
         - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
@@ -105,7 +105,7 @@ jobs:
       install:
         - *install_libreoffice
         - *set_loglevel_warn
-        - "src/bin/installDeps.sh"
+        - "bin/installDeps.sh"
         - *install_plugins
         - "cd src && pnpm install && cd -"
       script:
@@ -120,7 +120,7 @@ jobs:
     - name: "Load test Etherpad with Plugins"
       install:
         - *set_loglevel_warn
-        - "src/bin/installDeps.sh"
+        - "bin/installDeps.sh"
         - *install_plugins
         - "cd src && npm install && cd -"
         - "npm install -g etherpad-load-test"
@@ -135,7 +135,7 @@ jobs:
         - "docker run -p 8081:80 --rm --network ep_net --ip 172.23.42.1 -d nginx-latest"
         - "docker run --name etherpad-docker -p 9000:9001 --rm --network ep_net --ip 172.23.42.2 -e 'TRUST_PROXY=true' epl-debian-slim &"
         - "docker run --rm --network ep_net --ip 172.23.42.3 --name anotherip -dt anotherip"
-        - "./src/bin/installDeps.sh"
+        - "./bin/installDeps.sh"
       script:
         - "cd src/tests/ratelimit && bash testlimits.sh"
 

Dockerfile

@@ -88,40 +88,25 @@ COPY --chown=etherpad:etherpad ./.git/HEAD ./.git/HEAD
 COPY --chown=etherpad:etherpad ./.git/refs ./.git/refs
 COPY --chown=etherpad:etherpad ${SETTINGS} ./settings.json
 COPY --chown=etherpad:etherpad ./var ./var
-COPY --chown=etherpad:etherpad ./package.json ./package.json
+COPY --chown=etherpad:etherpad ./bin ./bin
-COPY --chown=etherpad:etherpad ./pnpm-workspace.yaml ./pnpm-workspace.yaml
+COPY --chown=etherpad:etherpad ./pnpm-workspace.yaml ./package.json ./
 
 FROM build as development
 
 COPY --chown=etherpad:etherpad ./src/package.json .npmrc ./src/pnpm-lock.yaml ./src/
 
-COPY --chown=etherpad:etherpad ./src/bin ./src/bin
+RUN bin/installDeps.sh && { [ -z "${ETHERPAD_PLUGINS}" ] || \
-
+      pnpm install --workspace-root ${ETHERPAD_PLUGINS}; }
-RUN { [ -z "${ETHERPAD_PLUGINS}" ] || \
+    
-      pnpm install --workspace-root ${ETHERPAD_PLUGINS}; } && \
-    src/bin/installDeps.sh
-
 FROM build as production
 
-# By default, Etherpad container is built and run in "production" mode. This is
-# leaner (development dependencies are not installed) and runs faster (among
-# other things, assets are minified & compressed).
 ENV NODE_ENV=production
 ENV ETHERPAD_PRODUCTION=true
 
 COPY --chown=etherpad:etherpad ./src ./src
 
-# Plugins must be installed before installing Etherpad's dependencies, otherwise
+RUN bin/installDeps.sh && { [ -z "${ETHERPAD_PLUGINS}" ] || \
-# npm will try to hoist common dependencies by removing them from
-# src/node_modules and installing them in the top-level node_modules. As of
-# v6.14.10, npm's hoist logic appears to be buggy, because it sometimes removes
-# dependencies from src/node_modules but fails to add them to the top-level
-# node_modules. Even if npm correctly hoists the dependencies, the hoisting
-# seems to confuse tools such as `npm outdated`, `npm update`, and some ESLint
-# rules.
-RUN { [ -z "${ETHERPAD_PLUGINS}" ] || \
       pnpm install --workspace-root ${ETHERPAD_PLUGINS}; } && \
-    src/bin/installDeps.sh && \
     rm -rf ~/.npm
 
 # Copy the configuration file.
@@ -137,4 +122,4 @@ HEALTHCHECK --interval=5s --timeout=3s \
   CMD curl --silent http://localhost:9001/health | grep -E "pass|ok|up" > /dev/null || exit 1
 
 EXPOSE 9001
-CMD ["npm", "run", "prod", "--prefix", "./src"]
+CMD ["pnpm", "run", "prod"]

bin

@@ -1 +0,0 @@
-src/bin

bin/buildDebian.sh

@@ -14,7 +14,7 @@ rm -rf ${DIST}
 mkdir -p ${DIST}/
 
 rm -rf ${SRC}
-rsync -a src/bin/deb-src/ ${SRC}/
+rsync -a bin/deb-src/ ${SRC}/
 mkdir -p ${SYSROOT}/opt/
 
 rsync --exclude '.git' -a . ${SYSROOT}/opt/etherpad/ --delete

bin/buildForWindows.sh

@@ -47,7 +47,7 @@ rm -rf node_modules || true
 rm -rf src/node_modules || true
 
 #log "do a normal unix install first..."
-#$(try cd src && ./bin/installDeps.sh)
+#$(try cd ./bin/installDeps.sh)
 
 log "copy the windows settings template..."
 try cp settings.json.template settings.json

bin/checkAllPads.js

@@ -7,12 +7,12 @@
 // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
 process.on('unhandledRejection', (err) => { throw err; });
 
-if (process.argv.length !== 2) throw new Error('Use: node src/bin/checkAllPads.js');
+if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js');
 
 (async () => {
-  const db = require('../node/db/DB');
+  const db = require('./src/node/db/DB');
   await db.init();
-  const padManager = require('../node/db/PadManager');
+  const padManager = require('./src/node/db/PadManager');
   await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId) => {
     const pad = await padManager.getPad(padId);
     try {

bin/checkPad.js

@@ -7,12 +7,12 @@
 // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
 process.on('unhandledRejection', (err) => { throw err; });
 
-if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPad.js $PADID');
+if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID');
 const padId = process.argv[2];
 (async () => {
-  const db = require('../node/db/DB');
+  const db = require('./src/node/db/DB');
   await db.init();
-  const padManager = require('../node/db/PadManager');
+  const padManager = require('./src/node/db/PadManager');
   if (!await padManager.doesPadExists(padId)) throw new Error('Pad does not exist');
   const pad = await padManager.getPad(padId);
   await pad.check();

bin/cleanRun.sh

@@ -2,10 +2,10 @@
 
 # Move to the Etherpad base directory.
 MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${MY_DIR}/../.." || exit 1
+cd "${MY_DIR}/.." || exit 1
 
 # Source constants and useful functions
-. src/bin/functions.sh
+. bin/functions.sh
 
 ignoreRoot=0
 for ARG in "$@"
@@ -31,7 +31,7 @@ fi
 rm -rf src/node_modules
 
 #Prepare the environment
-src/bin/installDeps.sh "$@" || exit 1
+bin/installDeps.sh "$@" || exit 1
 
 #Move to the node folder and start
 echo "Starting Etherpad..."

bin/createRelease.sh

@@ -134,7 +134,7 @@ function create_builds {
   git clone $ETHER_WEB_REPO
   echo "Creating windows build..."
   cd etherpad-lite
-  src/bin/buildForWindows.sh
+  bin/buildForWindows.sh
   [[ $? != 0 ]] && echo "Aborting: Error creating build for windows" && exit 1
   echo "Creating docs..."
   make docs

bin/createUserSession.js

@@ -12,13 +12,13 @@ process.on('unhandledRejection', (err) => { throw err; });
 const fs = require('fs');
 const path = require('path');
 const querystring = require('querystring');
-const settings = require('../node/utils/Settings');
+const settings = require('./src/node/utils/Settings');
 const supertest = require('supertest');
 
 (async () => {
   const api = supertest(`http://${settings.ip}:${settings.port}`);
 
-  const filePath = path.join(__dirname, '../../APIKEY.txt');
+  const filePath = path.join(__dirname, '../APIKEY.txt');
   const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
 
   let res;

bin/deb-src/sysroot/etc/init/etherpad.conf

@@ -15,7 +15,7 @@ pre-start script
     chown $EPUSER $EPLOGS                ||true
     chmod 0755 $EPLOGS                   ||true
     chown -R $EPUSER $EPHOME/var         ||true
-    $EPHOME/src/bin/installDeps.sh >> $EPLOGS/error.log || { stop; exit 1; }
+    $EPHOME/bin/installDeps.sh >> $EPLOGS/error.log || { stop; exit 1; }
 end script
 
 script

bin/debugRun.sh

@@ -2,13 +2,13 @@
 
 # Move to the Etherpad base directory.
 MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${MY_DIR}/../.." || exit 1
+cd "${MY_DIR}/.." || exit 1
 
 # Source constants and useful functions
-. src/bin/functions.sh
+. bin/functions.sh
 
 # Prepare the environment
-src/bin/installDeps.sh || exit 1
+bin/installDeps.sh || exit 1
 
 echo "If you are new to debugging Node.js with Chrome DevTools, take a look at this page:"
 echo "https://medium.com/@paul_irish/debugging-node-js-nightlies-with-chrome-devtools-7c4a1b95ae27"

bin/deleteAllGroupSessions.js

@@ -18,11 +18,11 @@ const supertest = require('supertest');
 let deleteCount = 0;
 
 // get the API Key
-const filePath = path.join(__dirname, '../../APIKEY.txt');
+const filePath = path.join(__dirname, '../APIKEY.txt');
 console.log('Deleting all group sessions, please be patient.');
 
 (async () => {
-  const settings = require('../tests/container/loadSettings').loadSettings();
+  const settings = require('./src/tests/container/loadSettings').loadSettings();
   const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
   const api = supertest(`http://${settings.ip}:${settings.port}`);
 

bin/deletePad.js

@@ -9,7 +9,7 @@
 // unhandled rejection into an uncaught exception, which does cause Node.js to exit.
 process.on('unhandledRejection', (err) => { throw err; });
 
-const settings = require('../tests/container/loadSettings').loadSettings();
+const settings = require('./src/tests/container/loadSettings').loadSettings();
 const path = require('path');
 const fs = require('fs');
 const supertest = require('supertest');
@@ -22,7 +22,7 @@ if (process.argv.length !== 3) throw new Error('Use: node deletePad.js $PADID');
 const padId = process.argv[2];
 
 // get the API Key
-const filePath = path.join(__dirname, '../../APIKEY.txt');
+const filePath = path.join(__dirname, '../APIKEY.txt');
 const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
 
 (async () => {

bin/extractPadData.js

@@ -19,13 +19,13 @@ const padId = process.argv[2];
 
 (async () => {
   // initialize database
-  require('../node/utils/Settings');
+  require('./src/node/utils/Settings');
-  const db = require('../node/db/DB');
+  const db = require('./src/node/db/DB');
   await db.init();
 
   // load extra modules
   const dirtyDB = require('dirty');
-  const padManager = require('../node/db/PadManager');
+  const padManager = require('./src/node/db/PadManager');
 
   // initialize output database
   const dirty = dirtyDB(`${padId}.db`);

bin/fastRun.sh

@@ -11,10 +11,10 @@ set -eu
 
 # Move to the Etherpad base directory.
 MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${MY_DIR}/../.." || exit 1
+cd "${MY_DIR}/.." || exit 1
 
 # Source constants and useful functions
-. src/bin/functions.sh
+. bin/functions.sh
 
 echo "Running directly, without checking/installing dependencies"
 

bin/importSqlFile.js

@@ -49,7 +49,7 @@ const unescape = (val) => {
   const fs = require('fs');
   const log4js = require('log4js');
   const readline = require('readline');
-  const settings = require('../node/utils/Settings');
+  const settings = require('./src/node/utils/Settings');
   const ueberDB = require('ueberdb2');
 
   const dbWrapperSettings = {

bin/installDeps.sh

@@ -3,10 +3,10 @@
 
 # Move to the Etherpad base directory.
 MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${MY_DIR}/../.." || exit 1
+cd "${MY_DIR}/.." || exit 1
 
 # Source constants and useful functions
-. src/bin/functions.sh
+. bin/functions.sh
 
 is_cmd pnpm || npm install pnpm -g
 
@@ -33,16 +33,13 @@ if [ ! -f "$settings" ]; then
   cp settings.json.template "$settings" || exit 1
 fi
 
-
 log "Installing dependencies..."
-cd src
-
 if [ -z "${ETHERPAD_PRODUCTION}" ]; then
-  log "Installing dev dependencies"
+  log "Installing dev dependencies with pnpm"
- pnpm i  || exit 1
+  pnpm --recursive i  || exit 1
 else
-  log "Installing production dependencies"
+  log "Installing production dependencies with pnpm"
-  pnpm i  --production || exit 1
+  pnpm --recursive i --production || exit 1
 fi
 
 # Remove all minified data to force node creating it new

bin/installOnWindows.bat

@@ -1,7 +1,7 @@
 @echo off
 
 :: Change directory to etherpad-lite root
-cd /D "%~dp0\..\.."
+cd /D "%~dp0\.."
 
 :: Is node installed?
 cmd /C node -e "" || ( echo "Please install node.js ( https://nodejs.org )" && exit /B 1 )
@@ -16,7 +16,7 @@ mklink /D "ep_etherpad-lite" "..\src"
 cd /D ..\src
 cmd /C pnpm i || exit /B 1
 
-cd /D "%~dp0\..\.."
+cd /D "%~dp0\.."
 
 echo _
 echo Clearing cache...

bin/migrateDirtyDBtoRealDB.js

@@ -14,7 +14,7 @@ process.on('unhandledRejection', (err) => { throw err; });
 
   const dirtyDb = require('dirty');
   const log4js = require('log4js');
-  const settings = require('../node/utils/Settings');
+  const settings = require('./src/node/utils/Settings');
   const ueberDB = require('ueberdb2');
   const util = require('util');
 
@@ -30,7 +30,7 @@ process.on('unhandledRejection', (err) => { throw err; });
   await db.init();
 
   console.log('Waiting for dirtyDB to parse its file.');
-  const dirty = dirtyDb(`${__dirname}/../../var/dirty.db`);
+  const dirty = dirtyDb(`${__dirname}/../var/dirty.db`);
   const length = await new Promise((resolve) => { dirty.once('load', resolve); });
 
   console.log(`Found ${length} records, processing now.`);

bin/plugins/checkPlugin.js

@@ -3,11 +3,11 @@
 /*
  * Usage -- see README.md
  *
- * Normal usage:                node src/bin/plugins/checkPlugin.js ep_whatever
+ * Normal usage:                node bin/plugins/checkPlugin.js ep_whatever
- * Auto fix the things it can:  node src/bin/plugins/checkPlugin.js ep_whatever autofix
+ * Auto fix the things it can:  node bin/plugins/checkPlugin.js ep_whatever autofix
- * Auto fix and commit:         node src/bin/plugins/checkPlugin.js ep_whatever autocommit
+ * Auto fix and commit:         node bin/plugins/checkPlugin.js ep_whatever autocommit
  * Auto fix, commit, push and publish to npm (highly dangerous):
- *                              node src/bin/plugins/checkPlugin.js ep_whatever autopush
+ *                              node bin/plugins/checkPlugin.js ep_whatever autopush
  */
 
 const process = require('process');
@@ -32,7 +32,7 @@ const logger = log4js.getLogger('checkPlugin');
   if (!pluginName) throw new Error('no plugin name specified');
   logger.info(`Checking the plugin: ${pluginName}`);
 
-  const epRootDir = await fsp.realpath(path.join(await fsp.realpath(__dirname), '../../..'));
+  const epRootDir = await fsp.realpath(path.join(await fsp.realpath(__dirname), '../..'));
   logger.info(`Etherpad root directory: ${epRootDir}`);
   process.chdir(epRootDir);
   const pluginPath = await fsp.realpath(`node_modules/${pluginName}`);
@@ -159,9 +159,9 @@ const logger = log4js.getLogger('checkPlugin');
 
   const workflows = ['backend-tests.yml', 'frontend-tests.yml', 'npmpublish.yml'];
   await Promise.all(workflows.map(async (fn) => {
-    await checkFile(`src/bin/plugins/lib/${fn}`, `.github/workflows/${fn}`);
+    await checkFile(`bin/plugins/lib/${fn}`, `.github/workflows/${fn}`);
   }));
-  await checkFile('src/bin/plugins/lib/dependabot.yml', '.github/dependabot.yml');
+  await checkFile('bin/plugins/lib/dependabot.yml', '.github/dependabot.yml');
 
   if (!files.includes('package.json')) {
     logger.warn('no package.json, please create');
@@ -214,7 +214,7 @@ const logger = log4js.getLogger('checkPlugin');
         logger.error(`both ${from} and ${to} exist; delete ${from}`);
       }
     } else {
-      checkFile('src/bin/plugins/lib/eslintrc.cjs', '.eslintrc.cjs', false);
+      checkFile('bin/plugins/lib/eslintrc.cjs', '.eslintrc.cjs', false);
     }
 
     if (checkEntries(parsedPackageJSON, {
@@ -251,7 +251,7 @@ const logger = log4js.getLogger('checkPlugin');
     if (autoFix) {
       logger.info('Autofixing missing README.md file');
       logger.info('please edit the README.md file further to include plugin specific details.');
-      await fillTemplate('src/bin/plugins/lib/README.md', `${pluginPath}/README.md`);
+      await fillTemplate('bin/plugins/lib/README.md', `${pluginPath}/README.md`);
     }
   }
 
@@ -260,7 +260,7 @@ const logger = log4js.getLogger('checkPlugin');
     if (autoFix) {
       logger.info('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md ' +
                   'file further to include plugin specific details.');
-      await fillTemplate('src/bin/plugins/lib/CONTRIBUTING.md', `${pluginPath}/CONTRIBUTING.md`);
+      await fillTemplate('bin/plugins/lib/CONTRIBUTING.md', `${pluginPath}/CONTRIBUTING.md`);
     }
   }
 
@@ -296,7 +296,7 @@ const logger = log4js.getLogger('checkPlugin');
     logger.warn('LICENSE file not found, please create');
     if (autoFix) {
       logger.info('Autofixing missing LICENSE file (Apache 2.0).');
-      await fsp.copyFile('src/bin/plugins/lib/LICENSE', `${pluginPath}/LICENSE`);
+      await fsp.copyFile('bin/plugins/lib/LICENSE', `${pluginPath}/LICENSE`);
     }
   }
 
@@ -306,7 +306,7 @@ const logger = log4js.getLogger('checkPlugin');
     if (autoFix) {
       logger.info('Autofixing missing .gitignore file');
       const gitignore =
-          await fsp.readFile('src/bin/plugins/lib/gitignore', {encoding: 'utf8', flag: 'r'});
+          await fsp.readFile('bin/plugins/lib/gitignore', {encoding: 'utf8', flag: 'r'});
       await fsp.writeFile(`${pluginPath}/.gitignore`, gitignore);
     }
   } else {

bin/plugins/getCorePlugins.sh

@@ -11,7 +11,7 @@ error () { log "ERROR: $@" >&2; }
 fatal () { error "$@"; exit 1; }
 
 mydir=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${mydir}/../../.."
+cd "${mydir}/../.."
 pdir=$(cd .. && pwd -P) || exit 1
 
 plugins=$("${mydir}/listOfficialPlugins") || exit 1

bin/plugins/lib/backend-tests.yml

@@ -49,7 +49,7 @@ jobs:
           cache: 'npm'
           cache-dependency-path: |
             src/package-lock.json
-            src/bin/doc/package-lock.json
+            bin/doc/package-lock.json
             node_modules/${{ steps.plugin_name.outputs.plugin_name }}/package-lock.json
       -
         name: Install plugin dependencies
@@ -69,7 +69,7 @@ jobs:
       # rules.
       -
         name: Install Etherpad core dependencies
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Run the backend tests
         run: cd src && pnpm test

bin/plugins/lib/frontend-tests.yml

@@ -38,7 +38,7 @@ jobs:
           cache: 'npm'
           cache-dependency-path: |
             src/package-lock.json
-            src/bin/doc/package-lock.json
+            bin/doc/package-lock.json
       -
         name: Check out the plugin
         uses: actions/checkout@v3
@@ -80,7 +80,7 @@ jobs:
       # rules.
       -
         name: Install Etherpad core dependencies
-        run: src/bin/installDeps.sh
+        run: bin/installDeps.sh
       -
         name: Create settings.json
         run: cp settings.json.template settings.json

bin/plugins/listOfficialPlugins

@@ -3,7 +3,7 @@ set -e
 newline='
 '
 mydir=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${mydir}/../../.."
+cd "${mydir}/../.."
 pdir=$(cd .. && pwd -P) || exit 1
 plugins=
 for p in "" "&page=2" "&page=3"; do

bin/plugins/reTestAllPlugins.sh

@@ -4,7 +4,7 @@ do
   echo $dir
   if [[ $dir == *"ep_"* ]]; then
     if [[ $dir != "ep_etherpad-lite" ]]; then
-      # node src/bin/plugins/checkPlugin.js $dir autopush
+      # node bin/plugins/checkPlugin.js $dir autopush
       cd node_modules/$dir
       git commit -m "Automatic update: bump update to re-run latest Etherpad tests" --allow-empty
       git push origin master

bin/plugins/updateAllPluginsScript.sh

@@ -10,7 +10,7 @@ do
   # echo $0
   if [[ $dir == *"ep_"* ]]; then
     if [[ $dir != "ep_etherpad-lite" ]]; then
-      node src/bin/plugins/checkPlugin.js $dir autopush
+      node bin/plugins/checkPlugin.js $dir autopush
     fi
   fi
   # echo $dir

bin/plugins/updateCorePlugins.sh

@@ -5,5 +5,5 @@ set -e
 for dir in node_modules/ep_*; do
   dir=${dir#node_modules/}
   [ "$dir" != ep_etherpad-lite ] || continue
-  node src/bin/plugins/checkPlugin.js "$dir" autopush
+  node bin/plugins/checkPlugin.js "$dir" autopush
 done

bin/push-after-release.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Specify the path to your package.json file
-PACKAGE_JSON_PATH="./src//package.json"
+PACKAGE_JSON_PATH="./src/package.json"
 
 # Check if the file exists
 if [ ! -f "$PACKAGE_JSON_PATH" ]; then

bin/rebuildPad.js

@@ -10,7 +10,7 @@
 process.on('unhandledRejection', (err) => { throw err; });
 
 if (process.argv.length !== 4 && process.argv.length !== 5) {
-  throw new Error('Use: node src/bin/repairPad.js $PADID $REV [$NEWPADID]');
+  throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]');
 }
 
 const padId = process.argv[2];
@@ -18,11 +18,11 @@ const newRevHead = process.argv[3];
 const newPadId = process.argv[4] || `${padId}-rebuilt`;
 
 (async () => {
-  const db = require('../node/db/DB');
+  const db = require('./src/node/db/DB');
   await db.init();
 
-  const PadManager = require('../node/db/PadManager');
+  const PadManager = require('./src/node/db/PadManager');
-  const Pad = require('../node/db/Pad').Pad;
+  const Pad = require('./src/node/db/Pad').Pad;
   // Validate the newPadId if specified and that a pad with that ID does
   // not already exist to avoid overwriting it.
   if (!PadManager.isValidPadId(newPadId)) {
@@ -43,8 +43,8 @@ const newPadId = process.argv[4] || `${padId}-rebuilt`;
   }));
 
   // Rebuild Pad from revisions up to and including the new revision head
-  const AuthorManager = require('../node/db/AuthorManager');
+  const AuthorManager = require('./src/node/db/AuthorManager');
-  const Changeset = require('../static/js/Changeset');
+  const Changeset = require('./src/static/js/Changeset');
   // Author attributes are derived from changesets, but there can also be
   // non-author attributes with specific mappings that changesets depend on
   // and, AFAICT, cannot be recreated any other way

bin/release.js

@@ -20,11 +20,11 @@ log4js.configure({appenders: {console: {type: 'console'}},
 
 Usage
 
-node src/bin/release.js patch
+node bin/release.js patch
 
 */
 const usage =
-    'node src/bin/release.js [patch/minor/major] -- example: "node src/bin/release.js patch"';
+    'node bin/release.js [patch/minor/major] -- example: "node bin/release.js patch"';
 
 const release = process.argv[2];
 
@@ -33,7 +33,7 @@ if (!release) {
   throw new Error('No release type included');
 }
 
-const cwd = path.join(fs.realpathSync(__dirname), '../../');
+const cwd = path.join(fs.realpathSync(__dirname), '../');
 process.chdir(cwd);
 
 // Run command capturing stdout. Trailing newlines are stripped (like the shell does).

bin/repairPad.js

@@ -10,7 +10,7 @@ process.on('unhandledRejection', (err) => { throw err; });
 
 console.warn('WARNING: This script must not be used while etherpad is running!');
 
-if (process.argv.length !== 3) throw new Error('Use: node src/bin/repairPad.js $PADID');
+if (process.argv.length !== 3) throw new Error('Use: node ./src/bin/repairPad.js $PADID');
 
 // get the padID
 const padId = process.argv[2];
@@ -19,12 +19,12 @@ let valueCount = 0;
 
 (async () => {
   // initialize database
-  require('../node/utils/Settings');
+  require('./src/node/utils/Settings');
-  const db = require('../node/db/DB');
+  const db = require('./src/node/db/DB');
   await db.init();
 
   // get the pad
-  const padManager = require('../node/db/PadManager');
+  const padManager = require('./src/node/db/PadManager');
   const pad = await padManager.getPad(padId);
 
   // accumulate the required keys

bin/run.sh

@@ -2,10 +2,10 @@
 
 # Move to the Etherpad base directory.
 MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${MY_DIR}/../.." || exit 1
+cd "${MY_DIR}/.." || exit 1
 
 # Source constants and useful functions
-. src/bin/functions.sh
+. bin/functions.sh
 
 ignoreRoot=0
 for ARG in "$@"; do
@@ -27,9 +27,10 @@ EOF
 fi
 
 # Prepare the environment
-src/bin/installDeps.sh "$@" || exit 1
+bin/installDeps.sh "$@" || exit 1
 
 # Move to the node folder and start
 log "Starting Etherpad..."
 
+# cd src
 exec pnpm run dev "$@"

bin/safeRun.sh

@@ -25,7 +25,7 @@ LAST_EMAIL_SEND=0
 
 # Move to the Etherpad base directory.
 MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${MY_DIR}/../.." || exit 1
+cd "${MY_DIR}/.." || exit 1
 
 # Check if a logfile parameter is set
 LOG="$1"
@@ -40,7 +40,7 @@ while true; do
   [ -w "${LOG}" ] || fatal "Logfile '${LOG}' is not writeable"
 
   # Start the application
-  src/bin/run.sh "$@" >>${LOG} 2>>${LOG}
+  bin/run.sh "$@" >>${LOG} 2>>${LOG}
 
   TIME_FMT=$(date +%Y-%m-%dT%H:%M:%S%z)
 

bin/updatePlugins.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 mydir=$(cd "${0%/*}" && pwd -P) || exit 1
-cd "${mydir}"/../..
+cd "${mydir}"/..
 OUTDATED=$(npm outdated --depth=0 | awk '{print $1}' | grep '^ep_') || {
   echo "All plugins are up-to-date"
   exit 0

doc/docker.adoc

@@ -30,7 +30,7 @@ docker compose up -d # will build and start the docker container on port 9001 wi
 Starting dev server:
 
 ```
-docker compose exec app bash -c "./src/bin/run.sh"
+docker compose exec app bash -c "./bin/run.sh"
 ```
 
 For production, please create your own docker compose file and change the `target` property in the build section to `production`. In addition, change the NODE_ENV in environment to production. For instance:

docker-compose.yml

@@ -15,10 +15,10 @@ services:
     tty: true
     stdin_open: true
     volumes:
+      # no volume mapping of node_modules as otherwise the build-time installed plugins will be overwritten with the mount
+      # the same applies to package.json and pnpm-lock.yaml in root dir as these would also get overwritten and build time installed plugins will be removed
       - ./src:/opt/etherpad-lite/src
-      - ./package.json:/opt/etherpad-lite/package.json
+      - ./bin:/opt/etherpad-lite/bin
-      - node_modules:/opt/etherpad-lite/src/node_modules
-      - pnpm-store:/home/etherpad/.local/share/pnpm/store/v3
     depends_on:
       - postgres
     environment:
@@ -60,6 +60,4 @@ services:
       - postgres_data:/var/lib/postgresql/data/pgdata
 
 volumes:
-  postgres_data:
+  postgres_data:
-  node_modules:
-  pnpm-store:

package.json

@@ -9,8 +9,8 @@
     "editor"
   ],
   "bin": {
-    "etherpad-healthcheck": "src/bin/etherpad-healthcheck",
+    "etherpad-healthcheck": "bin/etherpad-healthcheck",
-    "etherpad-lite": "src/bin/run"
+    "etherpad-lite": "bin/run"
   },
   "scripts": {
     "lint": "pnpm --filter ep_etherpad-lite run lint",

pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+packages:
+  - src

src/package.json

@@ -75,7 +75,7 @@
     "wtfnode": "^0.9.1"
   },
   "bin": {
-    "etherpad-healthcheck": "bin/etherpad-healthcheck",
+    "etherpad-healthcheck": "../bin/etherpad-healthcheck",
     "etherpad-lite": "node/server.ts"
   },
   "devDependencies": {

src/tests/frontend/travis/runner.sh

@@ -10,7 +10,7 @@ try() { "$@" || fatal "'$@' failed"; }
 MY_DIR=$(try cd "${0%/*}" && try pwd -P) || exit 1
 try cd "${MY_DIR}/../../../.."
 
-log "Assuming src/bin/installDeps.sh has already been run"
+log "Assuming bin/installDeps.sh has already been run"
 (cd src && npm run dev --experimental-worker "${@}" &
 ep_pid=$!)