Added new database rustydb (#6632)

Co-authored-by: SamTv12345 <samtv12345@samtv12345.com>

bin/migrateDB.ts

@@ -0,0 +1,83 @@
+// DB migration
+import {readFileSync} from 'node:fs'
+import {Database, DatabaseType} from "ueberdb2";
+import path from "node:path";
+const settings = require('ep_etherpad-lite/node/utils/Settings');
+
+
+// file1 = source, file2 = target
+// pnpm run migrateDB --file1 <db1.json> --file2 <db2.json>
+const arg = process.argv.slice(2);
+
+if (arg.length != 4) {
+  console.error('Wrong number of arguments!. Call with pnpm run migrateDB --file1 source.json target.json')
+  process.exit(1)
+}
+
+type SettingsConfig = {
+  dbType: string,
+  dbSettings: any
+}
+
+/*
+  {
+    "dbType": "<your-db-type>",
+    "dbSettings": {
+      <your-db-settings>
+     }
+  }
+ */
+
+let firstDBSettingsFile: string
+let secondDBSettingsFile: string
+
+
+if (arg[0] == "--file1") {
+    firstDBSettingsFile = arg[1]
+} else if (arg[0] === "--file2") {
+  secondDBSettingsFile = arg[1]
+}
+
+if (arg[2] == "--file1") {
+  firstDBSettingsFile = arg[3]
+} else if (arg[2] === "--file2") {
+  secondDBSettingsFile = arg[3]
+}
+
+
+
+const settingsfile = JSON.parse(readFileSync(path.join(settings.root,firstDBSettingsFile!)).toString()) as SettingsConfig
+const settingsfile2 = JSON.parse(readFileSync(path.join(settings.root,secondDBSettingsFile!)).toString()) as SettingsConfig
+
+console.log(settingsfile2)
+if ("filename" in settingsfile.dbSettings) {
+  settingsfile.dbSettings.filename = path.join(settings.root, settingsfile.dbSettings.filename)
+  console.log(settingsfile.dbType + " location is "+ settingsfile.dbSettings.filename)
+}
+
+if ("filename" in settingsfile2.dbSettings) {
+  settingsfile2.dbSettings.filename = path.join(settings.root, settingsfile2.dbSettings.filename)
+  console.log(settingsfile2.dbType + " location is "+ settingsfile2.dbSettings.filename)
+}
+
+const ueberdb1 = new Database(settingsfile.dbType as DatabaseType, settingsfile.dbSettings)
+const ueberdb2 = new Database(settingsfile2.dbType as DatabaseType, settingsfile2.dbSettings)
+
+const handleSync = async ()=>{
+  await ueberdb1.init()
+  await ueberdb2.init()
+
+  const allKeys = await ueberdb1.findKeys('*','')
+  for (const key of allKeys) {
+    const foundVal = await ueberdb1.get(key)!
+    await ueberdb2.set(key, foundVal)
+  }
+}
+
+handleSync().then(()=>{
+  console.log("Done syncing dbs")
+}).catch(e=>{
+  console.log(`Error syncing db ${e}`)
+})
+
+

bin/package.json

@@ -12,7 +12,7 @@
     "log4js": "^6.9.1",
     "semver": "^7.6.3",
     "tsx": "^4.19.0",
-    "ueberdb2": "^4.2.94"
+    "ueberdb2": "^4.2.100"
   },
   "devDependencies": {
     "@types/node": "^22.5.3",
@@ -34,7 +34,8 @@
     "stalePlugins": "node --import tsx ./plugins/stalePlugins.ts",
     "checkPlugin": "node --import tsx ./plugins/checkPlugin.ts",
     "plugins": "node --import tsx ./plugins.ts",
-    "generateChangelog": "node --import tsx generateReleaseNotes.ts"
+    "generateChangelog": "node --import tsx generateReleaseNotes.ts",
+    "migrateDB": "node --import tsx migrateDB.ts"
   },
   "author": "",
   "license": "ISC"

doc/cli.md

@@ -0,0 +1,29 @@
+# CLI
+
+You can find different tools for migrating things, checking your Etherpad health in the bin directory.
+One of these is the migrateDB command. It takes two settings.json files and copies data from one source to another one.
+In this example we migrate from the old dirty db to the new rustydb engine. So we copy these files to the root of the etherpad-directory.
+
+````json
+{
+  "dbType": "dirty",
+  "dbSettings": {
+    "filename": "./var/rusty.db"
+  }
+}
+````
+
+
+
+````json
+{
+  "dbType": "rustydb",
+  "dbSettings": {
+    "filename": "./var/rusty2.db"
+  }
+}
+````
+
+
+After that we need to move the data from dirty to rustydb.
+Therefore, we call `pnpm run migrateDB --file1 test1.json --file2 test2.json` with these two files in our root directories. After some time the data should be copied over to the new database.

pnpm-lock.yaml

@@ -119,8 +119,8 @@ importers:
         specifier: ^4.19.0
         version: 4.19.0
       ueberdb2:
-        specifier: ^4.2.94
+        specifier: ^4.2.100
-        version: 4.2.94
+        version: 4.2.101
     devDependencies:
       '@types/node':
         specifier: ^22.5.3
@@ -261,8 +261,8 @@ importers:
         specifier: 4.19.0
         version: 4.19.0
       ueberdb2:
-        specifier: ^4.2.94
+        specifier: ^4.2.100
-        version: 4.2.94
+        version: 4.2.101
       underscore:
         specifier: 1.13.7
         version: 1.13.7
@@ -348,6 +348,9 @@ importers:
       openapi-schema-validation:
         specifier: ^0.4.2
         version: 0.4.2
+      rusty-store-kv:
+        specifier: ^1.1.4
+        version: 1.1.4
       set-cookie-parser:
         specifier: ^2.7.0
         version: 2.7.0
@@ -3951,6 +3954,70 @@ packages:
   run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
 
+  rusty-store-kv-darwin-arm64@1.1.4:
+    resolution: {integrity: sha512-Pwedf7Fe0BBVmuM6Z/7JYGtwjVLI+rCEEUNoZ9jA31QtfWHHFYPk0XULk4mLnngP14eBy9opEpJO2wYAUra5bg==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [darwin]
+
+  rusty-store-kv-darwin-x64@1.1.4:
+    resolution: {integrity: sha512-R1vuflgIjP6JtFGnxRoDgZaw9kVF+m9FYEIZH0dG3v4AU55Zh53lQBflHLhqsC10OuCo3ZO0w++bSVmcGXmSJg==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [darwin]
+
+  rusty-store-kv-freebsd-x64@1.1.4:
+    resolution: {integrity: sha512-GbYN5+k44RJS2n6mqdDQz64ZLXl2mEZFIpdk5KblmUZaf7JPai6o6d/2kaldw+htTlVabdXVrmVRhytsHtmdcQ==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [freebsd]
+
+  rusty-store-kv-linux-arm-gnueabihf@1.1.4:
+    resolution: {integrity: sha512-hdPJTo/UR8kYSf6m6g530hzCRFCewyKhae/Dtol12fRqHlzr+RhygVkrxPblqwUFt8XJGG2A9xkc9CFJwbS7lg==}
+    engines: {node: '>= 10'}
+    cpu: [arm]
+    os: [linux]
+
+  rusty-store-kv-linux-arm64-gnu@1.1.4:
+    resolution: {integrity: sha512-KEsvvdbYHf6o01RY6DPGLt1x3BbDUnLvkJuMr6NLcX2mDgtvDtC2auNGuslb+mzVG3GqtCfiEesz4JRrJkfimA==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+
+  rusty-store-kv-linux-arm64-musl@1.1.4:
+    resolution: {integrity: sha512-o1PMDeMFc3nWHksO0pBHNdr5n2ZRFje8WmK6fwRMcicdsiK+fqv2zJyAjdDfVoP0RS4QXkM9DwCyqN+bziBz6A==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+
+  rusty-store-kv-linux-x64-gnu@1.1.4:
+    resolution: {integrity: sha512-EG7qO20CLY5F5kki0CFQY+p+CXyCVYtcah3vWI/Opbd6HcVlcoaZJ24KYWuoUCpRS4qIHAue/JtZ7kANlRUwCQ==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+
+  rusty-store-kv-linux-x64-musl@1.1.4:
+    resolution: {integrity: sha512-oJ5AJVjnTaWwvahGWceH73mVOjolE+BcBghJsxmlRgw0y4sYwk7qaF0r2heh1GDeMZsYMhZV0o8GAa3FH0W3lQ==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+
+  rusty-store-kv-win32-arm64-msvc@1.1.4:
+    resolution: {integrity: sha512-kqvaa4DqNNmOMQDBOoRBbiesTkE4E/nrghIk5eSalLaUHt1TNZpuOeWGS1+xRIYuRDBeHAqD72AkcqrP+9dzcQ==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [win32]
+
+  rusty-store-kv-win32-x64-msvc@1.1.4:
+    resolution: {integrity: sha512-SPI/Up8QF46MA2PkADeyIUd82PTa89i91TjkE81vVyKNIwcfXb9lFDOJThWds8gwMjiWcsMiMA1TclyCDZ0HoQ==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [win32]
+
+  rusty-store-kv@1.1.4:
+    resolution: {integrity: sha512-B8ibg3j9Wq2uY0FysjSpZ71GPj9x+oJUTvjzh0YUF4m+y7/4Lo+DwNLNU/4u5DTnGiXeAhWmXypjp+mWIN3GiQ==}
+    engines: {node: '>= 10'}
+
   safe-array-concat@1.1.2:
     resolution: {integrity: sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==}
     engines: {node: '>=0.4'}
@@ -4306,8 +4373,8 @@ packages:
     engines: {node: '>=14.17'}
     hasBin: true
 
-  ueberdb2@4.2.94:
+  ueberdb2@4.2.101:
-    resolution: {integrity: sha512-cfrMR3+wrUwGTFdujZb/BhqAC+pKHNJeqcCzF9ktn/y+8eG9I1xJXAzJvYWn6RxxjE/HosWXt36h0ZrxhbFOfg==}
+    resolution: {integrity: sha512-4WyfwvewluxElZ/FqoSupi/88FQANEmD0Cjy2cpSSExca3skZfe1thqw3kkpBvPOsyvdSekEkxDZbVe12wzn3A==}
     engines: {node: '>=16.20.1'}
 
   uid-safe@2.1.5:
@@ -8571,6 +8638,49 @@ snapshots:
     dependencies:
       queue-microtask: 1.2.3
 
+  rusty-store-kv-darwin-arm64@1.1.4:
+    optional: true
+
+  rusty-store-kv-darwin-x64@1.1.4:
+    optional: true
+
+  rusty-store-kv-freebsd-x64@1.1.4:
+    optional: true
+
+  rusty-store-kv-linux-arm-gnueabihf@1.1.4:
+    optional: true
+
+  rusty-store-kv-linux-arm64-gnu@1.1.4:
+    optional: true
+
+  rusty-store-kv-linux-arm64-musl@1.1.4:
+    optional: true
+
+  rusty-store-kv-linux-x64-gnu@1.1.4:
+    optional: true
+
+  rusty-store-kv-linux-x64-musl@1.1.4:
+    optional: true
+
+  rusty-store-kv-win32-arm64-msvc@1.1.4:
+    optional: true
+
+  rusty-store-kv-win32-x64-msvc@1.1.4:
+    optional: true
+
+  rusty-store-kv@1.1.4:
+    optionalDependencies:
+      rusty-store-kv-darwin-arm64: 1.1.4
+      rusty-store-kv-darwin-x64: 1.1.4
+      rusty-store-kv-freebsd-x64: 1.1.4
+      rusty-store-kv-linux-arm-gnueabihf: 1.1.4
+      rusty-store-kv-linux-arm64-gnu: 1.1.4
+      rusty-store-kv-linux-arm64-musl: 1.1.4
+      rusty-store-kv-linux-x64-gnu: 1.1.4
+      rusty-store-kv-linux-x64-musl: 1.1.4
+      rusty-store-kv-win32-arm64-msvc: 1.1.4
+      rusty-store-kv-win32-x64-msvc: 1.1.4
+
   safe-array-concat@1.1.2:
     dependencies:
       call-bind: 1.0.7
@@ -9009,7 +9119,7 @@ snapshots:
 
   typescript@5.5.4: {}
 
-  ueberdb2@4.2.94: {}
+  ueberdb2@4.2.101: {}
 
   uid-safe@2.1.5:
     dependencies:

src/node/utils/Settings.ts

@@ -169,11 +169,11 @@ exports.authenticationMethod = 'sso'
 /*
  * The Type of the database
  */
-exports.dbType = 'dirty';
+exports.dbType = 'rustydb';
 /**
  * This setting is passed with dbType to ueberDB to set up the database
  */
-exports.dbSettings = {filename: path.join(exports.root, 'var/dirty.db')};
+exports.dbSettings = {filename: path.join(exports.root, 'var/rusty.db')};
 
 /**
  * The default Text of a new pad
@@ -941,6 +941,11 @@ exports.reloadSettings = () => {
         logger.warn(`${dirtyWarning} File location: ${exports.dbSettings.filename}`);
     }
 
+    if (exports.dbType === 'rustydb') {
+      exports.dbSettings.filename = absolutePaths.makeAbsolute(exports.dbSettings.filename);
+      logger.warn(`File location: ${exports.dbSettings.filename}`);
+    }
+
     if (exports.ip === '') {
         // using Unix socket for connectivity
         logger.warn('The settings file contains an empty string ("") for the "ip" parameter. The ' +

src/package.json

@@ -70,7 +70,7 @@
     "superagent": "10.1.0",
     "tinycon": "0.6.8",
     "tsx": "4.19.0",
-    "ueberdb2": "^4.2.94",
+    "ueberdb2": "^4.2.100",
     "underscore": "1.13.7",
     "unorm": "1.6.0",
     "wtfnode": "^0.9.3"
@@ -110,7 +110,8 @@
     "split-grid": "^1.0.11",
     "supertest": "^7.0.0",
     "typescript": "^5.5.4",
-    "vitest": "^2.0.5"
+    "vitest": "^2.0.5",
+    "rusty-store-kv": "^1.1.4"
   },
   "engines": {
     "node": ">=18.18.2",

var/.gitignore

@@ -1,3 +1,5 @@
 sqlite.db
 minified*
 installed_plugins.json
+dirty.db
+rusty.db