From 28e04bdf71ae3e25d77f39900e2ff540c1d9c4d1 Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+SamTV12345@users.noreply.github.com> Date: Sun, 18 Aug 2024 12:14:24 +0200 Subject: [PATCH] Feat/changeset ts (#6594) * Migrated changeset * Added more tests. * Fixed test scopes --- src/node/db/API.ts | 12 +- src/node/db/Pad.ts | 33 +- src/node/handler/PadMessageHandler.ts | 49 +- src/node/types/PadType.ts | 5 +- src/node/utils/ExportHelper.ts | 10 +- src/node/utils/ExportHtml.ts | 15 +- src/node/utils/ExportTxt.ts | 13 +- src/node/utils/ImportHtml.ts | 7 +- src/node/utils/padDiff.ts | 61 +- src/static/js/AttributeManager.ts | 43 +- src/static/js/Builder.ts | 108 ++ src/static/js/Changeset.ts | 1399 ++++------------- src/static/js/ChangesetUtils.ts | 29 +- src/static/js/MergingOpAssembler.ts | 73 + src/static/js/Op.ts | 78 + src/static/js/OpAssembler.ts | 21 + src/static/js/OpIter.ts | 47 + src/static/js/SmartOpAssembler.ts | 115 ++ src/static/js/StringAssembler.ts | 18 + src/static/js/StringIterator.ts | 54 + src/static/js/TextLinesMutator.ts | 348 ++++ src/static/js/ace2_inner.ts | 83 +- src/static/js/broadcast.ts | 28 +- src/static/js/changesettracker.ts | 53 +- src/static/js/contentcollector.ts | 16 +- src/static/js/linestylefilter.ts | 9 +- src/static/js/types/ChangeSet.ts | 6 + src/static/js/types/ChangeSetBuilder.ts | 7 + src/static/js/undomodule.ts | 22 +- src/tests/backend-new/easysync-helper.ts | 220 +++ .../backend-new/specs/StringIteratorTest.ts | 47 + .../backend-new/specs/easysync-assembler.ts | 224 +++ .../backend-new/specs/easysync-compose.ts | 54 + .../backend-new/specs/easysync-mutations.ts | 320 ++++ .../backend-new/specs/easysync-other.test.ts | 166 ++ .../specs/easysync-subAttribution.ts | 55 + src/vitest.config.ts | 2 +- 37 files changed, 2540 insertions(+), 1310 deletions(-) create mode 100644 src/static/js/Builder.ts create mode 100644 src/static/js/MergingOpAssembler.ts create mode 100644 src/static/js/Op.ts create mode 100644 src/static/js/OpAssembler.ts create mode 100644 src/static/js/OpIter.ts create mode 100644 src/static/js/SmartOpAssembler.ts create mode 100644 src/static/js/StringAssembler.ts create mode 100644 src/static/js/StringIterator.ts create mode 100644 src/static/js/TextLinesMutator.ts create mode 100644 src/static/js/types/ChangeSet.ts create mode 100644 src/static/js/types/ChangeSetBuilder.ts create mode 100644 src/tests/backend-new/easysync-helper.ts create mode 100644 src/tests/backend-new/specs/StringIteratorTest.ts create mode 100644 src/tests/backend-new/specs/easysync-assembler.ts create mode 100644 src/tests/backend-new/specs/easysync-compose.ts create mode 100644 src/tests/backend-new/specs/easysync-mutations.ts create mode 100644 src/tests/backend-new/specs/easysync-other.test.ts create mode 100644 src/tests/backend-new/specs/easysync-subAttribution.ts diff --git a/src/node/db/API.ts b/src/node/db/API.ts index e9f445560..88910ca68 100644 --- a/src/node/db/API.ts +++ b/src/node/db/API.ts @@ -19,8 +19,10 @@ * limitations under the License. */ -const Changeset = require('../../static/js/Changeset'); +import {deserializeOps} from '../../static/js/Changeset'; import ChatMessage from '../../static/js/ChatMessage'; +import {Builder} from "../../static/js/Builder"; +import {Attribute} from "../../static/js/types/Attribute"; const CustomError = require('../utils/customError'); const padManager = require('./PadManager'); const padMessageHandler = require('../handler/PadMessageHandler'); @@ -563,11 +565,11 @@ exports.restoreRevision = async (padID: string, rev: number, authorId = '') => { const oldText = pad.text(); atext.text += '\n'; - const eachAttribRun = (attribs: string[], func:Function) => { + const eachAttribRun = (attribs: string, func:Function) => { let textIndex = 0; const newTextStart = 0; const newTextEnd = atext.text.length; - for (const op of Changeset.deserializeOps(attribs)) { + for (const op of deserializeOps(attribs)) { const nextIndex = textIndex + op.chars; if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) { func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs); @@ -577,10 +579,10 @@ exports.restoreRevision = async (padID: string, rev: number, authorId = '') => { }; // create a new changeset with a helper builder object - const builder = Changeset.builder(oldText.length); + const builder = new Builder(oldText.length); // assemble each line into the builder - eachAttribRun(atext.attribs, (start: number, end: number, attribs:string[]) => { + eachAttribRun(atext.attribs, (start: number, end: number, attribs:Attribute[]) => { builder.insert(atext.text.substring(start, end), attribs); }); diff --git a/src/node/db/Pad.ts b/src/node/db/Pad.ts index e117bb343..c3e11aa15 100644 --- a/src/node/db/Pad.ts +++ b/src/node/db/Pad.ts @@ -8,7 +8,7 @@ import {MapArrayType} from "../types/MapType"; */ import AttributeMap from '../../static/js/AttributeMap'; -const Changeset = require('../../static/js/Changeset'); +import {applyToAText, checkRep, copyAText, deserializeOps, makeAText, makeSplice, opsFromAText, pack, unpack} from '../../static/js/Changeset'; import ChatMessage from '../../static/js/ChatMessage'; import AttributePool from '../../static/js/AttributePool'; const Stream = require('../utils/Stream'); @@ -24,6 +24,7 @@ const readOnlyManager = require('./ReadOnlyManager'); const randomString = require('../utils/randomstring'); const hooks = require('../../static/js/pluginfw/hooks'); import pad_utils from "../../static/js/pad_utils"; +import {SmartOpAssembler} from "../../static/js/SmartOpAssembler"; const promises = require('../utils/promises'); /** @@ -56,7 +57,7 @@ class Pad { */ constructor(id:string, database = db) { this.db = database; - this.atext = Changeset.makeAText('\n'); + this.atext = makeAText('\n'); this.pool = new AttributePool(); this.head = -1; this.chatHead = -1; @@ -93,13 +94,13 @@ class Pad { * @param {String} authorId The id of the author * @return {Promise} */ - async appendRevision(aChangeset:AChangeSet, authorId = '') { - const newAText = Changeset.applyToAText(aChangeset, this.atext, this.pool); + async appendRevision(aChangeset:string, authorId = '') { + const newAText = applyToAText(aChangeset, this.atext, this.pool); if (newAText.text === this.atext.text && newAText.attribs === this.atext.attribs && this.head !== -1) { return this.head; } - Changeset.copyAText(newAText, this.atext); + copyAText(newAText, this.atext); const newRev = ++this.head; @@ -215,7 +216,7 @@ class Pad { ]); const apool = this.apool(); let atext = keyAText; - for (const cs of changesets) atext = Changeset.applyToAText(cs, atext, apool); + for (const cs of changesets) atext = applyToAText(cs, atext, apool); return atext; } @@ -293,7 +294,7 @@ class Pad { (!ins && start > 0 && orig[start - 1] === '\n'); if (!willEndWithNewline) ins += '\n'; if (ndel === 0 && ins.length === 0) return; - const changeset = Changeset.makeSplice(orig, start, ndel, ins); + const changeset = makeSplice(orig, start, ndel, ins); await this.appendRevision(changeset, authorId); } @@ -393,7 +394,7 @@ class Pad { if (context.type !== 'text') throw new Error(`unsupported content type: ${context.type}`); text = exports.cleanText(context.content); } - const firstChangeset = Changeset.makeSplice('\n', 0, 0, text); + const firstChangeset = makeSplice('\n', 0, 0, text); await this.appendRevision(firstChangeset, authorId); } await hooks.aCallAll('padLoad', {pad: this}); @@ -520,8 +521,8 @@ class Pad { const oldAText = this.atext; // based on Changeset.makeSplice - const assem = Changeset.smartOpAssembler(); - for (const op of Changeset.opsFromAText(oldAText)) assem.append(op); + const assem = new SmartOpAssembler(); + for (const op of opsFromAText(oldAText)) assem.append(op); assem.endDocument(); // although we have instantiated the dstPad with '\n', an additional '\n' is @@ -533,7 +534,7 @@ class Pad { // create a changeset that removes the previous text and add the newText with // all atributes present on the source pad - const changeset = Changeset.pack(oldLength, newLength, assem.toString(), newText); + const changeset = pack(oldLength, newLength, assem.toString(), newText); dstPad.appendRevision(changeset, authorId); await hooks.aCallAll('padCopy', { @@ -706,7 +707,7 @@ class Pad { } }) .batch(100).buffer(99); - let atext = Changeset.makeAText('\n'); + let atext = makeAText('\n'); for await (const [r, changeset, authorId, timestamp, isKeyRev, keyAText] of revs) { try { assert(authorId != null); @@ -717,10 +718,10 @@ class Pad { assert(timestamp > 0); assert(changeset != null); assert.equal(typeof changeset, 'string'); - Changeset.checkRep(changeset); - const unpacked = Changeset.unpack(changeset); + checkRep(changeset); + const unpacked = unpack(changeset); let text = atext.text; - for (const op of Changeset.deserializeOps(unpacked.ops)) { + for (const op of deserializeOps(unpacked.ops)) { if (['=', '-'].includes(op.opcode)) { assert(text.length >= op.chars); const consumed = text.slice(0, op.chars); @@ -731,7 +732,7 @@ class Pad { } assert.equal(op.attribs, AttributeMap.fromString(op.attribs, pool).toString()); } - atext = Changeset.applyToAText(changeset, atext, pool); + atext = applyToAText(changeset, atext, pool); if (isKeyRev) assert.deepEqual(keyAText, atext); } catch (err:any) { err.message = `(pad ${this.id} revision ${r}) ${err.message}`; diff --git a/src/node/handler/PadMessageHandler.ts b/src/node/handler/PadMessageHandler.ts index 2976381bb..6d2ee0d57 100644 --- a/src/node/handler/PadMessageHandler.ts +++ b/src/node/handler/PadMessageHandler.ts @@ -23,7 +23,7 @@ import {MapArrayType} from "../types/MapType"; import AttributeMap from '../../static/js/AttributeMap'; const padManager = require('../db/PadManager'); -const Changeset = require('../../static/js/Changeset'); +import {checkRep, cloneAText, compose, deserializeOps, follow, identity, inverse, makeAText, makeSplice, moveOpsToNewPool, mutateAttributionLines, mutateTextLines, oldLen, prepareForWire, splitAttributionLines, splitTextLines, unpack} from '../../static/js/Changeset'; import ChatMessage from '../../static/js/ChatMessage'; import AttributePool from '../../static/js/AttributePool'; const AttributeManager = require('../../static/js/AttributeManager'); @@ -44,6 +44,7 @@ import {ChangesetRequest, PadUserInfo, SocketClientRequest} from "../types/Socke import {APool, AText, PadAuthor, PadType} from "../types/PadType"; import {ChangeSet} from "../types/ChangeSet"; import {ChatMessageMessage, ClientReadyMessage, ClientSaveRevisionMessage, ClientSuggestUserName, ClientUserChangesMessage, ClientVarMessage, CustomMessage, UserNewInfoMessage} from "../../static/js/types/SocketIOMessage"; +import {Builder} from "../../static/js/Builder"; const webaccess = require('../hooks/express/webaccess'); const { checkValidRev } = require('../utils/checkValidRev'); @@ -594,10 +595,10 @@ const handleUserChanges = async (socket:any, message: { const pad = await padManager.getPad(thisSession.padId, null, thisSession.author); // Verify that the changeset has valid syntax and is in canonical form - Changeset.checkRep(changeset); + checkRep(changeset); // Validate all added 'author' attribs to be the same value as the current user - for (const op of Changeset.deserializeOps(Changeset.unpack(changeset).ops)) { + for (const op of deserializeOps(unpack(changeset).ops)) { // + can add text with attribs // = can change or add attribs // - can have attribs, but they are discarded and don't show up in the attribs - @@ -616,7 +617,7 @@ const handleUserChanges = async (socket:any, message: { // ex. adoptChangesetAttribs // Afaik, it copies the new attributes from the changeset, to the global Attribute Pool - let rebasedChangeset = Changeset.moveOpsToNewPool(changeset, wireApool, pad.pool); + let rebasedChangeset = moveOpsToNewPool(changeset, wireApool, pad.pool); // ex. applyUserChanges let r = baseRev; @@ -629,21 +630,21 @@ const handleUserChanges = async (socket:any, message: { const {changeset: c, meta: {author: authorId}} = await pad.getRevision(r); if (changeset === c && thisSession.author === authorId) { // Assume this is a retransmission of an already applied changeset. - rebasedChangeset = Changeset.identity(Changeset.unpack(changeset).oldLen); + rebasedChangeset = identity(unpack(changeset).oldLen); } // At this point, both "c" (from the pad) and "changeset" (from the // client) are relative to revision r - 1. The follow function // rebases "changeset" so that it is relative to revision r // and can be applied after "c". - rebasedChangeset = Changeset.follow(c, rebasedChangeset, false, pad.pool); + rebasedChangeset = follow(c, rebasedChangeset, false, pad.pool); } const prevText = pad.text(); - if (Changeset.oldLen(rebasedChangeset) !== prevText.length) { + if (oldLen(rebasedChangeset) !== prevText.length) { throw new Error( `Can't apply changeset ${rebasedChangeset} with oldLen ` + - `${Changeset.oldLen(rebasedChangeset)} to document of length ${prevText.length}`); + `${oldLen(rebasedChangeset)} to document of length ${prevText.length}`); } const newRev = await pad.appendRevision(rebasedChangeset, thisSession.author); @@ -658,7 +659,7 @@ const handleUserChanges = async (socket:any, message: { // Make sure the pad always ends with an empty line. if (pad.text().lastIndexOf('\n') !== pad.text().length - 1) { - const nlChangeset = Changeset.makeSplice(pad.text(), pad.text().length - 1, 0, '\n'); + const nlChangeset = makeSplice(pad.text(), pad.text().length - 1, 0, '\n'); await pad.appendRevision(nlChangeset, thisSession.author); } @@ -713,7 +714,7 @@ exports.updatePadClients = async (pad: PadType) => { const revChangeset = revision.changeset; const currentTime = revision.meta.timestamp; - const forWire = Changeset.prepareForWire(revChangeset, pad.pool); + const forWire = prepareForWire(revChangeset, pad.pool); const msg = { type: 'COLLABROOM', data: { @@ -748,7 +749,7 @@ const _correctMarkersInPad = (atext: AText, apool: AttributePool) => { // that aren't at the start of a line const badMarkers = []; let offset = 0; - for (const op of Changeset.deserializeOps(atext.attribs)) { + for (const op of deserializeOps(atext.attribs)) { const attribs = AttributeMap.fromString(op.attribs, apool); const hasMarker = AttributeManager.lineAttributes.some((a: string) => attribs.has(a)); if (hasMarker) { @@ -770,7 +771,7 @@ const _correctMarkersInPad = (atext: AText, apool: AttributePool) => { // create changeset that removes these bad markers offset = 0; - const builder = Changeset.builder(text.length); + const builder = new Builder(text.length); badMarkers.forEach((pos) => { builder.keepText(text.substring(offset, pos)); @@ -905,7 +906,7 @@ const handleClientReady = async (socket:any, message: ClientReadyMessage) => { // return pending changesets for (const r of revisionsNeeded) { - const forWire = Changeset.prepareForWire(changesets[r].changeset, pad.pool); + const forWire = prepareForWire(changesets[r].changeset, pad.pool); const wireMsg = {type: 'COLLABROOM', data: {type: 'CLIENT_RECONNECT', headRev: pad.getHeadRevisionNumber(), @@ -930,8 +931,8 @@ const handleClientReady = async (socket:any, message: ClientReadyMessage) => { let apool; // prepare all values for the wire, there's a chance that this throws, if the pad is corrupted try { - atext = Changeset.cloneAText(pad.atext); - const attribsForWire = Changeset.prepareForWire(atext.attribs, pad.pool); + atext = cloneAText(pad.atext); + const attribsForWire = prepareForWire(atext.attribs, pad.pool); apool = attribsForWire.pool.toJsonable(); atext.attribs = attribsForWire.translated; } catch (e:any) { @@ -1167,13 +1168,13 @@ const getChangesetInfo = async (pad: PadType, startNum: number, endNum:number, g if (compositeEnd > endNum || compositeEnd > headRevision + 1) break; const forwards = composedChangesets[`${compositeStart}/${compositeEnd}`]; - const backwards = Changeset.inverse(forwards, lines.textlines, lines.alines, pad.apool()); + const backwards = inverse(forwards, lines.textlines, lines.alines, pad.apool()); - Changeset.mutateAttributionLines(forwards, lines.alines, pad.apool()); - Changeset.mutateTextLines(forwards, lines.textlines); + mutateAttributionLines(forwards, lines.alines, pad.apool()); + mutateTextLines(forwards, lines.textlines); - const forwards2 = Changeset.moveOpsToNewPool(forwards, pad.apool(), apool); - const backwards2 = Changeset.moveOpsToNewPool(backwards, pad.apool(), apool); + const forwards2 = moveOpsToNewPool(forwards, pad.apool(), apool); + const backwards2 = moveOpsToNewPool(backwards, pad.apool(), apool); const t1 = (compositeStart === 0) ? revisionDate[0] : revisionDate[compositeStart - 1]; const t2 = revisionDate[compositeEnd - 1]; @@ -1199,12 +1200,12 @@ const getPadLines = async (pad: PadType, revNum: number) => { if (revNum >= 0) { atext = await pad.getInternalRevisionAText(revNum); } else { - atext = Changeset.makeAText('\n'); + atext = makeAText('\n'); } return { - textlines: Changeset.splitTextLines(atext.text), - alines: Changeset.splitAttributionLines(atext.attribs, atext.text), + textlines: splitTextLines(atext.text), + alines: splitAttributionLines(atext.attribs, atext.text), }; }; @@ -1239,7 +1240,7 @@ const composePadChangesets = async (pad: PadType, startNum: number, endNum: numb for (r = startNum + 1; r < endNum; r++) { const cs = changesets[r]; - changeset = Changeset.compose(changeset, cs, pool); + changeset = compose(changeset as string, cs as string, pool); } return changeset; } catch (e) { diff --git a/src/node/types/PadType.ts b/src/node/types/PadType.ts index 5b96ac5f6..f35b51361 100644 --- a/src/node/types/PadType.ts +++ b/src/node/types/PadType.ts @@ -1,10 +1,11 @@ import {MapArrayType} from "./MapType"; +import AttributePool from "../../static/js/AttributePool"; export type PadType = { id: string, - apool: ()=>APool, + apool: ()=>AttributePool, atext: AText, - pool: APool, + pool: AttributePool, getInternalRevisionAText: (text:number|string)=>Promise, getValidRevisionRange: (fromRev: string, toRev: string)=>PadRange, getRevisionAuthor: (rev: number)=>Promise, diff --git a/src/node/utils/ExportHelper.ts b/src/node/utils/ExportHelper.ts index e12332b06..4c29534f4 100644 --- a/src/node/utils/ExportHelper.ts +++ b/src/node/utils/ExportHelper.ts @@ -21,7 +21,7 @@ import AttributeMap from '../../static/js/AttributeMap'; import AttributePool from "../../static/js/AttributePool"; -const Changeset = require('../../static/js/Changeset'); +import {deserializeOps, splitAttributionLines, subattribution} from '../../static/js/Changeset'; const { checkValidRev } = require('./checkValidRev'); /* @@ -31,7 +31,7 @@ exports.getPadPlainText = (pad: { getInternalRevisionAText: (arg0: any) => any; const _analyzeLine = exports._analyzeLine; const atext = ((revNum !== undefined) ? pad.getInternalRevisionAText(checkValidRev(revNum)) : pad.atext); const textLines = atext.text.slice(0, -1).split('\n'); - const attribLines = Changeset.splitAttributionLines(atext.attribs, atext.text); + const attribLines = splitAttributionLines(atext.attribs, atext.text); const apool = pad.pool; const pieces = []; @@ -52,14 +52,14 @@ type LineModel = { [id:string]:string|number|LineModel } -exports._analyzeLine = (text:string, aline: LineModel, apool: AttributePool) => { +exports._analyzeLine = (text:string, aline: string, apool: AttributePool) => { const line: LineModel = {}; // identify list let lineMarker = 0; line.listLevel = 0; if (aline) { - const [op] = Changeset.deserializeOps(aline); + const [op] = deserializeOps(aline); if (op != null) { const attribs = AttributeMap.fromString(op.attribs, apool); let listType = attribs.get('list'); @@ -79,7 +79,7 @@ exports._analyzeLine = (text:string, aline: LineModel, apool: AttributePool) => } if (lineMarker) { line.text = text.substring(1); - line.aline = Changeset.subattribution(aline, 1); + line.aline = subattribution(aline, 1); } else { line.text = text; line.aline = aline; diff --git a/src/node/utils/ExportHtml.ts b/src/node/utils/ExportHtml.ts index 138f084ba..29b171c8f 100644 --- a/src/node/utils/ExportHtml.ts +++ b/src/node/utils/ExportHtml.ts @@ -18,7 +18,7 @@ import {MapArrayType} from "../types/MapType"; * limitations under the License. */ -const Changeset = require('../../static/js/Changeset'); +import {deserializeOps, splitAttributionLines, subattribution} from '../../static/js/Changeset'; const attributes = require('../../static/js/attributes'); const padManager = require('../db/PadManager'); const _ = require('underscore'); @@ -28,6 +28,8 @@ const eejs = require('../eejs'); const _analyzeLine = require('./ExportHelper')._analyzeLine; const _encodeWhitespace = require('./ExportHelper')._encodeWhitespace; import padutils from "../../static/js/pad_utils"; +import {StringIterator} from "../../static/js/StringIterator"; +import {StringAssembler} from "../../static/js/StringAssembler"; const getPadHTML = async (pad: PadType, revNum: string) => { let atext = pad.atext; @@ -44,7 +46,7 @@ const getPadHTML = async (pad: PadType, revNum: string) => { const getHTMLFromAtext = async (pad:PadType, atext: AText, authorColors?: string[]) => { const apool = pad.apool(); const textLines = atext.text.slice(0, -1).split('\n'); - const attribLines = Changeset.splitAttributionLines(atext.attribs, atext.text); + const attribLines = splitAttributionLines(atext.attribs, atext.text); const tags = ['h1', 'h2', 'strong', 'em', 'u', 's']; const props = ['heading1', 'heading2', 'bold', 'italic', 'underline', 'strikethrough']; @@ -80,6 +82,7 @@ const getHTMLFromAtext = async (pad:PadType, atext: AText, authorColors?: string css += '