feat(core): added MarkupManager to manage markdown, integrated with ExtensionsManager, parser and serializer

src/core/ExtensionsManager.ts

@@ -6,6 +6,7 @@ import {ExtensionBuilder} from './ExtensionBuilder';
 import {ParserTokensRegistry} from './ParserTokensRegistry';
 import {SchemaSpecRegistry} from './SchemaSpecRegistry';
 import {SerializerTokensRegistry} from './SerializerTokensRegistry';
+import {MarkupManager} from './markdown/MarkupManager';
 import {TransformFn} from './markdown/ProseMirrorTransformer';
 import type {ActionSpec} from './types/actions';
 import type {
@@ -28,6 +29,10 @@ type ExtensionsManagerOptions = {
     pmTransformers?: TransformFn[];
 };
 
+// TODO: move to props
+const RAW_MARKUP_TRACKED_TOKENS = ['yfm_table_open'];
+const TRACKED_NODES_TYPES = ['yfm_table'];
+
 export class ExtensionsManager {
     static process(extensions: Extension, options: ExtensionsManagerOptions) {
         return new this({extensions, options}).build();
@@ -53,6 +58,7 @@ export class ExtensionsManager {
     #actions: Record<string, ActionSpec> = {};
     #nodeViews: Record<string, NodeViewConstructor> = {};
     #markViews: Record<string, MarkViewConstructor> = {};
+    #markupManager: MarkupManager = new MarkupManager();
 
     constructor({extensions, options = {}}: ExtensionsManagerParams) {
         this.#extensions = extensions;
@@ -72,6 +78,9 @@ export class ExtensionsManager {
 
         // TODO: add prefilled context
         this.#builder = new ExtensionBuilder();
+
+        this.#markupManager.setTrackedTokensTypes(RAW_MARKUP_TRACKED_TOKENS);
+        this.#markupManager.setTrackedNodesTypes(TRACKED_NODES_TYPES);
     }
 
     build() {
@@ -105,6 +114,13 @@ export class ExtensionsManager {
 
     private processNode = (name: string, {spec, fromMd, toMd: toMd, view}: ExtensionNodeSpec) => {
         this.#schemaRegistry.addNode(name, spec);
+
+        // Inject nodeId attr for tracked nodes types
+        if (this.#markupManager.isTrackedNodeType(name)) {
+            spec.attrs = spec.attrs || {};
+            spec.attrs.nodeId = {default: null};
+        }
+
         this.#parserRegistry.addToken(fromMd.tokenName || name, fromMd.tokenSpec);
         this.#serializerRegistry.addNode(name, toMd);
         if (view) {
@@ -122,24 +138,31 @@ export class ExtensionsManager {
     };
 
     private createDeps() {
+        const actions = new ActionsManager();
+
         const schema = this.#schemaRegistry.createSchema();
+        const markupParser = this.#parserRegistry.createParser(
+            schema,
+            this.#mdForMarkup,
+            this.#pmTransformers,
+            this.#markupManager,
+        );
+        const textParser = this.#parserRegistry.createParser(
+            schema,
+            this.#mdForText,
+            this.#pmTransformers,
+            this.#markupManager,
+        );
+        const serializer = this.#serializerRegistry.createSerializer(this.#markupManager);
+
         this.#deps = {
             schema,
-            actions: new ActionsManager(),
-            markupParser: this.#parserRegistry.createParser(
-                schema,
-                this.#mdForMarkup,
-                this.#pmTransformers,
-            ),
-            textParser: this.#parserRegistry.createParser(
-                schema,
-                this.#mdForText,
-                this.#pmTransformers,
-            ),
-            serializer: this.#serializerRegistry.createSerializer(),
+            actions,
+            markupParser,
+            textParser,
+            serializer,
         };
     }
-
     private createDerived() {
         this.#plugins = this.#spec.plugins(this.#deps);
         Object.assign(this.#actions, this.#spec.actions(this.#deps));

src/core/ParserTokensRegistry.ts

@@ -2,6 +2,7 @@ import type MarkdownIt from 'markdown-it';
 import type {Schema} from 'prosemirror-model';
 
 import {MarkdownParser} from './markdown/MarkdownParser';
+import {MarkupManager} from './markdown/MarkupManager';
 import {TransformFn} from './markdown/ProseMirrorTransformer';
 import type {Parser, ParserToken} from './types/parser';
 
@@ -13,7 +14,12 @@ export class ParserTokensRegistry {
         return this;
     }
 
-    createParser(schema: Schema, tokenizer: MarkdownIt, pmTransformers: TransformFn[]): Parser {
-        return new MarkdownParser(schema, tokenizer, this.#tokens, pmTransformers);
+    createParser(
+        schema: Schema,
+        tokenizer: MarkdownIt,
+        pmTransformers: TransformFn[],
+        markupManager: MarkupManager,
+    ): Parser {
+        return new MarkdownParser(schema, tokenizer, this.#tokens, pmTransformers, markupManager);
     }
 }

src/core/SerializerTokensRegistry.test.ts

@@ -1,28 +1,32 @@
 import {SerializerTokensRegistry} from './SerializerTokensRegistry';
+import {MarkupManager} from './markdown/MarkupManager';
 import type {SerializerTests} from './types/serializer';
 
 describe('SerializerTokensRegistry', () => {
     it('should create empty serializer', () => {
-        const serializer = new SerializerTokensRegistry().createSerializer();
+        const markupManager = new MarkupManager();
+        const serializer = new SerializerTokensRegistry().createSerializer(markupManager);
         expect(typeof serializer.serialize).toBe('function');
     });
 
     it('should add nodes', () => {
         const nodeName = 'example_node';
+        const markupManager = new MarkupManager();
 
         const serializer = new SerializerTokensRegistry()
             .addNode(nodeName, () => {})
-            .createSerializer() as SerializerTests;
+            .createSerializer(markupManager) as SerializerTests;
 
         expect(serializer.containsNode(nodeName)).toBe(true);
     });
 
     it('should add marks', () => {
         const markName = 'example_mark';
+        const markupManager = new MarkupManager();
 
         const serializer = new SerializerTokensRegistry()
             .addMark(markName, {open: '1', close: '2'})
-            .createSerializer() as SerializerTests;
+            .createSerializer(markupManager) as SerializerTests;
 
         expect(serializer.containsMark(markName)).toBe(true);
     });

src/core/SerializerTokensRegistry.ts

@@ -1,4 +1,5 @@
 import {MarkdownSerializer} from './markdown/MarkdownSerializer';
+import {MarkupManager} from './markdown/MarkupManager';
 import type {Serializer, SerializerMarkToken, SerializerNodeToken} from './types/serializer';
 
 export class SerializerTokensRegistry {
@@ -15,7 +16,7 @@ export class SerializerTokensRegistry {
         return this;
     }
 
-    createSerializer(): Serializer {
-        return new MarkdownSerializer(this.#nodes, this.#marks);
+    createSerializer(markupManager: MarkupManager): Serializer {
+        return new MarkdownSerializer(this.#nodes, this.#marks, markupManager);
     }
 }

src/core/markdown/Markdown.test.ts

@@ -11,9 +11,12 @@ import type {SerializerNodeToken} from '../types/serializer';
 
 import {MarkdownParser} from './MarkdownParser';
 import {MarkdownSerializer} from './MarkdownSerializer';
+import {MarkupManager} from './MarkupManager';
 
 const {schema} = builder;
 schema.nodes['hard_break'].spec.isBreak = true;
+
+const markupManager = new MarkupManager();
 const parser: Parser = new MarkdownParser(
     schema,
     new MarkdownIt('commonmark'),
@@ -35,6 +38,7 @@ const parser: Parser = new MarkdownParser(
         code_inline: {type: 'mark', name: 'code', noCloseToken: true},
     },
     [],
+    markupManager,
 );
 const serializer = new MarkdownSerializer(
     {
@@ -85,6 +89,7 @@ const serializer = new MarkdownSerializer(
         strong: {open: '**', close: '**', mixable: true, expelEnclosingWhitespace: true},
         code: {open: '`', close: '`', escape: false, expelEnclosingWhitespace: true},
     },
+    markupManager,
 );
 
 const {doc, p, h1, h2, li, ul, ol, br, pre} = builder;

src/core/markdown/MarkdownParser.test.ts

@@ -5,8 +5,11 @@ import {doc, br as hardBreak, p, schema} from 'prosemirror-test-builder';
 import type {Parser} from '../types/parser';
 
 import {MarkdownParser} from './MarkdownParser';
+import {MarkupManager} from './MarkupManager';
 
 const md = MarkdownIt('commonmark', {html: false, breaks: true});
+const markupManager = new MarkupManager();
+
 const testParser: Parser = new MarkdownParser(
     schema,
     md,
@@ -16,6 +19,7 @@ const testParser: Parser = new MarkdownParser(
         softbreak: {type: 'node', name: 'hard_break'},
     },
     [],
+    markupManager,
 );
 
 function parseWith(parser: Parser) {

src/core/markdown/MarkdownParser.ts

@@ -6,6 +6,7 @@ import {Mark, MarkType, Node, NodeType, Schema} from 'prosemirror-model';
 import {logger} from '../../logger';
 import type {Parser, ParserToken} from '../types/parser';
 
+import {MarkupManager} from './MarkupManager';
 import {ProseMirrorTransformer, TransformFn} from './ProseMirrorTransformer';
 
 type TokenAttrs = {[name: string]: unknown};
@@ -18,25 +19,40 @@ enum TokenType {
     default = 'default',
 }
 
+/**
+ * Generate a unique token ID
+ */
+export function createUniqueId(prefix: string): string {
+    const randomLetters = Array.from(
+        {length: 5},
+        () => String.fromCharCode(97 + Math.floor(Math.random() * 26)), // a-z
+    ).join('');
+
+    return `${prefix}-${randomLetters}${Date.now()}`;
+}
+
 export class MarkdownParser implements Parser {
     schema: Schema;
     stack: Array<{type: NodeType; attrs?: TokenAttrs; content: Array<Node>}> = [];
     marks: readonly Mark[];
     tokens: Record<string, ParserToken>;
     tokenizer: MarkdownIt;
     pmTransformers: TransformFn[];
+    markupManager: MarkupManager;
 
     constructor(
         schema: Schema,
         tokenizer: MarkdownIt,
         tokens: Record<string, ParserToken>,
         pmTransformers: TransformFn[],
+        markupManager: MarkupManager,
     ) {
         this.schema = schema;
         this.marks = Mark.none;
         this.tokens = tokens;
         this.tokenizer = tokenizer;
         this.pmTransformers = pmTransformers;
+        this.markupManager = markupManager;
     }
 
     validateLink(url: string): boolean {
@@ -57,12 +73,21 @@ export class MarkdownParser implements Parser {
 
     parse(text: string) {
         const time = Date.now();
+        this.markupManager.setMarkup(text);
+
         try {
             this.stack = [{type: this.schema.topNodeType, content: []}];
 
             let mdItTokens;
             try {
                 mdItTokens = this.tokenizer.parse(text, {});
+                mdItTokens.forEach((token) => {
+                    if (this.markupManager.isTrackedTokenType(token.type) && token.map) {
+                        const tokenId = createUniqueId(token.type);
+                        token.attrSet('tokenId', tokenId);
+                        this.markupManager.setPos(tokenId, token.map);
+                    }
+                });
             } catch (err) {
                 const e = err as Error;
                 e.message = 'Unable to parse your markup. Please check for errors. ' + e.message;
@@ -102,10 +127,25 @@ export class MarkdownParser implements Parser {
         tokenStream: Token[],
         i: number,
     ): TokenAttrs | undefined {
-        if (tokenSpec.getAttrs) return tokenSpec.getAttrs(token, tokenStream, i);
-        else if (tokenSpec.attrs instanceof Function) return tokenSpec.attrs(token);
+        let attrs: TokenAttrs | undefined = {};
+
+        if (tokenSpec.getAttrs) {
+            attrs = tokenSpec.getAttrs(token, tokenStream, i);
+        } else if (tokenSpec.attrs instanceof Function) {
+            attrs = tokenSpec.attrs(token);
+        } else {
+            attrs = tokenSpec.attrs;
+        }
+
+        // Inject nodeId attr if the markdown token has tokenId
+        const tokdenId = token.attrGet('tokenId');
+        if (tokdenId) {
+            attrs = attrs ?? {};
+            attrs.nodeId = tokdenId;
+        }
 
-        return tokenSpec.attrs;
+        // TODO: @makhnatkin add cache
+        return attrs;
     }
 
     private getTokenSpec(token: Token) {

src/core/markdown/MarkdownSerializer.js

@@ -3,6 +3,8 @@
 // ::- A specification for serializing a ProseMirror document as
 // Markdown/CommonMark text.
 // prettier-ignore
+import {MarkupManager} from "./MarkupManager";
+
 export class MarkdownSerializer {
     // :: (Object<(state: MarkdownSerializerState, node: Node, parent: Node, index: number)>, Object)
     // Construct a serializer with the given configuration. The `nodes`
@@ -37,18 +39,23 @@ export class MarkdownSerializer {
     // outside the marks. This is necessary for emphasis marks as
     // CommonMark does not permit enclosing whitespace inside emphasis
     // marks, see: http://spec.commonmark.org/0.26/#example-330
-    constructor(nodes, marks) {
+    constructor(nodes, marks, markupManager) {
         // :: Object<(MarkdownSerializerState, Node)> The node serializer
         // functions for this serializer.
         this.nodes = nodes;
         // :: Object The mark serializer info.
         this.marks = marks;
+
+        this.markupManager = markupManager;
     }
 
     // :: (Node, ?Object) → string
     // Serialize the content of the given node to
     // [CommonMark](http://commonmark.org/).
     serialize(content, options) {
+        if (!this.markupManager.rawMarkdown) {
+            console.warn('[MarkdownSerializer] No raw markdown is set');
+        }
         const state = new MarkdownSerializerState(this.nodes, this.marks, options);
         state.renderContent(content);
         return state.out;
@@ -227,8 +234,8 @@ export class MarkdownSerializerState {
                 }
             }
 
-            const inner = marks.length && marks[marks.length - 1]; const
-                noEsc = inner && this.marks[inner.type.name].escape === false;
+            const inner = marks.length && marks[marks.length - 1];
+            const noEsc = inner && this.marks[inner.type.name].escape === false;
             const len = marks.length - (noEsc ? 1 : 0);
 
             // Try to reorder 'mixable' marks, such as em and strong, which