Refactor MFM

Co-authored-by: syuilo syuilotan@yahoo.co.jp

src/client/app/common/scripts/note-mixin.ts

@@ -80,8 +80,8 @@ export default (opts: Opts = {}) => ({
 				const ast = parse(this.appearNote.text);
 				// TODO: 再帰的にURL要素がないか調べる
 				return unique(ast
-					.filter(t => ((t.name == 'url' || t.name == 'link') && t.props.url && !t.props.silent))
-					.map(t => t.props.url));
+					.filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent))
+					.map(t => t.node.props.url));
 			} else {
 				return null;
 			}

src/client/app/common/views/components/messaging-room.message.vue

@@ -52,8 +52,8 @@ export default Vue.extend({
 			if (this.message.text) {
 				const ast = parse(this.message.text);
 				return unique(ast
-					.filter(t => ((t.name == 'url' || t.name == 'link') && t.props.url && !t.silent))
-					.map(t => t.props.url));
+					.filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent))
+					.map(t => t.node.props.url));
 			} else {
 				return null;
 			}

src/client/app/common/views/components/mfm.ts

@@ -1,6 +1,6 @@
 import Vue, { VNode } from 'vue';
 import { length } from 'stringz';
-import { Node } from '../../../../../mfm/parser';
+import { MfmForest } from '../../../../../mfm/parser';
 import parse from '../../../../../mfm/parse';
 import MkUrl from './url.vue';
 import MkMention from './mention.vue';
@@ -9,16 +9,11 @@ import MkFormula from './formula.vue';
 import MkGoogle from './google.vue';
 import syntaxHighlight from '../../../../../mfm/syntax-highlight';
 import { host } from '../../../config';
+import { preorderF, countNodesF } from '../../../../../prelude/tree';
 
-function getTextCount(tokens: Node[]): number {
-	const rootCount = sum(tokens.filter(x => x.name === 'text').map(x => length(x.props.text)));
-	const childrenCount = sum(tokens.filter(x => x.children).map(x => getTextCount(x.children)));
-	return rootCount + childrenCount;
-}
-
-function getChildrenCount(tokens: Node[]): number {
-	const countTree = tokens.filter(x => x.children).map(x => getChildrenCount(x.children));
-	return countTree.length + sum(countTree);
+function sumTextsLength(ts: MfmForest): number {
+	const textNodes = preorderF(ts).filter(n => n.type === 'text');
+	return sum(textNodes.map(x => length(x.props.text)));
 }
 
 export default Vue.component('misskey-flavored-markdown', {
@@ -27,10 +22,6 @@ export default Vue.component('misskey-flavored-markdown', {
 			type: String,
 			required: true
 		},
-		ast: {
-			type: [],
-			required: false
-		},
 		shouldBreak: {
 			type: Boolean,
 			default: true
@@ -55,17 +46,15 @@ export default Vue.component('misskey-flavored-markdown', {
 	render(createElement) {
 		if (this.text == null || this.text == '') return;
 
-		const ast = this.ast == null ?
-			parse(this.text, this.plainText) : // Parse text to ast
-			this.ast as Node[];
+		const ast = parse(this.text, this.plainText);
 
 		let bigCount = 0;
 		let motionCount = 0;
 
-		const genEl = (ast: Node[]) => concat(ast.map((token): VNode[] => {
-			switch (token.name) {
+		const genEl = (ast: MfmForest) => concat(ast.map((token): VNode[] => {
+			switch (token.node.type) {
 				case 'text': {
-					const text = token.props.text.replace(/(\r\n|\n|\r)/g, '\n');
+					const text = token.node.props.text.replace(/(\r\n|\n|\r)/g, '\n');
 
 					if (this.shouldBreak) {
 						const x = text.split('\n')
@@ -95,7 +84,7 @@ export default Vue.component('misskey-flavored-markdown', {
 
 				case 'big': {
 					bigCount++;
-					const isLong = getTextCount(token.children) > 10 || getChildrenCount(token.children) > 5;
+					const isLong = sumTextsLength(token.children) > 10 || countNodesF(token.children) > 5;
 					const isMany = bigCount > 3;
 					return (createElement as any)('strong', {
 						attrs: {
@@ -122,7 +111,7 @@ export default Vue.component('misskey-flavored-markdown', {
 
 				case 'motion': {
 					motionCount++;
-					const isLong = getTextCount(token.children) > 10 || getChildrenCount(token.children) > 5;
+					const isLong = sumTextsLength(token.children) > 10 || countNodesF(token.children) > 5;
 					const isMany = motionCount > 3;
 					return (createElement as any)('span', {
 						attrs: {
@@ -139,7 +128,7 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement(MkUrl, {
 						key: Math.random(),
 						props: {
-							url: token.props.url,
+							url: token.node.props.url,
 							target: '_blank',
 							style: 'color:var(--mfmLink);'
 						}
@@ -150,9 +139,9 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement('a', {
 						attrs: {
 							class: 'link',
-							href: token.props.url,
+							href: token.node.props.url,
 							target: '_blank',
-							title: token.props.url,
+							title: token.node.props.url,
 							style: 'color:var(--mfmLink);'
 						}
 					}, genEl(token.children))];
@@ -162,8 +151,8 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement(MkMention, {
 						key: Math.random(),
 						props: {
-							host: (token.props.host == null && this.author && this.author.host != null ? this.author.host : token.props.host) || host,
-							username: token.props.username
+							host: (token.node.props.host == null && this.author && this.author.host != null ? this.author.host : token.node.props.host) || host,
+							username: token.node.props.username
 						}
 					})];
 				}
@@ -172,10 +161,10 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement('router-link', {
 						key: Math.random(),
 						attrs: {
-							to: `/tags/${encodeURIComponent(token.props.hashtag)}`,
+							to: `/tags/${encodeURIComponent(token.node.props.hashtag)}`,
 							style: 'color:var(--mfmHashtag);'
 						}
-					}, `#${token.props.hashtag}`)];
+					}, `#${token.node.props.hashtag}`)];
 				}
 
 				case 'blockCode': {
@@ -184,7 +173,7 @@ export default Vue.component('misskey-flavored-markdown', {
 					}, [
 						createElement('code', {
 							domProps: {
-								innerHTML: syntaxHighlight(token.props.code)
+								innerHTML: syntaxHighlight(token.node.props.code)
 							}
 						})
 					])];
@@ -193,7 +182,7 @@ export default Vue.component('misskey-flavored-markdown', {
 				case 'inlineCode': {
 					return [createElement('code', {
 						domProps: {
-							innerHTML: syntaxHighlight(token.props.code)
+							innerHTML: syntaxHighlight(token.node.props.code)
 						}
 					})];
 				}
@@ -227,8 +216,8 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement('mk-emoji', {
 						key: Math.random(),
 						attrs: {
-							emoji: token.props.emoji,
-							name: token.props.name
+							emoji: token.node.props.emoji,
+							name: token.node.props.name
 						},
 						props: {
 							customEmojis: this.customEmojis || customEmojis,
@@ -242,7 +231,7 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement(MkFormula, {
 						key: Math.random(),
 						props: {
-							formula: token.props.formula
+							formula: token.node.props.formula
 						}
 					})];
 				}
@@ -252,13 +241,13 @@ export default Vue.component('misskey-flavored-markdown', {
 					return [createElement(MkGoogle, {
 						key: Math.random(),
 						props: {
-							q: token.props.query
+							q: token.node.props.query
 						}
 					})];
 				}
 
 				default: {
-					console.log('unknown ast type:', token.name);
+					console.log('unknown ast type:', token.node.type);
 
 					return [];
 				}

src/mfm/html.ts

@@ -2,10 +2,10 @@ const jsdom = require('jsdom');
 const { JSDOM } = jsdom;
 import config from '../config';
 import { INote } from '../models/note';
-import { Node } from './parser';
 import { intersperse } from '../prelude/array';
+import { MfmForest, MfmTree } from './parser';
 
-export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) => {
+export default (tokens: MfmForest, mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) => {
 	if (tokens == null) {
 		return null;
 	}
@@ -14,11 +14,11 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
 
 	const doc = window.document;
 
-	function appendChildren(children: Node[], targetElement: any): void {
-		for (const child of children.map(n => handlers[n.name](n))) targetElement.appendChild(child);
+	function appendChildren(children: MfmForest, targetElement: any): void {
+		for (const child of children.map(t => handlers[t.node.type](t))) targetElement.appendChild(child);
 	}
 
-	const handlers: { [key: string]: (token: Node) => any } = {
+	const handlers: { [key: string]: (token: MfmTree) => any } = {
 		bold(token) {
 			const el = doc.createElement('b');
 			appendChildren(token.children, el);
@@ -58,7 +58,7 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
 		blockCode(token) {
 			const pre = doc.createElement('pre');
 			const inner = doc.createElement('code');
-			inner.innerHTML = token.props.code;
+			inner.innerHTML = token.node.props.code;
 			pre.appendChild(inner);
 			return pre;
 		},
@@ -70,39 +70,39 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
 		},
 
 		emoji(token) {
-			return doc.createTextNode(token.props.emoji ? token.props.emoji : `:${token.props.name}:`);
+			return doc.createTextNode(token.node.props.emoji ? token.node.props.emoji : `:${token.node.props.name}:`);
 		},
 
 		hashtag(token) {
 			const a = doc.createElement('a');
-			a.href = `${config.url}/tags/${token.props.hashtag}`;
-			a.textContent = `#${token.props.hashtag}`;
+			a.href = `${config.url}/tags/${token.node.props.hashtag}`;
+			a.textContent = `#${token.node.props.hashtag}`;
 			a.setAttribute('rel', 'tag');
 			return a;
 		},
 
 		inlineCode(token) {
 			const el = doc.createElement('code');
-			el.textContent = token.props.code;
+			el.textContent = token.node.props.code;
 			return el;
 		},
 
 		math(token) {
 			const el = doc.createElement('code');
-			el.textContent = token.props.formula;
+			el.textContent = token.node.props.formula;
 			return el;
 		},
 
 		link(token) {
 			const a = doc.createElement('a');
-			a.href = token.props.url;
+			a.href = token.node.props.url;
 			appendChildren(token.children, a);
 			return a;
 		},
 
 		mention(token) {
 			const a = doc.createElement('a');
-			const { username, host, acct } = token.props;
+			const { username, host, acct } = token.node.props;
 			switch (host) {
 				case 'github.com':
 					a.href = `https://github.com/${username}`;
@@ -133,7 +133,7 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
 
 		text(token) {
 			const el = doc.createElement('span');
-			const nodes = (token.props.text as string).split('\n').map(x => doc.createTextNode(x));
+			const nodes = (token.node.props.text as string).split('\n').map(x => doc.createTextNode(x));
 
 			for (const x of intersperse('br', nodes)) {
 				el.appendChild(x === 'br' ? doc.createElement('br') : x);
@@ -144,15 +144,15 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
 
 		url(token) {
 			const a = doc.createElement('a');
-			a.href = token.props.url;
-			a.textContent = token.props.url;
+			a.href = token.node.props.url;
+			a.textContent = token.node.props.url;
 			return a;
 		},
 
 		search(token) {
 			const a = doc.createElement('a');
-			a.href = `https://www.google.com/?#q=${token.props.query}`;
-			a.textContent = token.props.content;
+			a.href = `https://www.google.com/?#q=${token.node.props.query}`;
+			a.textContent = token.node.props.content;
 			return a;
 		}
 	};

src/mfm/parse.ts

@@ -1,40 +1,36 @@
-import parser, { Node, plainParser } from './parser';
+import parser, { plainParser, MfmForest, MfmTree } from './parser';
 import * as A from '../prelude/array';
 import * as S from '../prelude/string';
+import { createTree, createLeaf } from '../prelude/tree';
 
-export default (source: string, plainText = false): Node[] => {
+function concatTextTrees(ts: MfmForest): MfmTree {
+	return createLeaf({ type: 'text', props: { text: S.concat(ts.map(x => x.node.props.text)) } });
+}
+
+function concatIfTextTrees(ts: MfmForest): MfmForest {
+	return ts[0].node.type === 'text' ? [concatTextTrees(ts)] : ts;
+}
+
+function concatConsecutiveTextTrees(ts: MfmForest): MfmForest {
+	const us = A.concat(A.groupOn(t => t.node.type, ts).map(concatIfTextTrees));
+	return us.map(t => createTree(t.node, concatConsecutiveTextTrees(t.children)));
+}
+
+function isEmptyTextTree(t: MfmTree): boolean {
+	return t.node.type == 'text' && t.node.props.text === '';
+}
+
+function removeEmptyTextNodes(ts: MfmForest): MfmForest {
+	return ts
+		.filter(t => !isEmptyTextTree(t))
+		.map(t => createTree(t.node, removeEmptyTextNodes(t.children)));
+}
+
+export default (source: string, plainText = false): MfmForest => {
 	if (source == null || source == '') {
 		return null;
 	}
 
-	let nodes: Node[] = plainText ? plainParser.root.tryParse(source) : parser.root.tryParse(source);
-
-	const combineText = (es: Node[]): Node =>
-		({ name: 'text', props: { text: S.concat(es.map(e => e.props.text)) } });
-
-	const concatText = (nodes: Node[]): Node[] =>
-		A.concat(A.groupOn(x => x.name, nodes).map(es =>
-			es[0].name === 'text' ? [combineText(es)] : es
-		));
-
-	const concatTextRecursive = (es: Node[]): void => {
-		for (const x of es.filter(x => x.children)) {
-			x.children = concatText(x.children);
-			concatTextRecursive(x.children);
-		}
-	};
-
-	nodes = concatText(nodes);
-	concatTextRecursive(nodes);
-
-	const removeEmptyTextNodes = (nodes: Node[]) => {
-		for (const n of nodes.filter(n => n.children)) {
-			n.children = removeEmptyTextNodes(n.children);
-		}
-		return nodes.filter(n => !(n.name == 'text' && n.props.text == ''));
-	};
-
-	nodes = removeEmptyTextNodes(nodes);
-
-	return nodes;
+	const raw = plainText ? plainParser.root.tryParse(source) : parser.root.tryParse(source) as MfmForest;
+	return removeEmptyTextNodes(concatConsecutiveTextTrees(raw));
 };

src/misc/extract-emojis.ts

@@ -0,0 +1,9 @@
+import { EmojiNode, MfmForest } from '../mfm/parser';
+import { preorderF } from '../prelude/tree';
+import { unique } from '../prelude/array';
+
+export default function(mfmForest: MfmForest): string[] {
+	const emojiNodes = preorderF(mfmForest).filter(x => x.type === 'emoji') as EmojiNode[];
+	const emojis = emojiNodes.filter(x => x.props.name && x.props.name.length <= 100).map(x => x.props.name);
+	return unique(emojis);
+}

src/misc/extract-hashtags.ts

@@ -0,0 +1,9 @@
+import { HashtagNode, MfmForest } from '../mfm/parser';
+import { preorderF } from '../prelude/tree';
+import { unique } from '../prelude/array';
+
+export default function(mfmForest: MfmForest): string[] {
+	const hashtagNodes = preorderF(mfmForest).filter(x => x.type === 'hashtag') as HashtagNode[];
+	const hashtags = hashtagNodes.map(x => x.props.hashtag);
+	return unique(hashtags);
+}

src/misc/extract-mentions.ts

@@ -1,19 +1,10 @@
-import parse from '../mfm/parse';
-import { Node, IMentionNode } from '../mfm/parser';
+// test is located in test/extract-mentions
 
-export default function(tokens: ReturnType<typeof parse>): IMentionNode['props'][] {
-	const mentions: IMentionNode['props'][] = [];
+import { MentionNode, MfmForest } from '../mfm/parser';
+import { preorderF } from '../prelude/tree';
 
-	const extract = (tokens: Node[]) => {
-		for (const x of tokens.filter(x => x.name === 'mention')) {
-			mentions.push(x.props);
-		}
-		for (const x of tokens.filter(x => x.children)) {
-			extract(x.children);
-		}
-	};
-
-	extract(tokens);
-
-	return mentions;
+export default function(mfmForest: MfmForest): MentionNode['props'][] {
+	// TODO: 重複を削除
+	const mentionNodes = preorderF(mfmForest).filter(x => x.type === 'mention') as MentionNode[];
+	return mentionNodes.map(x => x.props);
 }

src/prelude/tree.ts

@@ -0,0 +1,36 @@
+import { concat, sum } from './array';
+
+export type Tree<T> = {
+	node: T,
+	children: Forest<T>;
+};
+
+export type Forest<T> = Tree<T>[];
+
+export function createLeaf<T>(node: T): Tree<T> {
+	return { node, children: [] };
+}
+
+export function createTree<T>(node: T, children: Forest<T>): Tree<T> {
+	return { node, children };
+}
+
+export function hasChildren<T>(t: Tree<T>): boolean {
+	return t.children.length !== 0;
+}
+
+export function preorder<T>(t: Tree<T>): T[] {
+	return [t.node, ...preorderF(t.children)];
+}
+
+export function preorderF<T>(ts: Forest<T>): T[] {
+	return concat(ts.map(preorder));
+}
+
+export function countNodes<T>(t: Tree<T>): number {
+	return preorder(t).length;
+}
+
+export function countNodesF<T>(ts: Forest<T>): number {
+	return sum(ts.map(countNodes));
+}

src/server/api/endpoints/i/update.ts

@@ -7,7 +7,7 @@ import { publishToFollowers } from '../../../../services/i/update';
 import define from '../../define';
 import getDriveFileUrl from '../../../../misc/get-drive-file-url';
 import parse from '../../../../mfm/parse';
-import { extractEmojis } from '../../../../services/note/create';
+import extractEmojis from '../../../../misc/extract-emojis';
 const langmap = require('langmap');
 
 export const meta = {

src/services/note/create.ts

@@ -24,12 +24,13 @@ import isQuote from '../../misc/is-quote';
 import notesChart from '../../chart/notes';
 import perUserNotesChart from '../../chart/per-user-notes';
 
-import { erase, unique } from '../../prelude/array';
+import { erase } from '../../prelude/array';
 import insertNoteUnread from './unread';
 import registerInstance from '../register-instance';
 import Instance from '../../models/instance';
-import { Node } from '../../mfm/parser';
 import extractMentions from '../../misc/extract-mentions';
+import extractEmojis from '../../misc/extract-emojis';
+import extractHashtags from '../../misc/extract-hashtags';
 
 type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
 
@@ -466,44 +467,6 @@ async function insertNote(user: IUser, data: Option, tags: string[], emojis: str
 	}
 }
 
-function extractHashtags(tokens: ReturnType<typeof parse>): string[] {
-	const hashtags: string[] = [];
-
-	const extract = (tokens: Node[]) => {
-		for (const x of tokens.filter(x => x.name === 'hashtag')) {
-			hashtags.push(x.props.hashtag);
-		}
-		for (const x of tokens.filter(x => x.children)) {
-			extract(x.children);
-		}
-	};
-
-	// Extract hashtags
-	extract(tokens);
-
-	return unique(hashtags);
-}
-
-export function extractEmojis(tokens: ReturnType<typeof parse>): string[] {
-	const emojis: string[] = [];
-
-	const extract = (tokens: Node[]) => {
-		for (const x of tokens.filter(x => x.name === 'emoji')) {
-			if (x.props.name && x.props.name.length <= 100) {
-				emojis.push(x.props.name);
-			}
-		}
-		for (const x of tokens.filter(x => x.children)) {
-			extract(x.children);
-		}
-	};
-
-	// Extract emojis
-	extract(tokens);
-
-	return unique(emojis);
-}
-
 function index(note: INote) {
 	if (note.text == null || config.elasticsearch == null) return;
 

test/extract-mentions.ts

@@ -0,0 +1,48 @@
+import * as assert from 'assert';
+
+import extractMentions from '../src/misc/extract-mentions';
+import parse from '../src/mfm/parse';
+
+describe('Extract mentions', () => {
+	it('simple', () => {
+		const ast = parse('@foo @bar @baz');
+		const mentions = extractMentions(ast);
+		assert.deepStrictEqual(mentions, [{
+			username: 'foo',
+			acct: '@foo',
+			canonical: '@foo',
+			host: null
+		}, {
+			username: 'bar',
+			acct: '@bar',
+			canonical: '@bar',
+			host: null
+		}, {
+			username: 'baz',
+			acct: '@baz',
+			canonical: '@baz',
+			host: null
+		}]);
+	});
+
+	it('nested', () => {
+		const ast = parse('@foo **@bar** @baz');
+		const mentions = extractMentions(ast);
+		assert.deepStrictEqual(mentions, [{
+			username: 'foo',
+			acct: '@foo',
+			canonical: '@foo',
+			host: null
+		}, {
+			username: 'bar',
+			acct: '@bar',
+			canonical: '@bar',
+			host: null
+		}, {
+			username: 'baz',
+			acct: '@baz',
+			canonical: '@baz',
+			host: null
+		}]);
+	});
+});