From 61c532a85400a02e62fdbd0d91a474db0fcfe4f2 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Sat, 25 Nov 2023 22:25:24 +0100 Subject: [PATCH] [mastodon-client] Add html cache for user profiles and note contents --- .config/example-docker.yml | 19 +++ .config/example.yml | 19 +++ .pnp.cjs | 10 ++ ...ration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip | 3 + packages/backend/package.json | 1 + packages/backend/src/config/load.ts | 10 ++ packages/backend/src/config/types.ts | 7 + packages/backend/src/db/postgre.ts | 6 +- .../migration/1700962939886-add-html-cache.ts | 20 +++ .../models/entities/html-note-cache-entry.ts | 21 +++ .../models/entities/html-user-cache-entry.ts | 26 ++++ packages/backend/src/models/index.ts | 4 + .../src/remote/activitypub/models/person.ts | 11 +- .../server/api/mastodon/converters/note.ts | 99 ++++++++++++-- .../server/api/mastodon/converters/user.ts | 124 +++++++++++++++++- packages/backend/src/services/note/create.ts | 5 +- packages/backend/src/services/note/edit.ts | 3 + yarn.lock | 8 ++ 18 files changed, 373 insertions(+), 23 deletions(-) create mode 100644 .yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip create mode 100644 packages/backend/src/migration/1700962939886-add-html-cache.ts create mode 100644 packages/backend/src/models/entities/html-note-cache-entry.ts create mode 100644 packages/backend/src/models/entities/html-user-cache-entry.ts diff --git a/.config/example-docker.yml b/.config/example-docker.yml index 94d8c3f86..8105f5bec 100644 --- a/.config/example-docker.yml +++ b/.config/example-docker.yml @@ -184,6 +184,25 @@ reservedUsernames: [ # Upload or download file size limits (bytes) #maxFileSize: 262144000 +# ┌────────────────────────────────┐ +#───┘ Mastodon client API HTML Cache └────────────────────────── +# Caution: rendered post html content is stored in redis (in-memory cache) +# for the duration of ttl, so don't set it too high if you have little system memory. +# +# The prewarm option causes every incoming user/note create/update event to +# be rendered so the cache is always "warm". This trades background cpu load for +# better request response time and better scaling, as posts won't have to be rendered +# on request. +# +# The dbFallback option stores html data that expires into postgres, +# which is more expensive than fetching it from redis, +# but cheaper than re-rendering the HTML. + +#htmlCache: +# ttl: 1h +# prewarm: false +# dbFallback: false + #━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # Congrats, you've reached the end of the config file needed for most deployments! # Enjoy your Iceshrimp server! diff --git a/.config/example.yml b/.config/example.yml index d0bbe7a3c..53b21523c 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -184,6 +184,25 @@ reservedUsernames: [ # Upload or download file size limits (bytes) #maxFileSize: 262144000 +# ┌────────────────────────────────┐ +#───┘ Mastodon client API HTML Cache └────────────────────────── +# Caution: rendered post html content is stored in redis (in-memory cache) +# for the duration of ttl, so don't set it too high if you have little system memory. +# +# The prewarm option causes every incoming user/note create/update event to +# be rendered so the cache is always "warm". This trades background cpu load for +# better request response time and better scaling, as posts won't have to be rendered +# on request. +# +# The dbFallback option stores html data that expires into postgres, +# which is more expensive than fetching it from redis, +# but cheaper than re-rendering the HTML. + +#htmlCache: +# ttl: 1h +# prewarm: false +# dbFallback: false + #━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # Congrats, you've reached the end of the config file needed for most deployments! # Enjoy your Iceshrimp server! diff --git a/.pnp.cjs b/.pnp.cjs index 367e7018d..0d4507ec4 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -7275,6 +7275,7 @@ const RAW_RUNTIME_STATE = ["oauth", "npm:0.10.0"],\ ["os-utils", "npm:0.0.14"],\ ["otpauth", "npm:9.1.4"],\ + ["parse-duration", "npm:1.1.0"],\ ["parse5", "npm:7.1.2"],\ ["pg", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:8.11.1"],\ ["private-ip", "npm:2.3.4"],\ @@ -19221,6 +19222,15 @@ const RAW_RUNTIME_STATE = "linkType": "HARD"\ }]\ ]],\ + ["parse-duration", [\ + ["npm:1.1.0", {\ + "packageLocation": "./.yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip/node_modules/parse-duration/",\ + "packageDependencies": [\ + ["parse-duration", "npm:1.1.0"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["parse-entities", [\ ["npm:2.0.0", {\ "packageLocation": "./.yarn/cache/parse-entities-npm-2.0.0-b7b4f46ff6-feb46b5167.zip/node_modules/parse-entities/",\ diff --git a/.yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip b/.yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip new file mode 100644 index 000000000..81beef5f6 --- /dev/null +++ b/.yarn/cache/parse-duration-npm-1.1.0-cb12528e2a-c26ab1e3fd.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fdc1bc00d94b96b9464bc20dca15b4c0485a4d330cc5d229ee8ecb64268340f +size 4633 diff --git a/packages/backend/package.json b/packages/backend/package.json index 193b72b3a..7ff83b54d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -98,6 +98,7 @@ "oauth": "^0.10.0", "os-utils": "0.0.14", "otpauth": "^9.1.3", + "parse-duration": "^1.1.0", "parse5": "7.1.2", "pg": "8.11.1", "private-ip": "2.3.4", diff --git a/packages/backend/src/config/load.ts b/packages/backend/src/config/load.ts index 5d894b023..0302e45c9 100644 --- a/packages/backend/src/config/load.ts +++ b/packages/backend/src/config/load.ts @@ -8,6 +8,7 @@ import { dirname } from "node:path"; import * as yaml from "js-yaml"; import type { Source, Mixin } from "./types.js"; import Path from "node:path"; +import parseDuration from 'parse-duration' export default function load() { const _filename = fileURLToPath(import.meta.url); @@ -53,6 +54,15 @@ export default function load() { ...config.images, }; + config.htmlCache = { + ttlSeconds: parseDuration(config.htmlCache?.ttl ?? '1h', 's')!, + prewarm: false, + dbFallback: false, + ...config.htmlCache, + } + + if (config.htmlCache.ttlSeconds == null) throw new Error('Failed to parse config.ttl'); + config.searchEngine = config.searchEngine ?? 'https://duckduckgo.com/?q='; mixin.version = meta.version; diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index cfa521c97..c3def3add 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -41,6 +41,13 @@ export type Source = { info?: string; }; + htmlCache?: { + ttl?: string; + ttlSeconds?: number; + prewarm?: boolean; + dbFallback?: boolean; + } + searchEngine?: string; proxy?: string; diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 0a1d24843..6bab35e6b 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -71,12 +71,12 @@ import { UserPending } from "@/models/entities/user-pending.js"; import { Webhook } from "@/models/entities/webhook.js"; import { UserIp } from "@/models/entities/user-ip.js"; import { NoteEdit } from "@/models/entities/note-edit.js"; - import { entities as charts } from "@/services/chart/entities.js"; -import { envOption } from "../env.js"; import { dbLogger } from "./logger.js"; import { OAuthApp } from "@/models/entities/oauth-app.js"; import { OAuthToken } from "@/models/entities/oauth-token.js"; +import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js"; +import { HtmlUserCacheEntry } from "@/models/entities/html-user-cache-entry.js"; const sqlLogger = dbLogger.createSubLogger("sql", "gray", false); class MyCustomLogger implements Logger { @@ -179,6 +179,8 @@ export const entities = [ UserIp, OAuthApp, OAuthToken, + HtmlNoteCacheEntry, + HtmlUserCacheEntry, ...charts, ]; diff --git a/packages/backend/src/migration/1700962939886-add-html-cache.ts b/packages/backend/src/migration/1700962939886-add-html-cache.ts new file mode 100644 index 000000000..ee90725aa --- /dev/null +++ b/packages/backend/src/migration/1700962939886-add-html-cache.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddHtmlCache1700962939886 implements MigrationInterface { + name = 'AddHtmlCache1700962939886' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "html_note_cache_entry" ("noteId" character varying(32) NOT NULL, "updatedAt" TIMESTAMP WITH TIME ZONE, "content" text, CONSTRAINT "PK_6ef86ec901b2017cbe82d3a8286" PRIMARY KEY ("noteId"))`); + await queryRunner.query(`CREATE TABLE "html_user_cache_entry" ("userId" character varying(32) NOT NULL, "updatedAt" TIMESTAMP WITH TIME ZONE, "bio" text, "fields" jsonb NOT NULL DEFAULT '[]', CONSTRAINT "PK_920b9474e3c9cae3f3c37c057e1" PRIMARY KEY ("userId"))`); + await queryRunner.query(`ALTER TABLE "html_note_cache_entry" ADD CONSTRAINT "FK_6ef86ec901b2017cbe82d3a8286" FOREIGN KEY ("noteId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "html_user_cache_entry" ADD CONSTRAINT "FK_920b9474e3c9cae3f3c37c057e1" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "html_user_cache_entry" DROP CONSTRAINT "FK_920b9474e3c9cae3f3c37c057e1"`); + await queryRunner.query(`ALTER TABLE "html_note_cache_entry" DROP CONSTRAINT "FK_6ef86ec901b2017cbe82d3a8286"`); + await queryRunner.query(`DROP TABLE "html_user_cache_entry"`); + await queryRunner.query(`DROP TABLE "html_note_cache_entry"`); + } + +} diff --git a/packages/backend/src/models/entities/html-note-cache-entry.ts b/packages/backend/src/models/entities/html-note-cache-entry.ts new file mode 100644 index 000000000..58debc385 --- /dev/null +++ b/packages/backend/src/models/entities/html-note-cache-entry.ts @@ -0,0 +1,21 @@ +import { Entity, PrimaryColumn, Column, ManyToOne, JoinColumn } from "typeorm"; +import { id } from "../id.js"; +import { Note } from "./note.js"; + +@Entity() +export class HtmlNoteCacheEntry { + @PrimaryColumn(id()) + public noteId: Note["id"]; + + @ManyToOne((type) => Note, { + onDelete: "CASCADE", + }) + @JoinColumn() + public note: Note | null; + + @Column("timestamp with time zone", { nullable: true }) + public updatedAt: Date; + + @Column("text", { nullable: true }) + public content: string | null; +} diff --git a/packages/backend/src/models/entities/html-user-cache-entry.ts b/packages/backend/src/models/entities/html-user-cache-entry.ts new file mode 100644 index 000000000..aa65819a9 --- /dev/null +++ b/packages/backend/src/models/entities/html-user-cache-entry.ts @@ -0,0 +1,26 @@ +import { Entity, PrimaryColumn, Column, ManyToOne, JoinColumn } from "typeorm"; +import { User } from "@/models/entities/user.js"; +import { id } from "../id.js"; + +@Entity() +export class HtmlUserCacheEntry { + @PrimaryColumn(id()) + public userId: User["id"]; + + @ManyToOne((type) => User, { + onDelete: "CASCADE", + }) + @JoinColumn() + public user: User | null; + + @Column("timestamp with time zone", { nullable: true }) + public updatedAt: Date; + + @Column("text", { nullable: true }) + public bio: string | null; + + @Column("jsonb", { + default: [], + }) + public fields: MastodonEntity.Field[]; +} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index 1ca1857cd..2fafc9037 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -69,6 +69,8 @@ import { NoteEdit } from "./entities/note-edit.js"; import { OAuthApp } from "@/models/entities/oauth-app.js"; import { OAuthToken } from "@/models/entities/oauth-token.js"; import { UserProfileRepository } from "@/models/repositories/user-profile.js"; +import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js"; +import { HtmlUserCacheEntry } from "@/models/entities/html-user-cache-entry.js"; export const Announcements = db.getRepository(Announcement); export const AnnouncementReads = db.getRepository(AnnouncementRead); @@ -136,3 +138,5 @@ export const Webhooks = db.getRepository(Webhook); export const PasswordResetRequests = db.getRepository(PasswordResetRequest); export const OAuthApps = db.getRepository(OAuthApp); export const OAuthTokens = db.getRepository(OAuthToken); +export const HtmlUserCacheEntries = db.getRepository(HtmlUserCacheEntry); +export const HtmlNoteCacheEntries = db.getRepository(HtmlNoteCacheEntry); diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index 3870a8f08..c021dd679 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -54,6 +54,7 @@ import { getSubjectHostFromAcctParts } from "@/remote/resolve-user.js" import { RecursionLimiter } from "@/models/repositories/user-profile.js"; +import { UserConverter } from "@/server/api/mastodon/converters/user.js"; const logger = apLogger; @@ -397,8 +398,9 @@ export async function createPerson( // Hashtag update updateUsertags(user!, tags); - // Mentions update - if (await limiter.shouldContinue()) UserProfiles.updateMentions(user!.id, limiter); + // Mentions update, then prewarm html cache + if (await limiter.shouldContinue()) UserProfiles.updateMentions(user!.id, limiter) + .then(_ => UserConverter.prewarmCacheById(user!.id)); //#region Fetch avatar and header image const [avatar, banner] = await Promise.all( @@ -635,8 +637,9 @@ export async function updatePerson( // Hashtag Update updateUsertags(user, tags); - // Mentions update - UserProfiles.updateMentions(user!.id); + // Mentions update, then prewarm html cache + UserProfiles.updateMentions(user!.id) + .then(_ => UserConverter.prewarmCacheById(user!.id)); // If the user in question is a follower, followers will also be updated. await Followings.update( diff --git a/packages/backend/src/server/api/mastodon/converters/note.ts b/packages/backend/src/server/api/mastodon/converters/note.ts index 1cae2d2ba..a02249b97 100644 --- a/packages/backend/src/server/api/mastodon/converters/note.ts +++ b/packages/backend/src/server/api/mastodon/converters/note.ts @@ -8,7 +8,15 @@ import { VisibilityConverter } from "@/server/api/mastodon/converters/visibility import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js"; import { aggregateNoteEmojis, PopulatedEmoji, populateEmojis, prefetchEmojis } from "@/misc/populate-emojis.js"; import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js"; -import { DriveFiles, NoteFavorites, NoteReactions, Notes, NoteThreadMutings, UserNotePinings } from "@/models/index.js"; +import { + DriveFiles, + HtmlNoteCacheEntries, + NoteFavorites, + NoteReactions, + Notes, + NoteThreadMutings, + UserNotePinings +} from "@/models/index.js"; import { decodeReaction } from "@/misc/reaction-lib.js"; import { MentionConverter } from "@/server/api/mastodon/converters/mention.js"; import { PollConverter } from "@/server/api/mastodon/converters/poll.js"; @@ -23,8 +31,11 @@ import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js"; import isQuote from "@/misc/is-quote.js"; import { unique } from "@/prelude/array.js"; import { NoteReaction } from "@/models/entities/note-reaction.js"; +import { Cache } from "@/misc/cache.js"; +import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js"; export class NoteConverter { + private static noteContentHtmlCache = new Cache('html:note:content', config.htmlCache?.ttlSeconds ?? 60 * 60); public static async encode(note: Note, ctx: MastoContext, recurseCounter: number = 2): Promise { const user = ctx.user as ILocalUser | null; const noteUser = note.user ?? UserHelpers.getUserCached(note.userId, ctx); @@ -102,12 +113,16 @@ export class NoteConverter { return renote.url ?? renote.uri ?? `${config.url}/notes/${renote.id}`; }); + const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`; + const text = quoteUri.then(quoteUri => note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null); - const content = text.then(text => text !== null - ? quoteUri.then(quoteUri => MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri)) - .then(p => p ?? escapeMFM(text)) - : ""); + const content = this.noteContentHtmlCache.fetch(identifier, async () => + Promise.resolve(await this.fetchFromCacheWithFallback(note, ctx) ?? text.then(text => text !== null + ? quoteUri.then(quoteUri => MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri)) + .then(p => p ?? escapeMFM(text)) + : "")), true) + .then(p => p ?? ''); const isPinned = (ctx.pinAggregate as Map)?.get(note.id) ?? (user && note.userId === user.id @@ -174,16 +189,28 @@ export class NoteConverter { const reactionAggregate = new Map(); const renoteAggregate = new Map(); const mutingAggregate = new Map(); - const bookmarkAggregate = new Map();; + const bookmarkAggregate = new Map(); const pinAggregate = new Map(); + const htmlNoteCacheAggregate = new Map(); + + const renoteIds = notes + .filter((n) => n.renoteId != null) + .map((n) => n.renoteId!); + + const noteIds = unique(notes.map((n) => n.id)); + const targets = unique([...noteIds, ...renoteIds]); + + if (config.htmlCache?.dbFallback) { + const htmlNoteCacheEntries = await HtmlNoteCacheEntries.findBy({ + noteId: In(targets) + }); + + for (const target of targets) { + htmlNoteCacheAggregate.set(target, htmlNoteCacheEntries.find(n => n.noteId === target) ?? null); + } + } if (user?.id != null) { - const renoteIds = notes - .filter((n) => n.renoteId != null) - .map((n) => n.renoteId!); - - const noteIds = unique(notes.map((n) => n.id)); - const targets = unique([...noteIds, ...renoteIds]); const mutingTargets = unique([...notes.map(n => n.threadId ?? n.id)]); const pinTargets = unique([...notes.filter(n => n.userId === user.id).map(n => n.id)]); @@ -239,9 +266,12 @@ export class NoteConverter { ctx.mutingAggregate = mutingAggregate; ctx.bookmarkAggregate = bookmarkAggregate; ctx.pinAggregate = pinAggregate; + ctx.htmlNoteCacheAggregate = htmlNoteCacheAggregate; const users = notes.filter(p => !!p.user).map(p => p.user as User); + const renoteUserIds = notes.filter(p => p.renoteUserId !== null).map(p => p.renoteUserId as string); await UserConverter.aggregateData([...users], ctx) + await UserConverter.aggregateDataByIds(renoteUserIds, ctx); await prefetchEmojis(aggregateNoteEmojis(notes)); } @@ -268,4 +298,49 @@ export class NoteConverter { NoteHelpers.fixupEventNote(note); return NoteConverter.encode(note, ctx); } + + private static async fetchFromCacheWithFallback(note: Note, ctx: MastoContext): Promise { + if (!config.htmlCache?.dbFallback) return null; + + let dbHit: HtmlNoteCacheEntry | Promise | null | undefined = (ctx.htmlNoteCacheAggregate as Map | undefined)?.get(note.id); + if (dbHit === undefined) dbHit = HtmlNoteCacheEntries.findOneBy({ noteId: note.id }); + + return Promise.resolve(dbHit) + .then(res => { + if (res === null || (res.updatedAt !== note.updatedAt)) { + this.prewarmCache(note); + return null; + } + return res; + }) + .then(hit => hit?.updatedAt === note.updatedAt ? hit?.content ?? null : null); + } + + public static async prewarmCache(note: Note): Promise { + if (!config.htmlCache?.prewarm) return; + const identifier = `${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}`; + if (await this.noteContentHtmlCache.get(identifier) !== undefined) return; + + const quoteUri = note.renote + ? isQuote(note) + ? (note.renote.url ?? note.renote.uri ?? `${config.url}/notes/${note.renote.id}`) + : null + : null; + + const text = note.text !== null ? quoteUri !== null ? note.text.replaceAll(`RE: ${quoteUri}`, '').replaceAll(quoteUri, '').trimEnd() : note.text : null; + const content = text !== null + ? MfmHelpers.toHtml(mfm.parse(text), JSON.parse(note.mentionedRemoteUsers), note.userHost, false, quoteUri) + .then(p => p ?? escapeMFM(text)) + : null; + + if (note.user) UserConverter.prewarmCache(note.user); + else if (note.userId) UserConverter.prewarmCacheById(note.userId); + + if (note.replyUserId) UserConverter.prewarmCacheById(note.replyUserId); + if (note.renoteUserId) UserConverter.prewarmCacheById(note.renoteUserId); + this.noteContentHtmlCache.set(identifier, await content); + + if (config.htmlCache?.dbFallback) + HtmlNoteCacheEntries.upsert({ noteId: note.id, updatedAt: note.updatedAt, content: await content }, ["noteId"]); + } } diff --git a/packages/backend/src/server/api/mastodon/converters/user.ts b/packages/backend/src/server/api/mastodon/converters/user.ts index a4bb085dd..5cf5e559d 100644 --- a/packages/backend/src/server/api/mastodon/converters/user.ts +++ b/packages/backend/src/server/api/mastodon/converters/user.ts @@ -1,6 +1,6 @@ import { ILocalUser, User } from "@/models/entities/user.js"; import config from "@/config/index.js"; -import { DriveFiles, Followings, UserProfiles, Users } from "@/models/index.js"; +import {DriveFiles, Followings, HtmlUserCacheEntries, UserProfiles, Users} from "@/models/index.js"; import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js"; import { populateEmojis } from "@/misc/populate-emojis.js"; import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js"; @@ -9,10 +9,14 @@ import { awaitAll } from "@/prelude/await-all.js"; import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js"; import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js"; import { MastoContext } from "@/server/api/mastodon/index.js"; -import { IMentionedRemoteUsers } from "@/models/entities/note.js"; +import {IMentionedRemoteUsers, Note} from "@/models/entities/note.js"; import { UserProfile } from "@/models/entities/user-profile.js"; import { In } from "typeorm"; import { unique } from "@/prelude/array.js"; +import { Cache } from "@/misc/cache.js"; +import { getUser } from "../../common/getters.js"; +import { HtmlUserCacheEntry } from "@/models/entities/html-user-cache-entry.js"; +import AsyncLock from "async-lock"; type Field = { name: string; @@ -21,6 +25,9 @@ type Field = { }; export class UserConverter { + private static userBioHtmlCache = new Cache('html:user:bio', config.htmlCache?.ttlSeconds ?? 60 * 60); + private static userFieldsHtmlCache = new Cache('html:user:fields', config.htmlCache?.ttlSeconds ?? 60 * 60); + public static async encode(u: User, ctx: MastoContext): Promise { const localUser = ctx.user as ILocalUser | null; const cache = ctx.cache as AccountCache; @@ -28,6 +35,7 @@ export class UserConverter { const cacheHit = cache.accounts.find(p => p.id == u.id); if (cacheHit) return cacheHit; + const identifier = `${u.id}:${(u.updatedAt ?? u.createdAt).getTime()}`; let fqn = `${u.username}@${u.host ?? config.domain}`; let acct = u.username; let acctUrl = `https://${u.host || config.host}/@${u.username}`; @@ -38,15 +46,33 @@ export class UserConverter { const aggregateProfile = (ctx.userProfileAggregate as Map)?.get(u.id); + let htmlCacheEntry: HtmlUserCacheEntry | null | undefined = undefined; + const htmlCacheEntryLock = new AsyncLock(); + const profile = aggregateProfile !== undefined ? aggregateProfile : UserProfiles.findOneBy({ userId: u.id }); - const bio = Promise.resolve(profile).then(profile => MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, u.host).then(p => p ?? escapeMFM(profile?.description ?? ""))); + const bio = this.userBioHtmlCache.fetch(identifier, async () => { + return htmlCacheEntryLock.acquire(u.id, async () => { + if (htmlCacheEntry === undefined) await this.fetchFromCacheWithFallback(u, await profile, ctx); + if (htmlCacheEntry === null) { + return Promise.resolve(profile).then(async profile => { + return MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, u.host) + .then(p => p ?? escapeMFM(profile?.description ?? "")) + .then(p => p !== '

' ? p : null) + }); + } + return htmlCacheEntry?.bio ?? null; + }); + }, true) + .then(p => p ?? '

'); + const avatar = u.avatarId ? DriveFiles.getFinalUrlMaybe(u.avatarUrl) ?? (DriveFiles.findOneBy({ id: u.avatarId })) .then(p => p?.url ?? Users.getIdenticonUrl(u.id)) .then(p => DriveFiles.getFinalUrl(p)) : Users.getIdenticonUrl(u.id); + const banner = u.bannerId ? DriveFiles.getFinalUrlMaybe(u.bannerUrl) ?? (DriveFiles.findOneBy({ id: u.bannerId })) .then(p => p?.url ?? `${config.url}/static-assets/transparent.png`) @@ -75,6 +101,7 @@ export class UserConverter { return localUser?.id === profile.userId ? u.followersCount : 0; } }); + const followingCount = Promise.resolve(profile).then(async profile => { if (profile === null) return u.followingCount; switch (profile.ffVisibility) { @@ -87,6 +114,17 @@ export class UserConverter { } }); + const fields = + this.userFieldsHtmlCache.fetch(identifier, async () => { + return htmlCacheEntryLock.acquire(u.id, async () => { + if (htmlCacheEntry === undefined) await this.fetchFromCacheWithFallback(u, await profile, ctx); + if (htmlCacheEntry === null) { + return Promise.resolve(profile).then(profile => Promise.all(profile?.fields.map(async p => this.encodeField(p, u.host, profile?.mentions)) ?? [])); + } + return htmlCacheEntry?.fields ?? []; + }); + }, true); + return awaitAll({ id: u.id, username: u.username, @@ -106,7 +144,7 @@ export class UserConverter { header_static: banner, emojis: populateEmojis(u.emojis, u.host).then(emoji => emoji.map((e) => EmojiConverter.encode(e))), moved: null, //FIXME - fields: Promise.resolve(profile).then(profile => Promise.all(profile?.fields.map(async p => this.encodeField(p, u.host, profile?.mentions)) ?? [])), + fields: fields, bot: u.isBot, discoverable: u.isExplorable }).then(p => { @@ -124,6 +162,17 @@ export class UserConverter { const followedOrSelfAggregate = new Map(); const userProfileAggregate = new Map(); + const htmlUserCacheAggregate = ctx.htmlUserCacheAggregate ?? new Map(); + + if (config.htmlCache?.dbFallback) { + const htmlUserCacheEntries = await HtmlUserCacheEntries.findBy({ + userId: In(targets) + }); + + for (const target of targets) { + htmlUserCacheAggregate.set(target, htmlUserCacheEntries.find(n => n.userId === target) ?? null); + } + } if (user) { const targetsWithoutSelf = targets.filter(u => u !== user.id); @@ -152,6 +201,24 @@ export class UserConverter { } ctx.followedOrSelfAggregate = followedOrSelfAggregate; + ctx.htmlUserCacheAggregate = htmlUserCacheAggregate; + } + + public static async aggregateDataByIds(userIds: User["id"][], ctx: MastoContext): Promise { + const targets = unique(userIds); + const htmlUserCacheAggregate = ctx.htmlUserCacheAggregate ?? new Map(); + + if (config.htmlCache?.dbFallback) { + const htmlUserCacheEntries = await HtmlUserCacheEntries.findBy({ + userId: In(targets) + }); + + for (const target of targets) { + htmlUserCacheAggregate.set(target, htmlUserCacheEntries.find(n => n.userId === target) ?? null); + } + } + + ctx.htmlUserCacheAggregate = htmlUserCacheAggregate; } public static async encodeMany(users: User[], ctx: MastoContext): Promise { @@ -167,4 +234,53 @@ export class UserConverter { verified_at: f.verified ? (new Date()).toISOString() : null, } } + + private static async fetchFromCacheWithFallback(user: User, profile: UserProfile | null, ctx: MastoContext): Promise { + if (!config.htmlCache?.dbFallback) return null; + + let dbHit: HtmlUserCacheEntry | Promise | null | undefined = (ctx.htmlUserCacheAggregate as Map | undefined)?.get(user.id); + if (dbHit === undefined) dbHit = HtmlUserCacheEntries.findOneBy({ userId: user.id }); + + return Promise.resolve(dbHit) + .then(res => { + if (res === null || (res.updatedAt !== user.updatedAt ?? user.createdAt)) { + this.prewarmCache(user, profile); + return null; + } + return res; + }); + } + + public static async prewarmCache(user: User, profile?: UserProfile | null): Promise { + if (!config.htmlCache?.prewarm) return; + const identifier = `${user.id}:${(user.updatedAt ?? user.createdAt).getTime()}`; + if (profile !== null) { + if (profile === undefined) { + profile = await UserProfiles.findOneBy({userId: user.id}); + } + + if (await this.userBioHtmlCache.get(identifier) === undefined) { + const bio = MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, user.host) + .then(p => p ?? escapeMFM(profile?.description ?? "")) + .then(p => p !== '

' ? p : null); + + this.userBioHtmlCache.set(identifier, await bio); + + if (config.htmlCache?.dbFallback) + HtmlUserCacheEntries.upsert({ userId: user.id, bio: await bio }, ["userId"]); + } + + if (await this.userFieldsHtmlCache.get(identifier) === undefined) { + const fields = await Promise.all(profile!.fields.map(async p => this.encodeField(p, user.host, profile!.mentions)) ?? []); + this.userFieldsHtmlCache.set(identifier, fields); + + if (config.htmlCache?.dbFallback) + HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.updatedAt ?? user.createdAt, fields: fields }, ["userId"]); + } + } + } + + public static async prewarmCacheById(userId: string): Promise { + await this.prewarmCache(await getUser(userId)); + } } diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index f0b4cd7eb..9085f9c9d 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -65,6 +65,7 @@ import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; import { redisClient } from "@/db/redis.js"; import { Mutex } from "redis-semaphore"; import { RecursionLimiter } from "@/models/repositories/user-profile.js"; +import { NoteConverter } from "@/server/api/mastodon/converters/note.js"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] @@ -341,9 +342,11 @@ export default async ( ) { await incRenoteCount(data.renote); } - res(note); + // Prewarm html cache + NoteConverter.prewarmCache(note); + // 統計を更新 notesChart.update(note, true); perUserNotesChart.update(user, note, true); diff --git a/packages/backend/src/services/note/edit.ts b/packages/backend/src/services/note/edit.ts index 88dee2494..c100148cb 100644 --- a/packages/backend/src/services/note/edit.ts +++ b/packages/backend/src/services/note/edit.ts @@ -26,6 +26,7 @@ import { deliverToRelays } from "../relay.js"; import renderUpdate from "@/remote/activitypub/renderer/update.js"; import { extractMentionedUsers } from "@/services/note/create.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js"; +import { NoteConverter } from "@/server/api/mastodon/converters/note.js"; type Option = { text?: string | null; @@ -182,6 +183,8 @@ export default async function ( note = await Notes.findOneByOrFail({ id: note.id }); if (publishing) { + NoteConverter.prewarmCache(note); + // Publish update event for the updated note details publishNoteStream(note.id, "updated", { updatedAt: update.updatedAt, diff --git a/yarn.lock b/yarn.lock index ce9df6b49..508fa77bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5575,6 +5575,7 @@ __metadata: oauth: "npm:^0.10.0" os-utils: "npm:0.0.14" otpauth: "npm:^9.1.3" + parse-duration: "npm:^1.1.0" parse5: "npm:7.1.2" pg: "npm:8.11.1" private-ip: "npm:2.3.4" @@ -15999,6 +16000,13 @@ __metadata: languageName: node linkType: hard +"parse-duration@npm:^1.1.0": + version: 1.1.0 + resolution: "parse-duration@npm:1.1.0" + checksum: c26ab1e3fdf1dc4b7006e87a82fd33c7dbee3116413a59369bbc3b160a8e7ed88616852c4c3dde23b7a857e270cb18fccf629ff52220803194239f8e092774a9 + languageName: node + linkType: hard + "parse-entities@npm:^2.0.0": version: 2.0.0 resolution: "parse-entities@npm:2.0.0"