diff --git a/.pnp.cjs b/.pnp.cjs index 55b58603f..2392f24fa 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -14960,6 +14960,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@tensorflow/tfjs-core", "npm:4.9.0"],\ ["@tensorflow/tfjs-node", "npm:3.21.1"],\ ["@types/adm-zip", "npm:0.5.0"],\ + ["@types/async-lock", "npm:1.4.0"],\ ["@types/bcryptjs", "npm:2.4.2"],\ ["@types/cbor", "npm:6.0.0"],\ ["@types/escape-regexp", "npm:0.0.1"],\ @@ -15006,6 +15007,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["ajv", "npm:8.12.0"],\ ["archiver", "npm:5.3.1"],\ ["argon2", "npm:0.30.3"],\ + ["async-lock", "npm:1.4.0"],\ ["autolinker", "npm:4.0.0"],\ ["autwh", "npm:0.1.0"],\ ["aws-sdk", "npm:2.1413.0"],\ diff --git a/packages/backend/package.json b/packages/backend/package.json index 354425191..f3dbe5399 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -44,6 +44,7 @@ "ajv": "8.12.0", "archiver": "5.3.1", "argon2": "^0.30.3", + "async-lock": "1.4.0", "autolinker": "4.0.0", "autwh": "0.1.0", "aws-sdk": "2.1413.0", @@ -146,6 +147,7 @@ "@swc/cli": "^0.1.62", "@swc/core": "^1.3.68", "@types/adm-zip": "^0.5.0", + "@types/async-lock": "1.4.0", "@types/bcryptjs": "2.4.2", "@types/cbor": "6.0.0", "@types/escape-regexp": "0.0.1", diff --git a/packages/backend/src/server/api/mastodon/converters/note.ts b/packages/backend/src/server/api/mastodon/converters/note.ts index 9fda8275a..dd3f4ce94 100644 --- a/packages/backend/src/server/api/mastodon/converters/note.ts +++ b/packages/backend/src/server/api/mastodon/converters/note.ts @@ -16,10 +16,11 @@ import { PollConverter } from "@/server/api/mastodon/converters/poll.js"; import { populatePoll } from "@/models/repositories/note.js"; import { FileConverter } from "@/server/api/mastodon/converters/file.js"; import { awaitAll } from "@/prelude/await-all.js"; +import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js"; export class NoteConverter { - public static async encode(note: Note, user: ILocalUser | null): Promise { - const noteUser = note.user ?? getUser(note.userId); + public static async encode(note: Note, user: ILocalUser | null, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise { + const noteUser = note.user ?? UserHelpers.getUserCached(note.userId, cache); if (!await Notes.isVisibleForMe(note, user?.id ?? null)) throw new Error('Cannot encode note not visible for user'); @@ -72,22 +73,20 @@ export class NoteConverter { const files = DriveFiles.packMany(note.fileIds); const mentions = Promise.all(note.mentions.map(p => - getUser(p) + UserHelpers.getUserCached(p, cache) .then(u => MentionConverter.encode(u, JSON.parse(note.mentionedRemoteUsers))) .catch(() => null))) .then(p => p.filter(m => m)) as Promise; - // FIXME use await-all - // noinspection ES6MissingAwait return await awaitAll({ id: note.id, uri: note.uri ? note.uri : `https://${config.host}/notes/${note.id}`, url: note.uri ? note.uri : `https://${config.host}/notes/${note.id}`, - account: Promise.resolve(noteUser).then(p => UserConverter.encode(p)), + account: Promise.resolve(noteUser).then(p => UserConverter.encode(p, cache)), in_reply_to_id: note.replyId, in_reply_to_account_id: Promise.resolve(reply).then(reply => reply?.userId ?? null), - reblog: note.renote ? this.encode(note.renote, user) : null, + reblog: note.renote ? this.encode(note.renote, user, cache) : null, content: note.text ? toHtml(mfm.parse(note.text), JSON.parse(note.mentionedRemoteUsers)) ?? escapeMFM(note.text) : "", text: note.text ? note.text : null, created_at: note.createdAt.toISOString(), @@ -116,12 +115,12 @@ export class NoteConverter { // Use emojis list to provide URLs for emoji reactions. reactions: [], //FIXME: this.mapReactions(n.emojis, n.reactions, n.myReaction), bookmarked: isBookmarked, - quote: note.renote && note.text ? this.encode(note.renote, user) : null, + quote: note.renote && note.text ? this.encode(note.renote, user, cache) : null, }); } - public static async encodeMany(notes: Note[], user: ILocalUser | null): Promise { - const encoded = notes.map(n => this.encode(n, user)); + public static async encodeMany(notes: Note[], user: ILocalUser | null, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise { + const encoded = notes.map(n => this.encode(n, user, cache)); return Promise.all(encoded); } } diff --git a/packages/backend/src/server/api/mastodon/converters/user.ts b/packages/backend/src/server/api/mastodon/converters/user.ts index 1f2b650de..1a49b7eff 100644 --- a/packages/backend/src/server/api/mastodon/converters/user.ts +++ b/packages/backend/src/server/api/mastodon/converters/user.ts @@ -7,6 +7,7 @@ import { toHtml } from "@/mfm/to-html.js"; import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js"; import mfm from "mfm-js"; import { awaitAll } from "@/prelude/await-all.js"; +import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js"; type Field = { name: string; @@ -15,44 +16,52 @@ type Field = { }; export class UserConverter { - public static async encode(u: User): Promise { - let acct = u.username; - let acctUrl = `https://${u.host || config.host}/@${u.username}`; - if (u.host) { - acct = `${u.username}@${u.host}`; - acctUrl = `https://${u.host}/@${u.username}`; - } - const profile = UserProfiles.findOneBy({userId: u.id}); - const bio = profile.then(profile => toHtml(mfm.parse(profile?.description ?? "")) ?? escapeMFM(profile?.description ?? "")); - const avatar = u.avatarId - ? (DriveFiles.findOneBy({ id: u.avatarId })) - .then(p => p?.url ?? Users.getIdenticonUrl(u.id)) - : Users.getIdenticonUrl(u.id); - const banner = u.bannerId - ? (DriveFiles.findOneBy({ id: u.bannerId })) - .then(p => p?.url ?? `${config.url}/static-assets/transparent.png`) - : `${config.url}/static-assets/transparent.png`; + public static async encode(u: User, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise { + return cache.locks.acquire(u.id, async () => { + const cacheHit = cache.accounts.find(p => p.id == u.id); + if (cacheHit) return cacheHit; - return awaitAll({ - id: u.id, - username: u.username, - acct: acct, - display_name: u.name || u.username, - locked: u.isLocked, - created_at: new Date().toISOString(), - followers_count: u.followersCount, - following_count: u.followingCount, - statuses_count: u.notesCount, - note: bio, - url: u.uri ?? acctUrl, - avatar: avatar, - avatar_static: avatar, - header: banner, - header_static: banner, - emojis: populateEmojis(u.emojis, u.host).then(emoji => emoji.map((e) => EmojiConverter.encode(e))), - moved: null, //FIXME - fields: profile.then(profile => profile?.fields.map(p => this.encodeField(p)) ?? []), - bot: u.isBot + let acct = u.username; + let acctUrl = `https://${u.host || config.host}/@${u.username}`; + if (u.host) { + acct = `${u.username}@${u.host}`; + acctUrl = `https://${u.host}/@${u.username}`; + } + const profile = UserProfiles.findOneBy({userId: u.id}); + const bio = profile.then(profile => toHtml(mfm.parse(profile?.description ?? "")) ?? escapeMFM(profile?.description ?? "")); + const avatar = u.avatarId + ? (DriveFiles.findOneBy({ id: u.avatarId })) + .then(p => p?.url ?? Users.getIdenticonUrl(u.id)) + : Users.getIdenticonUrl(u.id); + const banner = u.bannerId + ? (DriveFiles.findOneBy({ id: u.bannerId })) + .then(p => p?.url ?? `${config.url}/static-assets/transparent.png`) + : `${config.url}/static-assets/transparent.png`; + + return awaitAll({ + id: u.id, + username: u.username, + acct: acct, + display_name: u.name || u.username, + locked: u.isLocked, + created_at: new Date().toISOString(), + followers_count: u.followersCount, + following_count: u.followingCount, + statuses_count: u.notesCount, + note: bio, + url: u.uri ?? acctUrl, + avatar: avatar, + avatar_static: avatar, + header: banner, + header_static: banner, + emojis: populateEmojis(u.emojis, u.host).then(emoji => emoji.map((e) => EmojiConverter.encode(e))), + moved: null, //FIXME + fields: profile.then(profile => profile?.fields.map(p => this.encodeField(p)) ?? []), + bot: u.isBot + }).then(p => { + cache.accounts.push(p); + return p; + }); }); } diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts index cc4a354aa..8d0fb1fc4 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/account.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts @@ -161,10 +161,11 @@ export function apiAccountMastodon(router: Router): void { } const userId = convertId(ctx.params.id, IdType.IceshrimpId); - const query = await getUser(userId); + const cache = UserHelpers.getFreshAccountCache(); + const query = await UserHelpers.getUserCached(userId, cache); const args = normalizeUrlQuery(convertTimelinesArgsId(argsToBools(limitToInt(ctx.query)))); const tl = await UserHelpers.getUserStatuses(query, user, args.max_id, args.since_id, args.min_id, args.limit, args.only_media, args.exclude_replies, args.exclude_reblogs, args.pinned, args.tagged) - .then(n => NoteConverter.encodeMany(n, user)); + .then(n => NoteConverter.encodeMany(n, user, cache)); ctx.body = tl.map(s => convertStatus(s)); } catch (e: any) { diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index 30f48c5ec..78b4a5d5f 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -11,6 +11,7 @@ import { getNote } from "@/server/api/common/getters.js"; import authenticate from "@/server/api/authenticate.js"; import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js"; import { Note } from "@/models/entities/note.js"; +import { UserHelpers } from "@/server/api/mastodon/helpers/user.js"; function normalizeQuery(data: any) { const str = querystring.stringify(data); @@ -197,6 +198,7 @@ export function apiStatusMastodon(router: Router): void { const user = auth[0] ?? null; const id = convertId(ctx.params.id, IdType.IceshrimpId); + const cache = UserHelpers.getFreshAccountCache(); const note = await getNote(id, user ?? null).then(n => n).catch(() => null); if (!note) { if (!note) { @@ -206,10 +208,10 @@ export function apiStatusMastodon(router: Router): void { } const ancestors = await NoteHelpers.getNoteAncestors(note, user, user ? 4096 : 60) - .then(n => NoteConverter.encodeMany(n, user)) + .then(n => NoteConverter.encodeMany(n, user, cache)) .then(n => n.map(s => convertStatus(s))); const descendants = await NoteHelpers.getNoteDescendants(note, user, user ? 4096 : 40, user ? 4096 : 20) - .then(n => NoteConverter.encodeMany(n, user)) + .then(n => NoteConverter.encodeMany(n, user, cache)) .then(n => n.map(s => convertStatus(s))); ctx.body = { diff --git a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts index 4ec22ca6f..bbdae6a63 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts @@ -12,6 +12,7 @@ import authenticate from "@/server/api/authenticate.js"; import { TimelineHelpers } from "@/server/api/mastodon/helpers/timeline.js"; import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js"; import { NoteConverter } from "@/server/api/mastodon/converters/note.js"; +import { UserHelpers } from "@/server/api/mastodon/helpers/user.js"; export function limitToInt(q: ParsedUrlQuery) { let object: any = q; @@ -82,8 +83,9 @@ export function apiTimelineMastodon(router: Router): void { } const args = normalizeUrlQuery(convertTimelinesArgsId(argsToBools(limitToInt(ctx.query)))); + const cache = UserHelpers.getFreshAccountCache(); const tl = await TimelineHelpers.getPublicTimeline(user, args.max_id, args.since_id, args.min_id, args.limit, args.only_media, args.local, args.remote) - .then(n => NoteConverter.encodeMany(n, user)); + .then(n => NoteConverter.encodeMany(n, user, cache)); ctx.body = tl.map(s => convertStatus(s)); } catch (e: any) { @@ -124,8 +126,9 @@ export function apiTimelineMastodon(router: Router): void { } const args = normalizeUrlQuery(convertTimelinesArgsId(limitToInt(ctx.query))); + const cache = UserHelpers.getFreshAccountCache(); const tl = await TimelineHelpers.getHomeTimeline(user, args.max_id, args.since_id, args.min_id, args.limit) - .then(n => NoteConverter.encodeMany(n, user)); + .then(n => NoteConverter.encodeMany(n, user, cache)); ctx.body = tl.map(s => convertStatus(s)); } catch (e: any) { diff --git a/packages/backend/src/server/api/mastodon/helpers/user.ts b/packages/backend/src/server/api/mastodon/helpers/user.ts index b02d74e19..ee2e41b83 100644 --- a/packages/backend/src/server/api/mastodon/helpers/user.ts +++ b/packages/backend/src/server/api/mastodon/helpers/user.ts @@ -1,20 +1,21 @@ import { Note } from "@/models/entities/note.js"; -import { User } from "@/models/entities/user.js"; -import { ILocalUser } from "@/models/entities/user.js"; -import { Followings, Notes } from "@/models/index.js"; +import { ILocalUser, User } from "@/models/entities/user.js"; +import { Notes } from "@/models/index.js"; import { makePaginationQuery } from "@/server/api/common/make-pagination-query.js"; -import { Brackets, SelectQueryBuilder } from "typeorm"; -import { generateChannelQuery } from "@/server/api/common/generate-channel-query.js"; import { generateRepliesQuery } from "@/server/api/common/generate-replies-query.js"; import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js"; import { generateMutedUserQuery } from "@/server/api/common/generate-muted-user-query.js"; -import { generateMutedNoteQuery } from "@/server/api/common/generate-muted-note-query.js"; import { generateBlockedUserQuery } from "@/server/api/common/generate-block-query.js"; -import { generateMutedUserRenotesQueryForNotes } from "@/server/api/common/generated-muted-renote-query.js"; -import { fetchMeta } from "@/misc/fetch-meta.js"; -import { ApiError } from "@/server/api/error.js"; -import { meta } from "@/server/api/endpoints/notes/global-timeline.js"; import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js"; +import Entity from "megalodon/src/entity.js"; +import AsyncLock from "async-lock"; +import { getUser } from "@/server/api/common/getters.js"; + +export type AccountCache = { + locks: AsyncLock; + accounts: Entity.Account[]; + users: User[]; +}; export class UserHelpers { public static async getUserStatuses(user: User, localUser: ILocalUser | null, maxId: string | undefined, sinceId: string | undefined, minId: string | undefined, limit: number = 20, onlyMedia: boolean = false, excludeReplies: boolean = false, excludeReblogs: boolean = false, pinned: boolean = false, tagged: string | undefined): Promise { @@ -67,4 +68,23 @@ export class UserHelpers { return NoteHelpers.execQuery(query, limit); } + + public static async getUserCached(id: string, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise { + return cache.locks.acquire(id, async () => { + const cacheHit = cache.users.find(p => p.id == id); + if (cacheHit) return cacheHit; + return getUser(id).then(p => { + cache.users.push(p); + return p; + }); + }); + } + + public static getFreshAccountCache(): AccountCache { + return { + locks: new AsyncLock(), + accounts: [], + users: [], + }; + } } diff --git a/yarn.lock b/yarn.lock index b6d04fcfa..9d5ebe68c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5757,6 +5757,7 @@ __metadata: "@tensorflow/tfjs-core": ^4.2.0 "@tensorflow/tfjs-node": 3.21.1 "@types/adm-zip": ^0.5.0 + "@types/async-lock": 1.4.0 "@types/bcryptjs": 2.4.2 "@types/cbor": 6.0.0 "@types/escape-regexp": 0.0.1 @@ -5803,6 +5804,7 @@ __metadata: ajv: 8.12.0 archiver: 5.3.1 argon2: ^0.30.3 + async-lock: 1.4.0 autolinker: 4.0.0 autwh: 0.1.0 aws-sdk: 2.1413.0