From 1e4a45ff7c6c1db57ca9ce03bef60db541b35722 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 15 Jun 2023 20:59:17 -0400 Subject: [PATCH 1/5] fix aode-relay compatibility --- .../src/server/api/mastodon/endpoints/meta.ts | 2 +- packages/backend/src/services/note/create.ts | 33 ++++++++++++++++--- packages/backend/src/services/relay.ts | 26 +++++++++++---- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/endpoints/meta.ts b/packages/backend/src/server/api/mastodon/endpoints/meta.ts index d362d1b9e..c68a09585 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/meta.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/meta.ts @@ -12,7 +12,7 @@ export async function getInstance(response: Entity.Instance) { uri: response.uri, title: response.title || "Calckey", short_description: - response.description.substring(0, 50) || "See real server website", + response.description?.substring(0, 50) || "See real server website", description: response.description || "This is a vanilla Calckey Instance. It doesnt seem to have a description. BTW you are using the Mastodon api to access this server :)", diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index ab3f36570..35548a174 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -36,6 +36,7 @@ import { ChannelFollowings, Blockings, NoteThreadMutings, + Relays, } from "@/models/index.js"; import type { DriveFile } from "@/models/entities/drive-file.js"; import type { App } from "@/models/entities/app.js"; @@ -56,7 +57,7 @@ import { checkHitAntenna } from "@/misc/check-hit-antenna.js"; import { getWordHardMute } from "@/misc/check-word-mute.js"; import { addNoteToAntenna } from "../add-note-to-antenna.js"; import { countSameRenotes } from "@/misc/count-same-renotes.js"; -import { deliverToRelays } from "../relay.js"; +import { deliverToRelays, getCachedRelays } from "../relay.js"; import type { Channel } from "@/models/entities/channel.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js"; import { getAntennas } from "@/misc/antenna-cache.js"; @@ -72,6 +73,7 @@ import meilisearch from "../../db/meilisearch.js"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] >(1000 * 60 * 5); +const publishedNoteCache = new Cache(1000 * 10); type NotificationType = "reply" | "renote" | "quote" | "mention"; @@ -165,6 +167,7 @@ export default async ( isSilenced: User["isSilenced"]; createdAt: User["createdAt"]; isBot: User["isBot"]; + inbox?: User["inbox"]; }, data: Option, silent = false, @@ -453,7 +456,29 @@ export default async ( } if (!dontFederateInitially) { - publishNotesStream(note); + const relays = await getCachedRelays(); + if (!note.uri) { + publishNotesStream(note); + } else if ( + data.renote?.uri && + publishedNoteCache.get(data.renote.uri) !== true && + user.inbox && + relays.map((relay) => relay.inbox).includes(user.inbox) + ) { + const uri = data.renote.uri; + publishNotesStream(data.renote); + publishedNoteCache.set(uri, true); + setTimeout(() => { + publishedNoteCache.delete(uri); + }, 1000 * 10); + } else if (note.uri && publishedNoteCache.get(note.uri) !== true) { + const uri = note.uri; + publishNotesStream(note); + publishedNoteCache.set(uri, true); + setTimeout(() => { + publishedNoteCache.delete(uri); + }, 1000 * 10); + } } if (note.replyId != null) { // Only provide the reply note id here as the recipient may not be authorized to see the note. @@ -524,7 +549,6 @@ export default async ( nm.push(data.renote.userId, type); } } - // Fetch watchers nmRelatedPromises.push( notifyToWatchersOfRenotee(data.renote, user, nm, type), @@ -537,8 +561,9 @@ export default async ( }); publishMainStream(data.renote.userId, "renote", packedRenote); + const renote = data.renote; const webhooks = (await getActiveWebhooks()).filter( - (x) => x.userId === data.renote!.userId && x.on.includes("renote"), + (x) => x.userId === renote.userId && x.on.includes("renote"), ); for (const webhook of webhooks) { webhookDeliver(webhook, "renote", { diff --git a/packages/backend/src/services/relay.ts b/packages/backend/src/services/relay.ts index 244e05c03..bec4b1f86 100644 --- a/packages/backend/src/services/relay.ts +++ b/packages/backend/src/services/relay.ts @@ -37,7 +37,7 @@ export async function addRelay(inbox: string) { }).then((x) => Relays.findOneByOrFail(x.identifiers[0])); const relayActor = await getRelayActor(); - const follow = await renderFollowRelay(relay, relayActor); + const follow = renderFollowRelay(relay, relayActor); const activity = renderActivity(follow); deliver(relayActor, activity, relay.inbox); @@ -60,6 +60,7 @@ export async function removeRelay(inbox: string) { deliver(relayActor, activity, relay.inbox); await Relays.delete(relay.id); + await updateRelaysCache(); } export async function listRelay() { @@ -67,14 +68,31 @@ export async function listRelay() { return relays; } +export async function getCachedRelays(): Promise { + return await relaysCache.fetch(null, () => + Relays.findBy({ + status: "accepted", + }), + ); +} + export async function relayAccepted(id: string) { const result = await Relays.update(id, { status: "accepted", }); + await updateRelaysCache(); + return JSON.stringify(result); } +async function updateRelaysCache() { + const relays = await Relays.findBy({ + status: "accepted", + }); + relaysCache.set(null, relays); +} + export async function relayRejected(id: string) { const result = await Relays.update(id, { status: "rejected", @@ -89,11 +107,7 @@ export async function deliverToRelays( ) { if (activity == null) return; - const relays = await relaysCache.fetch(null, () => - Relays.findBy({ - status: "accepted", - }), - ); + const relays = await getCachedRelays(); if (relays.length === 0) return; // TODO From f43e9e4121cb7b9ed0dad52f75f2cbf46443c9cb Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 15 Jun 2023 21:56:24 -0400 Subject: [PATCH 2/5] use redis --- packages/backend/src/services/note/create.ts | 31 ++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 35548a174..9a7d057ee 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -69,11 +69,11 @@ import { db } from "@/db/postgre.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; import meilisearch from "../../db/meilisearch.js"; +import { redisClient } from "@/db/redis.js"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] >(1000 * 60 * 5); -const publishedNoteCache = new Cache(1000 * 10); type NotificationType = "reply" | "renote" | "quote" | "mention"; @@ -457,27 +457,28 @@ export default async ( if (!dontFederateInitially) { const relays = await getCachedRelays(); + const boostedByRelay = + !!user.inbox && + relays.map((relay) => relay.inbox).includes(user.inbox); + if (!note.uri) { publishNotesStream(note); } else if ( + boostedByRelay && data.renote?.uri && - publishedNoteCache.get(data.renote.uri) !== true && - user.inbox && - relays.map((relay) => relay.inbox).includes(user.inbox) + (await redisClient.exists(`publishedNote:${data.renote.uri}`)) === 0 ) { - const uri = data.renote.uri; publishNotesStream(data.renote); - publishedNoteCache.set(uri, true); - setTimeout(() => { - publishedNoteCache.delete(uri); - }, 1000 * 10); - } else if (note.uri && publishedNoteCache.get(note.uri) !== true) { - const uri = note.uri; + const key = `publishedNote:${data.renote.uri}`; + await redisClient.set(key, 1, "EX", 10); + } else if ( + !boostedByRelay && + note.uri && + (await redisClient.exists(`publishedNote:${note.uri}`)) === 0 + ) { + const key = `publishedNote:${note.uri}`; publishNotesStream(note); - publishedNoteCache.set(uri, true); - setTimeout(() => { - publishedNoteCache.delete(uri); - }, 1000 * 10); + await redisClient.set(key, 1, "EX", 10); } } if (note.replyId != null) { From 7a35aaa51d6f2c0289e6e7f1e48f3578bf53b29f Mon Sep 17 00:00:00 2001 From: Namekuji Date: Fri, 16 Jun 2023 02:48:57 -0400 Subject: [PATCH 3/5] wait a bit more --- packages/backend/src/services/note/create.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 9a7d057ee..3423ef01d 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -470,7 +470,7 @@ export default async ( ) { publishNotesStream(data.renote); const key = `publishedNote:${data.renote.uri}`; - await redisClient.set(key, 1, "EX", 10); + await redisClient.set(key, 1, "EX", 30); } else if ( !boostedByRelay && note.uri && @@ -478,7 +478,7 @@ export default async ( ) { const key = `publishedNote:${note.uri}`; publishNotesStream(note); - await redisClient.set(key, 1, "EX", 10); + await redisClient.set(key, 1, "EX", 30); } } if (note.replyId != null) { From eb7b1d67881826b35919527df59a55e31fca57d2 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Fri, 16 Jun 2023 03:45:53 -0400 Subject: [PATCH 4/5] add comments --- packages/backend/src/services/note/create.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 3423ef01d..b493ae5e0 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -457,17 +457,21 @@ export default async ( if (!dontFederateInitially) { const relays = await getCachedRelays(); + // Some relays (e.g., aode-relay) deliver posts by boosting them as + // Announce activities. In that case, user is the relay's actor. const boostedByRelay = !!user.inbox && relays.map((relay) => relay.inbox).includes(user.inbox); if (!note.uri) { + // Publish if the post is local publishNotesStream(note); } else if ( boostedByRelay && data.renote?.uri && (await redisClient.exists(`publishedNote:${data.renote.uri}`)) === 0 ) { + // Publish if the post was boosted by a relay and not yet published. publishNotesStream(data.renote); const key = `publishedNote:${data.renote.uri}`; await redisClient.set(key, 1, "EX", 30); @@ -476,6 +480,9 @@ export default async ( note.uri && (await redisClient.exists(`publishedNote:${note.uri}`)) === 0 ) { + // Publish if the post came directly from a remote server, or from a + // relay that doesn't boost the post (e.g, YUKIMOCHI Activity-Relay), + // and not yet published. const key = `publishedNote:${note.uri}`; publishNotesStream(note); await redisClient.set(key, 1, "EX", 30); From 6bea9f8bfc3b4c39734e280367bff681f94e304d Mon Sep 17 00:00:00 2001 From: Namekuji Date: Fri, 16 Jun 2023 04:17:32 -0400 Subject: [PATCH 5/5] remove unused import --- packages/backend/src/services/note/create.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index b493ae5e0..9696c3cca 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -36,7 +36,6 @@ import { ChannelFollowings, Blockings, NoteThreadMutings, - Relays, } from "@/models/index.js"; import type { DriveFile } from "@/models/entities/drive-file.js"; import type { App } from "@/models/entities/app.js";