diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index eecbf93e6..abd248e46 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -97,36 +97,48 @@ export default hasConfig ? { filter: constructedFilters }); }, - ingestNote: (note : Note) => { - logger.info("Indexing note in MeiliSearch: " + note.id); - - let attachmentType = ""; - if (note.attachedFileTypes.length > 0) { - attachmentType = note.attachedFileTypes[0].split("/")[0]; - switch (attachmentType) { - case "image": - case "video": - case "audio": - case "text": - break; - default: - attachmentType = "file" - break - } + ingestNote: (note: Note | Note[]) => { + if (note instanceof Note) { + note = [note]; } - return posts.addDocuments([ - { - id: note.id.toString(), - text: note.text, - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - mediaAttachment: attachmentType, - userName: note.user?.username, - createdAt: note.createdAt.getTime() / 1000 // division by 1000 is necessary because Node returns in ms-accuracy + let indexingBatch: MeilisearchNote[] = []; + + note.forEach(note => { + + let attachmentType = ""; + if (note.attachedFileTypes.length > 0) { + attachmentType = note.attachedFileTypes[0].split("/")[0]; + switch (attachmentType) { + case "image": + case "video": + case "audio": + case "text": + break; + default: + attachmentType = "file" + break + } } - ]); + + indexingBatch.push({ + id: note.id.toString(), + text: note.text ? note.text : "", + userId: note.userId, + userHost: note.userHost ? note.userHost : "", + channelId: note.channelId ? note.channelId : "", + mediaAttachment: attachmentType, + userName: note.user?.username ? note.user.username : "", + createdAt: note.createdAt.getTime() / 1000 // division by 1000 is necessary because Node returns in ms-accuracy + } + ) + }); + + let indexingIDs = indexingBatch.map(note => note.id); + + logger.info("Indexing notes in MeiliSearch: " + indexingIDs.join(",")); + + return posts.addDocuments(indexingBatch); }, serverStats: async () => { let health : Health = await client.health(); diff --git a/packages/backend/src/queue/processors/background/index-all-notes.ts b/packages/backend/src/queue/processors/background/index-all-notes.ts index 03219199d..9bed4eb73 100644 --- a/packages/backend/src/queue/processors/background/index-all-notes.ts +++ b/packages/backend/src/queue/processors/background/index-all-notes.ts @@ -4,7 +4,8 @@ import { queueLogger } from "../../logger.js"; import { Notes } from "@/models/index.js"; import { MoreThan } from "typeorm"; import { index } from "@/services/note/create.js"; -import { Note } from "@/models/entities/note.js"; +import {Note} from "@/models/entities/note.js"; +import meilisearch from "../../../db/meilisearch.js"; const logger = queueLogger.createSubLogger("index-all-notes"); @@ -58,11 +59,16 @@ export default async function indexAllNotes( for (let i = 0; i < notes.length; i += batch) { const chunk = notes.slice(i, i + batch); - await Promise.all(chunk.map((note) => index(note))); + + if (meilisearch) { + await meilisearch.ingestNote(chunk) + } + + await Promise.all(chunk.map((note) => index(note, true))); indexedCount += chunk.length; const pct = (indexedCount / total) * 100; - job.update({ indexedCount, cursor, total }); + job.update({indexedCount, cursor, total}); job.progress(+pct.toFixed(1)); logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`); } diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index f6285a61d..bd54db7e2 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -749,7 +749,7 @@ async function insertNote( } } -export async function index(note: Note): Promise { +export async function index(note: Note, reindexing: boolean): Promise { if (!note.text) return; if (config.elasticsearch && es) { @@ -778,7 +778,7 @@ export async function index(note: Note): Promise { ); } - if (meilisearch) { + if (meilisearch && !reindexing) { await meilisearch.ingestNote(note); } }