From 99b4e5e13c83be4a03bfc819809fd5adc3c246b3 Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 25 May 2023 00:55:33 +0200 Subject: [PATCH 01/34] Implement Meilisearch Indexing --- packages/backend/package.json | 1 + packages/backend/src/config/types.ts | 6 ++ packages/backend/src/db/meilisearch.ts | 61 ++++++++++++++++ .../src/server/api/endpoints/notes/search.ts | 70 ++++++++++++++++++- packages/backend/src/services/note/create.ts | 5 ++ pnpm-lock.yaml | 39 +++++++++-- 6 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 packages/backend/src/db/meilisearch.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 96edb7f02..968b0af80 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -85,6 +85,7 @@ "koa-send": "5.0.1", "koa-slow": "2.1.0", "koa-views": "7.0.2", + "meilisearch": "^0.32.4", "mfm-js": "0.23.3", "mime-types": "2.1.35", "multer": "1.4.4-lts.1", diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 01a98f9f0..b6a449f54 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -40,6 +40,12 @@ export type Source = { bucket?: string; }; + meilisearch: { + host: string; + port: number; + apiKey?: string; + }; + proxy?: string; proxySmtp?: string; proxyBypassHosts?: string[]; diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts new file mode 100644 index 000000000..756c092e9 --- /dev/null +++ b/packages/backend/src/db/meilisearch.ts @@ -0,0 +1,61 @@ +import { MeiliSearch } from 'meilisearch'; +import { dbLogger } from "./logger.js"; + +import config from "@/config/index.js"; +import {Note} from "@/models/entities/note"; +import {normalizeForSearch} from "@/misc/normalize-for-search"; + +const logger = dbLogger.createSubLogger("meilisearch", "gray", false); + +logger.info("Connecting to MeiliSearch"); + +const hasConfig = + config.meilisearch && (config.meilisearch.host || config.meilisearch.port || config.meilisearch.apiKey); + +const host = hasConfig ? config.meilisearch.host ?? "localhost" : ""; +const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; +const auth = hasConfig ? config.meilisearch.apiKey ?? "" : ""; + +const client = new MeiliSearch({ + host: 'http://127.0.0.1:7700', + apiKey: 'masterKey', +}) + +const posts = client.index('posts'); + +posts.updateSearchableAttributes(['text']); + +logger.info("Connected to MeiliSearch"); + + +export type MeilisearchNote = { + id: string; + text: string; + userId: string; + userHost: string; + channelId: string; +} + +export default hasConfig ? { + search: (query : string, limit : number, offset : number) => { + logger.info(`Searching for ${query}`); + + return posts.search(query, { + limit: limit, + offset: offset, + }); + }, + ingestNote: (note : Note) => { + logger.info("Indexing note in MeiliSearch: " + note.id); + + return posts.addDocuments([ + { + id: note.id.toString(), + text: note.text, + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + } + ]) + }, +} : null; diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 93392acdd..0a6737c9b 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -4,6 +4,7 @@ import { Note } from "@/models/entities/note.js"; import config from "@/config/index.js"; import es from "../../../../db/elasticsearch.js"; import sonic from "../../../../db/sonic.js"; +import meilisearch, {MeilisearchNote} from "../../../../db/meilisearch.js"; import define from "../../define.js"; import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { generateVisibilityQuery } from "../../common/generate-visibility-query.js"; @@ -62,7 +63,7 @@ export const paramDef = { } as const; export default define(meta, paramDef, async (ps, me) => { - if (es == null && sonic == null) { + if (es == null && sonic == null && meilisearch == null) { const query = makePaginationQuery( Notes.createQueryBuilder("note"), ps.sinceId, @@ -171,6 +172,73 @@ export default define(meta, paramDef, async (ps, me) => { } return found; + } else if(meilisearch) { + let start = 0; + const chunkSize = 100; + + // Use meilisearch to fetch and step through all search results that could match the requirements + const ids = []; + while (true) { + const results = await meilisearch.search(ps.query, start, chunkSize); + + start += chunkSize; + + if (results.hits.length === 0) { + break; + } + + const res = results.hits + .filter((key) => { + let note = key as MeilisearchNote; + + if (ps.userId && note.userId !== ps.userId) { + return false; + } + if (ps.channelId && note.channelId !== ps.channelId) { + return false; + } + if (ps.sinceId && note.id <= ps.sinceId) { + return false; + } + if (ps.untilId && note.id >= ps.untilId) { + return false; + } + return true; + }) + .map((key) => key.id); + + ids.push(...res); + } + + // Sort all the results by note id DESC (newest first) + ids.sort((a, b) => b - a); + + // Fetch the notes from the database until we have enough to satisfy the limit + start = 0; + const found = []; + while (found.length < ps.limit && start < ids.length) { + const chunk = ids.slice(start, start + chunkSize); + const notes: Note[] = await Notes.find({ + where: { + id: In(chunk), + }, + order: { + id: "DESC", + }, + }); + + // The notes are checked for visibility and muted/blocked users when packed + found.push(...(await Notes.packMany(notes, me))); + start += chunkSize; + } + + // If we have more results than the limit, trim them + if (found.length > ps.limit) { + found.length = ps.limit; + } + + return found; + } else { const userQuery = ps.userId != null diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 66c5b8508..158460421 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -67,6 +67,7 @@ import type { UserProfile } from "@/models/entities/user-profile.js"; import { db } from "@/db/postgre.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; +import meilisearch from "@/db/meilisearch"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] @@ -776,6 +777,10 @@ export async function index(note: Note): Promise { note.text, ); } + + if (meilisearch) { + await meilisearch.ingestNote(note); + } } async function notifyToWatchersOfRenotee( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 493e9fc06..48a0e498b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -263,6 +263,9 @@ importers: koa-views: specifier: 7.0.2 version: 7.0.2(@types/koa@2.13.5)(ejs@3.1.8)(pug@3.0.2) + meilisearch: + specifier: ^0.32.4 + version: 0.32.4 mfm-js: specifier: 0.23.3 version: 0.23.3 @@ -2819,7 +2822,7 @@ packages: '@types/webgl-ext': 0.0.30 '@webgpu/types': 0.1.16 long: 4.0.0 - node-fetch: 2.6.8 + node-fetch: 2.6.11 seedrandom: 3.0.5 transitivePeerDependencies: - encoding @@ -2835,7 +2838,7 @@ packages: '@types/webgl-ext': 0.0.30 '@webgpu/types': 0.1.21 long: 4.0.0 - node-fetch: 2.6.8 + node-fetch: 2.6.11 seedrandom: 3.0.5 transitivePeerDependencies: - encoding @@ -2849,7 +2852,7 @@ packages: dependencies: '@tensorflow/tfjs-core': 3.21.0 '@types/node-fetch': 2.6.2 - node-fetch: 2.6.8 + node-fetch: 2.6.11 seedrandom: 3.0.5 string_decoder: 1.3.0 transitivePeerDependencies: @@ -2864,7 +2867,7 @@ packages: dependencies: '@tensorflow/tfjs-core': 4.2.0 '@types/node-fetch': 2.6.2 - node-fetch: 2.6.8 + node-fetch: 2.6.11 seedrandom: 3.0.5 string_decoder: 1.3.0 transitivePeerDependencies: @@ -5938,6 +5941,14 @@ packages: - encoding dev: true + /cross-fetch@3.1.6: + resolution: {integrity: sha512-riRvo06crlE8HiqOwIpQhxwdOk4fOeR7FVM/wXoxchFEqMNUjvbs3bfo4OTgMEMHzppd4DxFBDbyySj8Cv781g==} + dependencies: + node-fetch: 2.6.11 + transitivePeerDependencies: + - encoding + dev: false + /cross-spawn@5.1.0: resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==} dependencies: @@ -10386,6 +10397,14 @@ packages: engines: {node: '>= 0.6'} dev: false + /meilisearch@0.32.4: + resolution: {integrity: sha512-QvPtQ6F2TaqAT9fw072/MDjSCMpQifdtUBFeIk3M5jSnFpeSiv1iwfJWNfP6ByaCgR/s++K1Cqtf9vjcZe7prg==} + dependencies: + cross-fetch: 3.1.6 + transitivePeerDependencies: + - encoding + dev: false + /meow@9.0.0: resolution: {integrity: sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==} engines: {node: '>=10'} @@ -10854,6 +10873,18 @@ packages: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} + /node-fetch@2.6.11: + resolution: {integrity: sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + dev: false + /node-fetch@2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} engines: {node: 4.x || >=6.0.0} From 89f1b6357ebfc211b5becefc864fd78f4e361e82 Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 25 May 2023 01:06:03 +0200 Subject: [PATCH 02/34] Meilisearch Config --- .config/example.yml | 7 +++++++ docker-compose.yml | 16 ++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 7d8ba32be..7f83446a4 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -82,6 +82,13 @@ redis: # user: # pass: +# ┌───────────────────────────┐ +#───┘ Meilisearch configuration └───────────────────────────────────── +#meilisearch: +# host: meilisearch +# port: 7700 +# pass: + # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/docker-compose.yml b/docker-compose.yml index 5de14d0c8..e1f9c7224 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,14 +40,18 @@ services: volumes: - ./db:/var/lib/postgresql/data - sonic: - restart: unless-stopped - image: docker.io/valeriansaliou/sonic:v1.4.0 + meilisearch: + container_name: meilisearch + image: getmeili/meilisearch:v0.25.2 + environment: + - MEILI_ENV=${MEILI_ENV:-development} + ports: + - "7700:7700" networks: - - calcnet + - meilisearch volumes: - - ./sonic:/var/lib/sonic/store - - ./sonic/config.cfg:/etc/sonic.cfg + - ./data.ms:/data.ms + restart: unless-stopped networks: calcnet: From 090b5724b478476b590825d4a3624463bd89907f Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 25 May 2023 02:19:42 +0200 Subject: [PATCH 03/34] Fix wrong parameter ordering --- packages/backend/src/db/meilisearch.ts | 6 ++++-- .../src/server/api/endpoints/notes/search.ts | 14 ++++++-------- packages/backend/src/services/note/create.ts | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index 756c092e9..f404b4721 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -17,8 +17,8 @@ const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; const auth = hasConfig ? config.meilisearch.apiKey ?? "" : ""; const client = new MeiliSearch({ - host: 'http://127.0.0.1:7700', - apiKey: 'masterKey', + host: `http://${host}:${port}`, + apiKey: auth, }) const posts = client.index('posts'); @@ -39,6 +39,8 @@ export type MeilisearchNote = { export default hasConfig ? { search: (query : string, limit : number, offset : number) => { logger.info(`Searching for ${query}`); + logger.info(`Limit: ${limit}`); + logger.info(`Offset: ${offset}`); return posts.search(query, { limit: limit, diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 0a6737c9b..425414561 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -179,7 +179,7 @@ export default define(meta, paramDef, async (ps, me) => { // Use meilisearch to fetch and step through all search results that could match the requirements const ids = []; while (true) { - const results = await meilisearch.search(ps.query, start, chunkSize); + const results = await meilisearch.search(ps.query, chunkSize, start); start += chunkSize; @@ -188,19 +188,17 @@ export default define(meta, paramDef, async (ps, me) => { } const res = results.hits - .filter((key) => { - let note = key as MeilisearchNote; - - if (ps.userId && note.userId !== ps.userId) { + .filter((key: MeilisearchNote) => { + if (ps.userId && key.userId !== ps.userId) { return false; } - if (ps.channelId && note.channelId !== ps.channelId) { + if (ps.channelId && key.channelId !== ps.channelId) { return false; } - if (ps.sinceId && note.id <= ps.sinceId) { + if (ps.sinceId && key.id <= ps.sinceId) { return false; } - if (ps.untilId && note.id >= ps.untilId) { + if (ps.untilId && key.id >= ps.untilId) { return false; } return true; diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 158460421..f6285a61d 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -67,7 +67,7 @@ import type { UserProfile } from "@/models/entities/user-profile.js"; import { db } from "@/db/postgre.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; -import meilisearch from "@/db/meilisearch"; +import meilisearch from "../../db/meilisearch.js"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] From 55ce94b951ca5697b9c94c6e90d5d581188f68f8 Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 25 May 2023 09:53:04 +0200 Subject: [PATCH 04/34] Add Meilisearch widget --- packages/backend/src/daemons/server-stats.ts | 11 ++++ packages/backend/src/db/meilisearch.ts | 16 ++++-- .../src/server/api/endpoints/server-info.ts | 12 +++++ .../client/src/widgets/server-metric/disk.vue | 18 +++++++ .../src/widgets/server-metric/index.vue | 8 ++- .../src/widgets/server-metric/meilisearch.vue | 51 +++++++++++++++++++ 6 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 packages/client/src/widgets/server-metric/meilisearch.vue diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index b0bf1288f..c7aaea035 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -1,6 +1,7 @@ import si from "systeminformation"; import Xev from "xev"; import * as osUtils from "os-utils"; +import meilisearch from "../db/meilisearch"; const ev = new Xev(); @@ -24,6 +25,7 @@ export default function () { const memStats = await mem(); const netStats = await net(); const fsStats = await fs(); + const meilisearchStats = await meilisearchStatus(); const stats = { cpu: roundCpu(cpu), @@ -39,6 +41,7 @@ export default function () { r: round(Math.max(0, fsStats.rIO_sec ?? 0)), w: round(Math.max(0, fsStats.wIO_sec ?? 0)), }, + meilisearch: meilisearchStats }; ev.emit("serverStats", stats); log.unshift(stats); @@ -77,3 +80,11 @@ async function fs() { const data = await si.disksIO().catch(() => ({ rIO_sec: 0, wIO_sec: 0 })); return data || { rIO_sec: 0, wIO_sec: 0 }; } + +async function meilisearchStatus() { + if (meilisearch) { + return meilisearch.serverStats(); + } else { + return null; + } +} diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index f404b4721..f3561b807 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -1,4 +1,4 @@ -import { MeiliSearch } from 'meilisearch'; +import {Health, MeiliSearch, Stats } from 'meilisearch'; import { dbLogger } from "./logger.js"; import config from "@/config/index.js"; @@ -16,7 +16,7 @@ const host = hasConfig ? config.meilisearch.host ?? "localhost" : ""; const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; const auth = hasConfig ? config.meilisearch.apiKey ?? "" : ""; -const client = new MeiliSearch({ +const client : MeiliSearch = new MeiliSearch({ host: `http://${host}:${port}`, apiKey: auth, }) @@ -58,6 +58,16 @@ export default hasConfig ? { userHost: note.userHost, channelId: note.channelId, } - ]) + ]); }, + serverStats: async () => { + let health : Health = await client.health(); + let stats: Stats = await client.getStats(); + + return { + health: health.status, + size: stats.databaseSize, + indexed_count: stats.indexes["posts"].numberOfDocuments + } + } } : null; diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index 1ce27e262..4b2a4078a 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -1,6 +1,7 @@ import * as os from "node:os"; import si from "systeminformation"; import define from "../define.js"; +import meilisearch from "../../../db/meilisearch"; export const meta = { requireCredential: false, @@ -18,6 +19,7 @@ export const paramDef = { export default define(meta, paramDef, async () => { const memStats = await si.mem(); const fsStats = await si.fsSize(); + const meilisearchStats = await meilisearchStatus(); return { machine: os.hostname(), @@ -32,5 +34,15 @@ export default define(meta, paramDef, async () => { total: fsStats[0].size, used: fsStats[0].used, }, + meilisearch: meilisearchStats + }; }); + +async function meilisearchStatus() { + if (meilisearch) { + return meilisearch.serverStats(); + } else { + return null; + } +} diff --git a/packages/client/src/widgets/server-metric/disk.vue b/packages/client/src/widgets/server-metric/disk.vue index 67ea398c1..0457cf7f0 100644 --- a/packages/client/src/widgets/server-metric/disk.vue +++ b/packages/client/src/widgets/server-metric/disk.vue @@ -8,6 +8,12 @@

Used: {{ bytes(used, 1) }}

+
+
+

MeiliSearch

+ +
+ From 2abf027d3351debd961c7db132dd833f7d0375be Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 25 May 2023 10:21:29 +0200 Subject: [PATCH 05/34] Fix property names --- .../client/src/widgets/server-metric/disk.vue | 17 ----------------- .../src/widgets/server-metric/meilisearch.vue | 4 ++-- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/packages/client/src/widgets/server-metric/disk.vue b/packages/client/src/widgets/server-metric/disk.vue index 0457cf7f0..99191e62a 100644 --- a/packages/client/src/widgets/server-metric/disk.vue +++ b/packages/client/src/widgets/server-metric/disk.vue @@ -9,11 +9,6 @@
-
-

MeiliSearch

- -
- + + + + + From 40ad37b870d32f79aedd3605575e41b2c53d116f Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Fri, 26 May 2023 10:55:51 +0200 Subject: [PATCH 29/34] Default meilisearch data response + linting + prettyfier --- packages/backend/src/daemons/server-stats.ts | 6 +++++- .../src/queue/processors/background/index-all-notes.ts | 6 +++--- packages/backend/src/server/api/endpoints/server-info.ts | 7 +++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index 4227ce6ee..2f1dd42ae 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -85,6 +85,10 @@ async function meilisearchStatus() { if (meilisearch) { return meilisearch.serverStats(); } else { - return null; + return { + health: "unconfigured", + size: 0, + indexed_count: 0, + }; } } diff --git a/packages/backend/src/queue/processors/background/index-all-notes.ts b/packages/backend/src/queue/processors/background/index-all-notes.ts index 9bc53d2d3..646984c93 100644 --- a/packages/backend/src/queue/processors/background/index-all-notes.ts +++ b/packages/backend/src/queue/processors/background/index-all-notes.ts @@ -1,8 +1,8 @@ import type Bull from "bull"; -import { queueLogger } from "../../logger.js"; -import { Notes } from "@/models/index.js"; -import { MoreThan } from "typeorm"; +import {queueLogger} from "../../logger.js"; +import {Notes} from "@/models/index.js"; +import {MoreThan} from "typeorm"; import {index} from "@/services/note/create.js"; import {Note} from "@/models/entities/note.js"; import meilisearch from "../../../db/meilisearch.js"; diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index 5ba920301..cc9aa91b2 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -34,7 +34,6 @@ export default define(meta, paramDef, async () => { total: fsStats[0].size, used: fsStats[0].used, }, - meilisearch: meilisearchStats, }; }); @@ -42,6 +41,10 @@ async function meilisearchStatus() { if (meilisearch) { return meilisearch.serverStats(); } else { - return null; + return { + health: "unconfigured", + size: 0, + indexed_count: 0, + }; } } From 0a2c9a6c27c12df88b9c9c3e9a42898d88e03c29 Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Fri, 26 May 2023 10:56:14 +0200 Subject: [PATCH 30/34] add semicolon after property --- packages/backend/src/config/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index e61ef85ae..da0f5571e 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -43,7 +43,7 @@ export type Source = { host: string; port: number; apiKey?: string; - ssl: boolean + ssl: boolean; }; proxy?: string; From 282fdf347a4a831c8305fc4dce9d2c28d76b55ad Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Sun, 28 May 2023 02:15:13 +0200 Subject: [PATCH 31/34] Implement follower and following searches --- packages/backend/src/db/meilisearch.ts | 164 +++++++++++++----- .../src/server/api/endpoints/notes/search.ts | 2 +- 2 files changed, 123 insertions(+), 43 deletions(-) diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index a58425c54..5d294b95d 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -4,8 +4,8 @@ import {dbLogger} from "./logger.js"; import config from "@/config/index.js"; import {Note} from "@/models/entities/note.js"; import * as url from "url"; -import {User} from "@/models/entities/user.js"; -import {Users} from "@/models/index.js"; +import {ILocalUser, User} from "@/models/entities/user.js"; +import {Followings, Users} from "@/models/index.js"; const logger = dbLogger.createSubLogger("meilisearch", "gray", false); @@ -41,6 +41,7 @@ posts "userHost", "mediaAttachment", "createdAt", + "userId", ]) .catch((e) => logger.error( @@ -48,6 +49,14 @@ posts ), ); +posts + .updateSortableAttributes(["createdAt"]) + .catch((e) => + logger.error( + `Setting sortable attr failed, placeholder searches won't sort properly: ${e}`, + ), + ); + logger.info("Connected to MeiliSearch"); export type MeilisearchNote = { @@ -63,60 +72,130 @@ export type MeilisearchNote = { export default hasConfig ? { - search: (query: string, limit: number, offset: number) => { + search: async ( + query: string, + limit: number, + offset: number, + userCtx: ILocalUser | null, + ) => { /// Advanced search syntax /// from:user => filter by user + optional domain /// has:image/video/audio/text/file => filter by attachment types /// domain:domain.com => filter by domain /// before:Date => show posts made before Date /// after: Date => show posts made after Date + /// "text" => get posts with exact text between quotes + /// filter:following => show results only from users you follow + /// filter:followers => show results only from followers let constructedFilters: string[] = []; let splitSearch = query.split(" "); // Detect search operators and remove them from the actual query - splitSearch = splitSearch.filter((term) => { - if (term.startsWith("has:")) { - let fileType = term.slice(4); - constructedFilters.push(`mediaAttachment = "${fileType}"`); - return false; - } else if (term.startsWith("from:")) { - let user = term.slice(5); - constructedFilters.push(`userName = ${user}`); - return false; - } else if (term.startsWith("domain:")) { - let domain = term.slice(7); - constructedFilters.push(`userHost = ${domain}`); - return false; - } else if (term.startsWith("after:")) { - let timestamp = term.slice(6); - // Try to parse the timestamp as JavaScript Date - let date = Date.parse(timestamp); - if (isNaN(date)) return false; - constructedFilters.push(`createdAt > ${date}`); - return false; - } else if (term.startsWith("before:")) { - let timestamp = term.slice(7); - // Try to parse the timestamp as JavaScript Date - let date = Date.parse(timestamp); - if (isNaN(date)) return false; - constructedFilters.push(`createdAt < ${date}`); - return false; - } + let filteredSearchTerms = ( + await Promise.all( + splitSearch.map(async (term) => { + if (term.startsWith("has:")) { + let fileType = term.slice(4); + constructedFilters.push(`mediaAttachment = "${fileType}"`); + return null; + } else if (term.startsWith("from:")) { + let user = term.slice(5); + constructedFilters.push(`userName = ${user}`); + return null; + } else if (term.startsWith("domain:")) { + let domain = term.slice(7); + constructedFilters.push(`userHost = ${domain}`); + return null; + } else if (term.startsWith("after:")) { + let timestamp = term.slice(6); + // Try to parse the timestamp as JavaScript Date + let date = Date.parse(timestamp); + if (isNaN(date)) return null; + constructedFilters.push(`createdAt > ${date / 1000}`); + return null; + } else if (term.startsWith("before:")) { + let timestamp = term.slice(7); + // Try to parse the timestamp as JavaScript Date + let date = Date.parse(timestamp); + if (isNaN(date)) return null; + constructedFilters.push(`createdAt < ${date / 1000}`); + return null; + } else if (term.startsWith("filter:following")) { + // Check if we got a context user + if (userCtx) { + // Fetch user follows from DB + let followedUsers = await Followings.find({ + where: { + followerId: userCtx.id, + }, + select: { + followeeId: true, + }, + }); + let followIDs = followedUsers.map((user) => user.followeeId); - return true; - }); + if (followIDs.length === 0) return null; - logger.info(`Searching for ${splitSearch.join(" ")}`); + constructedFilters.push(`userId IN [${followIDs.join(",")}]`); + } else { + logger.warn( + "search filtered to follows called without user context", + ); + } + + return null; + } else if (term.startsWith("filter:followers")) { + // Check if we got a context user + if (userCtx) { + // Fetch users follows from DB + let followedUsers = await Followings.find({ + where: { + followeeId: userCtx.id, + }, + select: { + followerId: true, + }, + }); + let followIDs = followedUsers.map((user) => user.followerId); + + if (followIDs.length === 0) return null; + + constructedFilters.push(`userId IN [${followIDs.join(",")}]`); + } else { + logger.warn( + "search filtered to followers called without user context", + ); + } + + return null; + } + + return term; + }), + ) + ).filter((term) => term !== null); + + let sortRules = []; + + // An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search + // These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want + if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) { + sortRules.push("createdAt:desc"); + } + + logger.info(`Searching for ${filteredSearchTerms.join(" ")}`); logger.info(`Limit: ${limit}`); logger.info(`Offset: ${offset}`); logger.info(`Filters: ${constructedFilters}`); + logger.info(`Ordering: ${sortRules}`); - return posts.search(splitSearch.join(" "), { + return posts.search(filteredSearchTerms.join(" "), { limit: limit, offset: offset, filter: constructedFilters, + sort: sortRules, }); }, ingestNote: async (ingestNotes: Note | Note[]) => { @@ -128,12 +207,11 @@ export default hasConfig for (let note of ingestNotes) { if (note.user === undefined) { - let user = await Users.findOne({ + note.user = await Users.findOne({ where: { id: note.userId, }, }); - note.user = user; } let attachmentType = ""; @@ -166,11 +244,13 @@ export default hasConfig }); } - let indexingIDs = indexingBatch.map((note) => note.id); - - return posts.addDocuments(indexingBatch, { - primaryKey: "id", - }); + return posts + .addDocuments(indexingBatch, { + primaryKey: "id", + }) + .then(() => + console.log(`sent ${indexingBatch.length} posts for indexing`), + ); }, serverStats: async () => { let health: Health = await client.health(); diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 60f264708..346304470 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -179,7 +179,7 @@ export default define(meta, paramDef, async (ps, me) => { // Use meilisearch to fetch and step through all search results that could match the requirements const ids = []; while (true) { - const results = await meilisearch.search(ps.query, chunkSize, start); + const results = await meilisearch.search(ps.query, chunkSize, start, me); start += chunkSize; From 40b9f87bef02afaed6f5db7011bc966999f530be Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Sun, 28 May 2023 02:19:57 +0200 Subject: [PATCH 32/34] add advanced search parameters in search popup --- packages/client/src/scripts/search.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/client/src/scripts/search.ts b/packages/client/src/scripts/search.ts index c7b8582ac..e405115bf 100644 --- a/packages/client/src/scripts/search.ts +++ b/packages/client/src/scripts/search.ts @@ -12,7 +12,10 @@ export async function search() { "has:image/video/audio/text/file => filter by attachment types\n" + "domain:domain.com => filter by domain\n" + "before:Date => show posts made before Date\n" + - "after:Date => show posts made after Date", + "after:Date => show posts made after Date\n" + + '"text" => get posts with exact text between quotes\n' + + "filter:following => show results only from users you follow\n" + + "filter:followers => show results only from followers\n", }); if (canceled || query == null || query === "") return; From 01ce43e6ad5d28d754a933be97d8f3700ecb8e2f Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Sun, 28 May 2023 02:58:20 +0200 Subject: [PATCH 33/34] Fix Meilisearch widget reactivity --- .../src/widgets/server-metric/meilisearch.vue | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/client/src/widgets/server-metric/meilisearch.vue b/packages/client/src/widgets/server-metric/meilisearch.vue index cb50c0301..48bf629b0 100644 --- a/packages/client/src/widgets/server-metric/meilisearch.vue +++ b/packages/client/src/widgets/server-metric/meilisearch.vue @@ -11,17 +11,32 @@