mirror of
https://iceshrimp.dev/crimekillz/trashposs
synced 2024-11-22 00:43:49 +01:00
[backend] Remove external search backends
This commit is contained in:
Laura Hausmann
parent
cdec8c4efd
commit
9b2e966c19
44
.pnp.cjs
generated
44
.pnp.cjs
generated
@ -1513,19 +1513,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
"linkType": "HARD"\
|
"linkType": "HARD"\
|
||||||
}]\
|
}]\
|
||||||
]],\
|
]],\
|
||||||
["@elastic/elasticsearch", [\
|
|
||||||
["npm:7.17.0", {\
|
|
||||||
"packageLocation": "./.yarn/cache/@elastic-elasticsearch-npm-7.17.0-f4178789c0-d54330ce50.zip/node_modules/@elastic/elasticsearch/",\
|
|
||||||
"packageDependencies": [\
|
|
||||||
["@elastic/elasticsearch", "npm:7.17.0"],\
|
|
||||||
["debug", "virtual:ac3d8e680759ce54399273724d44e041d6c9b73454d191d411a8c44bb27e22f02aaf6ed9d3ad0ac1c298eac4833cff369c9c7b84c573016112c4f84be2cd8543#npm:4.3.4"],\
|
|
||||||
["hpagent", "npm:0.1.2"],\
|
|
||||||
["ms", "npm:2.1.3"],\
|
|
||||||
["secure-json-parse", "npm:2.7.0"]\
|
|
||||||
],\
|
|
||||||
"linkType": "HARD"\
|
|
||||||
}]\
|
|
||||||
]],\
|
|
||||||
["@es-joy/jsdoccomment", [\
|
["@es-joy/jsdoccomment", [\
|
||||||
["npm:0.39.4", {\
|
["npm:0.39.4", {\
|
||||||
"packageLocation": "./.yarn/cache/@es-joy-jsdoccomment-npm-0.39.4-48cba32ec8-10d18c2de8.zip/node_modules/@es-joy/jsdoccomment/",\
|
"packageLocation": "./.yarn/cache/@es-joy-jsdoccomment-npm-0.39.4-48cba32ec8-10d18c2de8.zip/node_modules/@es-joy/jsdoccomment/",\
|
||||||
@ -6991,7 +6978,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
["@bull-board/koa", "npm:5.6.0"],\
|
["@bull-board/koa", "npm:5.6.0"],\
|
||||||
["@bull-board/ui", "npm:5.6.0"],\
|
["@bull-board/ui", "npm:5.6.0"],\
|
||||||
["@discordapp/twemoji", "npm:14.1.2"],\
|
["@discordapp/twemoji", "npm:14.1.2"],\
|
||||||
["@elastic/elasticsearch", "npm:7.17.0"],\
|
|
||||||
["@koa/cors", "npm:3.4.3"],\
|
["@koa/cors", "npm:3.4.3"],\
|
||||||
["@koa/multer", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:3.0.2"],\
|
["@koa/multer", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:3.0.2"],\
|
||||||
["@koa/router", "npm:9.0.1"],\
|
["@koa/router", "npm:9.0.1"],\
|
||||||
@ -7105,7 +7091,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
["koa-send", "npm:5.0.1"],\
|
["koa-send", "npm:5.0.1"],\
|
||||||
["koa-slow", "npm:2.1.0"],\
|
["koa-slow", "npm:2.1.0"],\
|
||||||
["koa-views", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:7.0.2"],\
|
["koa-views", "virtual:aa59773ac87791c4813d53447077fcf8a847d6de5a301d34dc31286584b1dbb26d30d3adb5b4c41c1e8aea04371e926fda05c09c6253647c432e11d872a304ba#npm:7.0.2"],\
|
||||||
["meilisearch", "npm:0.33.0"],\
|
|
||||||
["mfm-js", "npm:0.23.3"],\
|
["mfm-js", "npm:0.23.3"],\
|
||||||
["mime-types", "npm:2.1.35"],\
|
["mime-types", "npm:2.1.35"],\
|
||||||
["mocha", "npm:10.2.0"],\
|
["mocha", "npm:10.2.0"],\
|
||||||
@ -7140,7 +7125,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
["seedrandom", "npm:3.0.5"],\
|
["seedrandom", "npm:3.0.5"],\
|
||||||
["semver", "npm:7.5.4"],\
|
["semver", "npm:7.5.4"],\
|
||||||
["sharp", "npm:0.32.1"],\
|
["sharp", "npm:0.32.1"],\
|
||||||
["sonic-channel", "npm:1.3.1"],\
|
|
||||||
["strict-event-emitter-types", "npm:2.0.0"],\
|
["strict-event-emitter-types", "npm:2.0.0"],\
|
||||||
["stringz", "npm:2.1.0"],\
|
["stringz", "npm:2.1.0"],\
|
||||||
["summaly", "npm:2.7.0"],\
|
["summaly", "npm:2.7.0"],\
|
||||||
@ -17519,16 +17503,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
"linkType": "HARD"\
|
"linkType": "HARD"\
|
||||||
}]\
|
}]\
|
||||||
]],\
|
]],\
|
||||||
["meilisearch", [\
|
|
||||||
["npm:0.33.0", {\
|
|
||||||
"packageLocation": "./.yarn/cache/meilisearch-npm-0.33.0-a8742f194e-d2aff57b3d.zip/node_modules/meilisearch/",\
|
|
||||||
"packageDependencies": [\
|
|
||||||
["meilisearch", "npm:0.33.0"],\
|
|
||||||
["cross-fetch", "npm:3.1.8"]\
|
|
||||||
],\
|
|
||||||
"linkType": "HARD"\
|
|
||||||
}]\
|
|
||||||
]],\
|
|
||||||
["meow", [\
|
["meow", [\
|
||||||
["npm:9.0.0", {\
|
["npm:9.0.0", {\
|
||||||
"packageLocation": "./.yarn/cache/meow-npm-9.0.0-8b2707248e-3d0f199b9c.zip/node_modules/meow/",\
|
"packageLocation": "./.yarn/cache/meow-npm-9.0.0-8b2707248e-3d0f199b9c.zip/node_modules/meow/",\
|
||||||
@ -21742,15 +21716,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
"linkType": "HARD"\
|
"linkType": "HARD"\
|
||||||
}]\
|
}]\
|
||||||
]],\
|
]],\
|
||||||
["secure-json-parse", [\
|
|
||||||
["npm:2.7.0", {\
|
|
||||||
"packageLocation": "./.yarn/cache/secure-json-parse-npm-2.7.0-d5b89b0a3e-9743865870.zip/node_modules/secure-json-parse/",\
|
|
||||||
"packageDependencies": [\
|
|
||||||
["secure-json-parse", "npm:2.7.0"]\
|
|
||||||
],\
|
|
||||||
"linkType": "HARD"\
|
|
||||||
}]\
|
|
||||||
]],\
|
|
||||||
["seedrandom", [\
|
["seedrandom", [\
|
||||||
["npm:2.4.2", {\
|
["npm:2.4.2", {\
|
||||||
"packageLocation": "./.yarn/cache/seedrandom-npm-2.4.2-b435b54ae9-a0b6707cb7.zip/node_modules/seedrandom/",\
|
"packageLocation": "./.yarn/cache/seedrandom-npm-2.4.2-b435b54ae9-a0b6707cb7.zip/node_modules/seedrandom/",\
|
||||||
@ -22159,15 +22124,6 @@ const RAW_RUNTIME_STATE =
|
|||||||
"linkType": "HARD"\
|
"linkType": "HARD"\
|
||||||
}]\
|
}]\
|
||||||
]],\
|
]],\
|
||||||
["sonic-channel", [\
|
|
||||||
["npm:1.3.1", {\
|
|
||||||
"packageLocation": "./.yarn/cache/sonic-channel-npm-1.3.1-8f21a07e24-ee849863a3.zip/node_modules/sonic-channel/",\
|
|
||||||
"packageDependencies": [\
|
|
||||||
["sonic-channel", "npm:1.3.1"]\
|
|
||||||
],\
|
|
||||||
"linkType": "HARD"\
|
|
||||||
}]\
|
|
||||||
]],\
|
|
||||||
["sort-keys", [\
|
["sort-keys", [\
|
||||||
["npm:1.1.2", {\
|
["npm:1.1.2", {\
|
||||||
"packageLocation": "./.yarn/cache/sort-keys-npm-1.1.2-2ac0ab2d94-0ac2ea2327.zip/node_modules/sort-keys/",\
|
"packageLocation": "./.yarn/cache/sort-keys-npm-1.1.2-2ac0ab2d94-0ac2ea2327.zip/node_modules/sort-keys/",\
|
||||||
|
|||||||
@ -1,3 +0,0 @@
|
|||||||
version https://git-lfs.github.com/spec/v1
|
|
||||||
oid sha256:ac323648dbeb6e1e6fd781cfdf84e5a1094ed3614ac2e6ad031380a120bf757c
|
|
||||||
size 279263
|
|
||||||
BIN
.yarn/cache/meilisearch-npm-0.33.0-a8742f194e-d2aff57b3d.zip
(Stored with Git LFS)
vendored
BIN
.yarn/cache/meilisearch-npm-0.33.0-a8742f194e-d2aff57b3d.zip
(Stored with Git LFS)
vendored
Binary file not shown.
@ -1,3 +0,0 @@
|
|||||||
version https://git-lfs.github.com/spec/v1
|
|
||||||
oid sha256:1a489e82c8d3507c677532513a02f770a3f300ebd4f1a953154441174ff79578
|
|
||||||
size 14270
|
|
||||||
BIN
.yarn/cache/sonic-channel-npm-1.3.1-8f21a07e24-ee849863a3.zip
(Stored with Git LFS)
vendored
BIN
.yarn/cache/sonic-channel-npm-1.3.1-8f21a07e24-ee849863a3.zip
(Stored with Git LFS)
vendored
Binary file not shown.
@ -27,7 +27,6 @@
|
|||||||
"@bull-board/koa": "5.6.0",
|
"@bull-board/koa": "5.6.0",
|
||||||
"@bull-board/ui": "5.6.0",
|
"@bull-board/ui": "5.6.0",
|
||||||
"@discordapp/twemoji": "14.1.2",
|
"@discordapp/twemoji": "14.1.2",
|
||||||
"@elastic/elasticsearch": "7.17.0",
|
|
||||||
"@koa/cors": "3.4.3",
|
"@koa/cors": "3.4.3",
|
||||||
"@koa/multer": "3.0.2",
|
"@koa/multer": "3.0.2",
|
||||||
"@koa/router": "9.0.1",
|
"@koa/router": "9.0.1",
|
||||||
@ -88,7 +87,6 @@
|
|||||||
"koa-send": "5.0.1",
|
"koa-send": "5.0.1",
|
||||||
"koa-slow": "2.1.0",
|
"koa-slow": "2.1.0",
|
||||||
"koa-views": "7.0.2",
|
"koa-views": "7.0.2",
|
||||||
"meilisearch": "0.33.0",
|
|
||||||
"mfm-js": "0.23.3",
|
"mfm-js": "0.23.3",
|
||||||
"mime-types": "2.1.35",
|
"mime-types": "2.1.35",
|
||||||
"msgpackr": "1.9.5",
|
"msgpackr": "1.9.5",
|
||||||
@ -121,7 +119,6 @@
|
|||||||
"seedrandom": "^3.0.5",
|
"seedrandom": "^3.0.5",
|
||||||
"semver": "7.5.4",
|
"semver": "7.5.4",
|
||||||
"sharp": "0.32.1",
|
"sharp": "0.32.1",
|
||||||
"sonic-channel": "^1.3.1",
|
|
||||||
"stringz": "2.1.0",
|
"stringz": "2.1.0",
|
||||||
"summaly": "2.7.0",
|
"summaly": "2.7.0",
|
||||||
"syslog-pro": "1.0.0",
|
"syslog-pro": "1.0.0",
|
||||||
|
|||||||
@ -27,27 +27,6 @@ export type Source = {
|
|||||||
user?: string;
|
user?: string;
|
||||||
tls?: { [y: string]: string };
|
tls?: { [y: string]: string };
|
||||||
};
|
};
|
||||||
elasticsearch: {
|
|
||||||
host: string;
|
|
||||||
port: number;
|
|
||||||
ssl?: boolean;
|
|
||||||
user?: string;
|
|
||||||
pass?: string;
|
|
||||||
index?: string;
|
|
||||||
};
|
|
||||||
sonic: {
|
|
||||||
host: string;
|
|
||||||
port: number;
|
|
||||||
auth?: string;
|
|
||||||
collection?: string;
|
|
||||||
bucket?: string;
|
|
||||||
};
|
|
||||||
meilisearch: {
|
|
||||||
host: string;
|
|
||||||
port: number;
|
|
||||||
apiKey?: string;
|
|
||||||
ssl: boolean;
|
|
||||||
};
|
|
||||||
|
|
||||||
mediaCleanup?: {
|
mediaCleanup?: {
|
||||||
cron?: boolean;
|
cron?: boolean;
|
||||||
|
|||||||
@ -2,7 +2,6 @@ import si from "systeminformation";
|
|||||||
import Xev from "xev";
|
import Xev from "xev";
|
||||||
import * as osUtils from "os-utils";
|
import * as osUtils from "os-utils";
|
||||||
import { fetchMeta } from "@/misc/fetch-meta.js";
|
import { fetchMeta } from "@/misc/fetch-meta.js";
|
||||||
import meilisearch from "../db/meilisearch.js";
|
|
||||||
|
|
||||||
const ev = new Xev();
|
const ev = new Xev();
|
||||||
|
|
||||||
@ -30,7 +29,6 @@ export default function () {
|
|||||||
const memStats = await mem();
|
const memStats = await mem();
|
||||||
const netStats = await net();
|
const netStats = await net();
|
||||||
const fsStats = await fs();
|
const fsStats = await fs();
|
||||||
const meilisearchStats = await meilisearchStatus();
|
|
||||||
|
|
||||||
const stats = {
|
const stats = {
|
||||||
cpu: roundCpu(cpu),
|
cpu: roundCpu(cpu),
|
||||||
@ -46,8 +44,7 @@ export default function () {
|
|||||||
fs: {
|
fs: {
|
||||||
r: round(Math.max(0, fsStats.rIO_sec ?? 0)),
|
r: round(Math.max(0, fsStats.rIO_sec ?? 0)),
|
||||||
w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
|
w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
|
||||||
},
|
}
|
||||||
meilisearch: meilisearchStats,
|
|
||||||
};
|
};
|
||||||
ev.emit("serverStats", stats);
|
ev.emit("serverStats", stats);
|
||||||
log.unshift(stats);
|
log.unshift(stats);
|
||||||
@ -86,16 +83,3 @@ async function fs() {
|
|||||||
const data = await si.disksIO().catch(() => ({ rIO_sec: 0, wIO_sec: 0 }));
|
const data = await si.disksIO().catch(() => ({ rIO_sec: 0, wIO_sec: 0 }));
|
||||||
return data || { rIO_sec: 0, wIO_sec: 0 };
|
return data || { rIO_sec: 0, wIO_sec: 0 };
|
||||||
}
|
}
|
||||||
|
|
||||||
// MEILI STAT
|
|
||||||
async function meilisearchStatus() {
|
|
||||||
if (meilisearch) {
|
|
||||||
return meilisearch.serverStats();
|
|
||||||
} else {
|
|
||||||
return {
|
|
||||||
health: "unconfigured",
|
|
||||||
size: 0,
|
|
||||||
indexed_count: 0,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,65 +0,0 @@
|
|||||||
import * as elasticsearch from "@elastic/elasticsearch";
|
|
||||||
import config from "@/config/index.js";
|
|
||||||
|
|
||||||
const index = {
|
|
||||||
settings: {
|
|
||||||
analysis: {
|
|
||||||
analyzer: {
|
|
||||||
ngram: {
|
|
||||||
tokenizer: "ngram",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
mappings: {
|
|
||||||
properties: {
|
|
||||||
text: {
|
|
||||||
type: "text",
|
|
||||||
index: true,
|
|
||||||
analyzer: "ngram",
|
|
||||||
},
|
|
||||||
userId: {
|
|
||||||
type: "keyword",
|
|
||||||
index: true,
|
|
||||||
},
|
|
||||||
userHost: {
|
|
||||||
type: "keyword",
|
|
||||||
index: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
// Init ElasticSearch connection
|
|
||||||
const client = config.elasticsearch
|
|
||||||
? new elasticsearch.Client({
|
|
||||||
node: `${config.elasticsearch.ssl ? "https://" : "http://"}${
|
|
||||||
config.elasticsearch.host
|
|
||||||
}:${config.elasticsearch.port}`,
|
|
||||||
auth:
|
|
||||||
config.elasticsearch.user && config.elasticsearch.pass
|
|
||||||
? {
|
|
||||||
username: config.elasticsearch.user,
|
|
||||||
password: config.elasticsearch.pass,
|
|
||||||
}
|
|
||||||
: undefined,
|
|
||||||
pingTimeout: 30000,
|
|
||||||
})
|
|
||||||
: null;
|
|
||||||
|
|
||||||
if (client) {
|
|
||||||
client.indices
|
|
||||||
.exists({
|
|
||||||
index: config.elasticsearch.index || "misskey_note",
|
|
||||||
})
|
|
||||||
.then((exist) => {
|
|
||||||
if (!exist.body) {
|
|
||||||
client.indices.create({
|
|
||||||
index: config.elasticsearch.index || "misskey_note",
|
|
||||||
body: index,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export default client;
|
|
||||||
@ -1,411 +0,0 @@
|
|||||||
import { Health, Index, MeiliSearch, Stats } from "meilisearch";
|
|
||||||
import { dbLogger } from "./logger.js";
|
|
||||||
|
|
||||||
import config from "@/config/index.js";
|
|
||||||
import { Note } from "@/models/entities/note.js";
|
|
||||||
import * as url from "url";
|
|
||||||
import { ILocalUser } from "@/models/entities/user.js";
|
|
||||||
import { Followings, Users } from "@/models/index.js";
|
|
||||||
|
|
||||||
const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
|
|
||||||
|
|
||||||
let posts: Index;
|
|
||||||
let client: MeiliSearch;
|
|
||||||
|
|
||||||
const hasConfig =
|
|
||||||
config.meilisearch &&
|
|
||||||
(config.meilisearch.host ||
|
|
||||||
config.meilisearch.port ||
|
|
||||||
config.meilisearch.apiKey);
|
|
||||||
|
|
||||||
if (hasConfig) {
|
|
||||||
const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
|
|
||||||
const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
|
|
||||||
const auth = hasConfig ? config.meilisearch.apiKey ?? "" : "";
|
|
||||||
const ssl = hasConfig ? config.meilisearch.ssl ?? false : false;
|
|
||||||
|
|
||||||
logger.info("Connecting to MeiliSearch");
|
|
||||||
|
|
||||||
client = new MeiliSearch({
|
|
||||||
host: `${ssl ? "https" : "http"}://${host}:${port}`,
|
|
||||||
apiKey: auth,
|
|
||||||
});
|
|
||||||
|
|
||||||
posts = client.index("posts");
|
|
||||||
|
|
||||||
posts
|
|
||||||
.updateSearchableAttributes(["text"])
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(`Setting searchable attr failed, searches won't work: ${e}`),
|
|
||||||
);
|
|
||||||
|
|
||||||
posts
|
|
||||||
.updateFilterableAttributes([
|
|
||||||
"userName",
|
|
||||||
"userHost",
|
|
||||||
"mediaAttachment",
|
|
||||||
"createdAt",
|
|
||||||
"userId",
|
|
||||||
])
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
`Setting filterable attr failed, advanced searches won't work: ${e}`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
posts
|
|
||||||
.updateSortableAttributes(["createdAt"])
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
`Setting sortable attr failed, placeholder searches won't sort properly: ${e}`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
posts
|
|
||||||
.updateStopWords([
|
|
||||||
"the",
|
|
||||||
"a",
|
|
||||||
"as",
|
|
||||||
"be",
|
|
||||||
"of",
|
|
||||||
"they",
|
|
||||||
"these",
|
|
||||||
"これ",
|
|
||||||
"それ",
|
|
||||||
"あれ",
|
|
||||||
"この",
|
|
||||||
"その",
|
|
||||||
"あの",
|
|
||||||
"ここ",
|
|
||||||
"そこ",
|
|
||||||
"あそこ",
|
|
||||||
"こちら",
|
|
||||||
"どこ",
|
|
||||||
"だれ",
|
|
||||||
"なに",
|
|
||||||
"なん",
|
|
||||||
"何",
|
|
||||||
"私",
|
|
||||||
"貴方",
|
|
||||||
"貴方方",
|
|
||||||
"我々",
|
|
||||||
"私達",
|
|
||||||
"あの人",
|
|
||||||
"あのか",
|
|
||||||
"彼女",
|
|
||||||
"彼",
|
|
||||||
"です",
|
|
||||||
"ありま",
|
|
||||||
"おりま",
|
|
||||||
"います",
|
|
||||||
"は",
|
|
||||||
"が",
|
|
||||||
"の",
|
|
||||||
"に",
|
|
||||||
"を",
|
|
||||||
"で",
|
|
||||||
"え",
|
|
||||||
"から",
|
|
||||||
"まで",
|
|
||||||
"より",
|
|
||||||
"も",
|
|
||||||
"どの",
|
|
||||||
"と",
|
|
||||||
"し",
|
|
||||||
"それで",
|
|
||||||
"しかし",
|
|
||||||
])
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
`Failed to set Meilisearch stop words, database size will be larger: ${e}`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.info("Connected to MeiliSearch");
|
|
||||||
}
|
|
||||||
|
|
||||||
export type MeilisearchNote = {
|
|
||||||
id: string;
|
|
||||||
text: string;
|
|
||||||
userId: string;
|
|
||||||
userHost: string;
|
|
||||||
userName: string;
|
|
||||||
channelId: string;
|
|
||||||
mediaAttachment: string;
|
|
||||||
createdAt: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
function timestampToUnix(timestamp: string) {
|
|
||||||
let unix = 0;
|
|
||||||
|
|
||||||
// Only contains numbers => UNIX timestamp
|
|
||||||
if (/^\d+$/.test(timestamp)) {
|
|
||||||
unix = Number.parseInt(timestamp);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unix === 0) {
|
|
||||||
// Try to parse the timestamp as JavaScript Date
|
|
||||||
const date = Date.parse(timestamp);
|
|
||||||
if (isNaN(date)) return 0;
|
|
||||||
unix = date / 1000;
|
|
||||||
}
|
|
||||||
|
|
||||||
return unix;
|
|
||||||
}
|
|
||||||
|
|
||||||
export default hasConfig
|
|
||||||
? {
|
|
||||||
search: async (
|
|
||||||
query: string,
|
|
||||||
limit: number,
|
|
||||||
offset: number,
|
|
||||||
userCtx: ILocalUser | null,
|
|
||||||
) => {
|
|
||||||
/// Advanced search syntax
|
|
||||||
/// from:user => filter by user + optional domain
|
|
||||||
/// has:image/video/audio/text/file => filter by attachment types
|
|
||||||
/// domain:domain.com => filter by domain
|
|
||||||
/// before:Date => show posts made before Date
|
|
||||||
/// after: Date => show posts made after Date
|
|
||||||
/// "text" => get posts with exact text between quotes
|
|
||||||
/// filter:following => show results only from users you follow
|
|
||||||
/// filter:followers => show results only from followers
|
|
||||||
|
|
||||||
const constructedFilters: string[] = [];
|
|
||||||
|
|
||||||
const splitSearch = query.split(" ");
|
|
||||||
|
|
||||||
// Detect search operators and remove them from the actual query
|
|
||||||
const filteredSearchTerms = (
|
|
||||||
await Promise.all(
|
|
||||||
splitSearch.map(async (term) => {
|
|
||||||
if (term.startsWith("has:")) {
|
|
||||||
const fileType = term.slice(4);
|
|
||||||
constructedFilters.push(`mediaAttachment = "${fileType}"`);
|
|
||||||
return null;
|
|
||||||
} else if (term.startsWith("from:")) {
|
|
||||||
let user = term.slice(5);
|
|
||||||
|
|
||||||
if (user.length === 0) return null;
|
|
||||||
|
|
||||||
// Cut off leading @, those aren't saved in the DB
|
|
||||||
if (user.charAt(0) === "@") {
|
|
||||||
user = user.slice(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine if we got a webfinger address or a single username
|
|
||||||
if (user.split("@").length > 1) {
|
|
||||||
let splitUser = user.split("@");
|
|
||||||
|
|
||||||
let domain = splitUser.pop();
|
|
||||||
user = splitUser.join("@");
|
|
||||||
|
|
||||||
constructedFilters.push(
|
|
||||||
`userName = ${user} AND userHost = ${domain}`,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
constructedFilters.push(`userName = ${user}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
} else if (term.startsWith("domain:")) {
|
|
||||||
const domain = term.slice(7);
|
|
||||||
constructedFilters.push(`userHost = ${domain}`);
|
|
||||||
return null;
|
|
||||||
} else if (term.startsWith("after:")) {
|
|
||||||
const timestamp = term.slice(6);
|
|
||||||
|
|
||||||
let unix = timestampToUnix(timestamp);
|
|
||||||
|
|
||||||
if (unix !== 0) constructedFilters.push(`createdAt > ${unix}`);
|
|
||||||
|
|
||||||
return null;
|
|
||||||
} else if (term.startsWith("before:")) {
|
|
||||||
const timestamp = term.slice(7);
|
|
||||||
|
|
||||||
let unix = timestampToUnix(timestamp);
|
|
||||||
if (unix !== 0) constructedFilters.push(`createdAt < ${unix}`);
|
|
||||||
|
|
||||||
return null;
|
|
||||||
} else if (term.startsWith("filter:following")) {
|
|
||||||
// Check if we got a context user
|
|
||||||
if (userCtx) {
|
|
||||||
// Fetch user follows from DB
|
|
||||||
const followedUsers = await Followings.find({
|
|
||||||
where: {
|
|
||||||
followerId: userCtx.id,
|
|
||||||
},
|
|
||||||
select: {
|
|
||||||
followeeId: true,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const followIDs = followedUsers.map(
|
|
||||||
(user) => user.followeeId,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (followIDs.length === 0) return null;
|
|
||||||
|
|
||||||
constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
|
|
||||||
} else {
|
|
||||||
logger.warn(
|
|
||||||
"search filtered to follows called without user context",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
} else if (term.startsWith("filter:followers")) {
|
|
||||||
// Check if we got a context user
|
|
||||||
if (userCtx) {
|
|
||||||
// Fetch users follows from DB
|
|
||||||
const followedUsers = await Followings.find({
|
|
||||||
where: {
|
|
||||||
followeeId: userCtx.id,
|
|
||||||
},
|
|
||||||
select: {
|
|
||||||
followerId: true,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const followIDs = followedUsers.map(
|
|
||||||
(user) => user.followerId,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (followIDs.length === 0) return null;
|
|
||||||
|
|
||||||
constructedFilters.push(`userId IN [${followIDs.join(",")}]`);
|
|
||||||
} else {
|
|
||||||
logger.warn(
|
|
||||||
"search filtered to followers called without user context",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return term;
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
).filter((term) => term !== null);
|
|
||||||
|
|
||||||
const sortRules = [];
|
|
||||||
|
|
||||||
// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
|
|
||||||
// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
|
|
||||||
if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) {
|
|
||||||
sortRules.push("createdAt:desc");
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
|
|
||||||
logger.info(`Limit: ${limit}`);
|
|
||||||
logger.info(`Offset: ${offset}`);
|
|
||||||
logger.info(`Filters: ${constructedFilters}`);
|
|
||||||
logger.info(`Ordering: ${sortRules}`);
|
|
||||||
|
|
||||||
return posts.search(filteredSearchTerms.join(" "), {
|
|
||||||
limit: limit,
|
|
||||||
offset: offset,
|
|
||||||
filter: constructedFilters,
|
|
||||||
sort: sortRules,
|
|
||||||
});
|
|
||||||
},
|
|
||||||
ingestNote: async (ingestNotes: Note | Note[]) => {
|
|
||||||
if (ingestNotes instanceof Note) {
|
|
||||||
ingestNotes = [ingestNotes];
|
|
||||||
}
|
|
||||||
|
|
||||||
const indexingBatch: MeilisearchNote[] = [];
|
|
||||||
|
|
||||||
for (const note of ingestNotes) {
|
|
||||||
if (note.user === undefined) {
|
|
||||||
note.user = await Users.findOne({
|
|
||||||
where: {
|
|
||||||
id: note.userId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let attachmentType = "";
|
|
||||||
if (note.attachedFileTypes.length > 0) {
|
|
||||||
attachmentType = note.attachedFileTypes[0].split("/")[0];
|
|
||||||
switch (attachmentType) {
|
|
||||||
case "image":
|
|
||||||
case "video":
|
|
||||||
case "audio":
|
|
||||||
case "text":
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
attachmentType = "file";
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
indexingBatch.push(<MeilisearchNote>{
|
|
||||||
id: note.id.toString(),
|
|
||||||
text: note.text ? note.text : "",
|
|
||||||
userId: note.userId,
|
|
||||||
userHost:
|
|
||||||
note.userHost !== ""
|
|
||||||
? note.userHost
|
|
||||||
: config.domain,
|
|
||||||
channelId: note.channelId ? note.channelId : "",
|
|
||||||
mediaAttachment: attachmentType,
|
|
||||||
userName: note.user?.username ?? "UNKNOWN",
|
|
||||||
createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return posts
|
|
||||||
.addDocuments(indexingBatch, {
|
|
||||||
primaryKey: "id",
|
|
||||||
})
|
|
||||||
.then(() =>
|
|
||||||
logger.info(`sent ${indexingBatch.length} posts for indexing`),
|
|
||||||
);
|
|
||||||
},
|
|
||||||
serverStats: async () => {
|
|
||||||
const health: Health = await client.health();
|
|
||||||
const stats: Stats = await client.getStats();
|
|
||||||
|
|
||||||
return {
|
|
||||||
health: health.status,
|
|
||||||
size: stats.databaseSize,
|
|
||||||
indexed_count: stats.indexes["posts"].numberOfDocuments,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
deleteNotes: async (note: Note | Note[] | string | string[]) => {
|
|
||||||
if (note instanceof Note) {
|
|
||||||
note = [note];
|
|
||||||
}
|
|
||||||
if (typeof note === "string") {
|
|
||||||
note = [note];
|
|
||||||
}
|
|
||||||
|
|
||||||
const deletionBatch = note
|
|
||||||
.map((n) => {
|
|
||||||
if (n instanceof Note) {
|
|
||||||
return n.id;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (n.length > 0) return n;
|
|
||||||
|
|
||||||
logger.error(
|
|
||||||
`Failed to delete note from Meilisearch, invalid post ID: ${JSON.stringify(
|
|
||||||
n,
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
throw new Error(
|
|
||||||
`Invalid note ID passed to meilisearch deleteNote: ${JSON.stringify(
|
|
||||||
n,
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
})
|
|
||||||
.filter((el) => el !== null);
|
|
||||||
|
|
||||||
await posts.deleteDocuments(deletionBatch as string[]).then(() => {
|
|
||||||
logger.info(
|
|
||||||
`submitted ${deletionBatch.length} large batch for deletion`,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
},
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
@ -1,51 +0,0 @@
|
|||||||
import * as SonicChannel from "sonic-channel";
|
|
||||||
import { dbLogger } from "./logger.js";
|
|
||||||
|
|
||||||
import config from "@/config/index.js";
|
|
||||||
|
|
||||||
const logger = dbLogger.createSubLogger("sonic", "gray", false);
|
|
||||||
|
|
||||||
const handlers = (type: string): SonicChannel.Handlers => ({
|
|
||||||
connected: () => {
|
|
||||||
logger.succ(`Connected to Sonic ${type}`);
|
|
||||||
},
|
|
||||||
disconnected: (error) => {
|
|
||||||
logger.warn(`Disconnected from Sonic ${type}, error: ${error}`);
|
|
||||||
},
|
|
||||||
error: (error) => {
|
|
||||||
logger.warn(`Sonic ${type} error: ${error}`);
|
|
||||||
},
|
|
||||||
retrying: () => {
|
|
||||||
logger.info(`Sonic ${type} retrying`);
|
|
||||||
},
|
|
||||||
timeout: () => {
|
|
||||||
logger.warn(`Sonic ${type} timeout`);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const hasConfig =
|
|
||||||
config.sonic && (config.sonic.host || config.sonic.port || config.sonic.auth);
|
|
||||||
|
|
||||||
if (hasConfig) {
|
|
||||||
logger.info("Connecting to Sonic");
|
|
||||||
}
|
|
||||||
|
|
||||||
const host = hasConfig ? config.sonic.host ?? "localhost" : "";
|
|
||||||
const port = hasConfig ? config.sonic.port ?? 1491 : 0;
|
|
||||||
const auth = hasConfig ? config.sonic.auth ?? "SecretPassword" : "";
|
|
||||||
const collection = hasConfig ? config.sonic.collection ?? "main" : "";
|
|
||||||
const bucket = hasConfig ? config.sonic.bucket ?? "default" : "";
|
|
||||||
|
|
||||||
export default hasConfig
|
|
||||||
? {
|
|
||||||
search: new SonicChannel.Search({ host, port, auth }).connect(
|
|
||||||
handlers("search"),
|
|
||||||
),
|
|
||||||
ingest: new SonicChannel.Ingest({ host, port, auth }).connect(
|
|
||||||
handlers("ingest"),
|
|
||||||
),
|
|
||||||
|
|
||||||
collection,
|
|
||||||
bucket,
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
@ -1,88 +0,0 @@
|
|||||||
import type Bull from "bull";
|
|
||||||
import type { DoneCallback } from "bull";
|
|
||||||
|
|
||||||
import { queueLogger } from "../../logger.js";
|
|
||||||
import { Notes } from "@/models/index.js";
|
|
||||||
import { MoreThan } from "typeorm";
|
|
||||||
import { index } from "@/services/note/create.js";
|
|
||||||
import { Note } from "@/models/entities/note.js";
|
|
||||||
import meilisearch from "../../../db/meilisearch.js";
|
|
||||||
|
|
||||||
const logger = queueLogger.createSubLogger("index-all-notes");
|
|
||||||
|
|
||||||
export default async function indexAllNotes(
|
|
||||||
job: Bull.Job<Record<string, unknown>>,
|
|
||||||
done: DoneCallback,
|
|
||||||
): Promise<void> {
|
|
||||||
logger.info("Indexing all notes...");
|
|
||||||
|
|
||||||
let cursor: string | null = (job.data.cursor as string) ?? null;
|
|
||||||
let indexedCount: number = (job.data.indexedCount as number) ?? 0;
|
|
||||||
let total: number = (job.data.total as number) ?? 0;
|
|
||||||
|
|
||||||
let running = true;
|
|
||||||
const take = 10000;
|
|
||||||
const batch = 100;
|
|
||||||
while (running) {
|
|
||||||
logger.info(
|
|
||||||
`Querying for ${take} notes ${indexedCount}/${
|
|
||||||
total ? total : "?"
|
|
||||||
} at ${cursor}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
let notes: Note[] = [];
|
|
||||||
try {
|
|
||||||
notes = await Notes.find({
|
|
||||||
where: {
|
|
||||||
...(cursor ? { id: MoreThan(cursor) } : {}),
|
|
||||||
},
|
|
||||||
take: take,
|
|
||||||
order: {
|
|
||||||
id: 1,
|
|
||||||
},
|
|
||||||
relations: ["user"],
|
|
||||||
});
|
|
||||||
} catch (e: any) {
|
|
||||||
logger.error(`Failed to query notes ${e}`);
|
|
||||||
done(e);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (notes.length === 0) {
|
|
||||||
await job.progress(100);
|
|
||||||
running = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const count = await Notes.count();
|
|
||||||
total = count;
|
|
||||||
await job.update({ indexedCount, cursor, total });
|
|
||||||
} catch (e) {}
|
|
||||||
|
|
||||||
for (let i = 0; i < notes.length; i += batch) {
|
|
||||||
const chunk = notes.slice(i, i + batch);
|
|
||||||
|
|
||||||
if (meilisearch) {
|
|
||||||
await meilisearch.ingestNote(chunk);
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all(chunk.map((note) => index(note, true)));
|
|
||||||
|
|
||||||
indexedCount += chunk.length;
|
|
||||||
const pct = (indexedCount / total) * 100;
|
|
||||||
await job.update({ indexedCount, cursor, total });
|
|
||||||
await job.progress(+pct.toFixed(1));
|
|
||||||
logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`);
|
|
||||||
}
|
|
||||||
cursor = notes[notes.length - 1].id;
|
|
||||||
await job.update({ indexedCount, cursor, total });
|
|
||||||
|
|
||||||
if (notes.length < take) {
|
|
||||||
running = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
done();
|
|
||||||
logger.info("All notes have been indexed.");
|
|
||||||
}
|
|
||||||
@ -1,9 +1,6 @@
|
|||||||
import type Bull from "bull";
|
import type Bull from "bull";
|
||||||
import indexAllNotes from "./index-all-notes.js";
|
|
||||||
|
|
||||||
const jobs = {
|
const jobs = {} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>>>;
|
||||||
indexAllNotes,
|
|
||||||
} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>>>;
|
|
||||||
|
|
||||||
export default function (q: Bull.Queue) {
|
export default function (q: Bull.Queue) {
|
||||||
for (const [k, v] of Object.entries(jobs)) {
|
for (const [k, v] of Object.entries(jobs)) {
|
||||||
|
|||||||
@ -7,7 +7,6 @@ import type { DriveFile } from "@/models/entities/drive-file.js";
|
|||||||
import { MoreThan } from "typeorm";
|
import { MoreThan } from "typeorm";
|
||||||
import { deleteFileSync } from "@/services/drive/delete-file.js";
|
import { deleteFileSync } from "@/services/drive/delete-file.js";
|
||||||
import { sendEmail } from "@/services/send-email.js";
|
import { sendEmail } from "@/services/send-email.js";
|
||||||
import meilisearch from "@/db/meilisearch.js";
|
|
||||||
import { publishInternalEvent } from "@/services/stream.js";
|
import { publishInternalEvent } from "@/services/stream.js";
|
||||||
|
|
||||||
const logger = queueLogger.createSubLogger("delete-account");
|
const logger = queueLogger.createSubLogger("delete-account");
|
||||||
@ -44,9 +43,6 @@ export async function deleteAccount(
|
|||||||
cursor = notes[notes.length - 1].id;
|
cursor = notes[notes.length - 1].id;
|
||||||
|
|
||||||
await Notes.delete(notes.map((note) => note.id));
|
await Notes.delete(notes.map((note) => note.id));
|
||||||
if (meilisearch) {
|
|
||||||
await meilisearch.deleteNotes(notes);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.succ("All of notes deleted");
|
logger.succ("All of notes deleted");
|
||||||
|
|||||||
@ -489,7 +489,7 @@ export default define(meta, paramDef, async (ps, me) => {
|
|||||||
recommendedTimeline: !instance.disableRecommendedTimeline,
|
recommendedTimeline: !instance.disableRecommendedTimeline,
|
||||||
globalTimeLine: !instance.disableGlobalTimeline,
|
globalTimeLine: !instance.disableGlobalTimeline,
|
||||||
emailRequiredForSignup: instance.emailRequiredForSignup,
|
emailRequiredForSignup: instance.emailRequiredForSignup,
|
||||||
searchFilters: !!config.meilisearch,
|
searchFilters: true,
|
||||||
hcaptcha: instance.enableHcaptcha,
|
hcaptcha: instance.enableHcaptcha,
|
||||||
recaptcha: instance.enableRecaptcha,
|
recaptcha: instance.enableRecaptcha,
|
||||||
objectStorage: instance.useObjectStorage,
|
objectStorage: instance.useObjectStorage,
|
||||||
|
|||||||
@ -1,10 +1,5 @@
|
|||||||
import { In } from "typeorm";
|
|
||||||
import { Notes } from "@/models/index.js";
|
import { Notes } from "@/models/index.js";
|
||||||
import { Note } from "@/models/entities/note.js";
|
import { Note } from "@/models/entities/note.js";
|
||||||
import config from "@/config/index.js";
|
|
||||||
import es from "@/db/elasticsearch.js";
|
|
||||||
import sonic from "@/db/sonic.js";
|
|
||||||
import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js";
|
|
||||||
import define from "../../define.js";
|
import define from "../../define.js";
|
||||||
import { makePaginationQuery } from "../../common/make-pagination-query.js";
|
import { makePaginationQuery } from "../../common/make-pagination-query.js";
|
||||||
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
|
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
|
||||||
@ -63,258 +58,39 @@ export const paramDef = {
|
|||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export default define(meta, paramDef, async (ps, me) => {
|
export default define(meta, paramDef, async (ps, me) => {
|
||||||
if (es == null && sonic == null && meilisearch == null) {
|
const query = makePaginationQuery(
|
||||||
const query = makePaginationQuery(
|
Notes.createQueryBuilder("note"),
|
||||||
Notes.createQueryBuilder("note"),
|
ps.sinceId,
|
||||||
ps.sinceId,
|
ps.untilId,
|
||||||
ps.untilId,
|
);
|
||||||
);
|
|
||||||
|
|
||||||
if (ps.userId) {
|
if (ps.userId) {
|
||||||
query.andWhere("note.userId = :userId", { userId: ps.userId });
|
query.andWhere("note.userId = :userId", { userId: ps.userId });
|
||||||
} else if (ps.channelId) {
|
} else if (ps.channelId) {
|
||||||
query.andWhere("note.channelId = :channelId", {
|
query.andWhere("note.channelId = :channelId", {
|
||||||
channelId: ps.channelId,
|
channelId: ps.channelId,
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
query
|
|
||||||
.andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` })
|
|
||||||
.innerJoinAndSelect("note.user", "user")
|
|
||||||
.leftJoinAndSelect("user.avatar", "avatar")
|
|
||||||
.leftJoinAndSelect("user.banner", "banner")
|
|
||||||
.leftJoinAndSelect("note.reply", "reply")
|
|
||||||
.leftJoinAndSelect("note.renote", "renote")
|
|
||||||
.leftJoinAndSelect("reply.user", "replyUser")
|
|
||||||
.leftJoinAndSelect("replyUser.avatar", "replyUserAvatar")
|
|
||||||
.leftJoinAndSelect("replyUser.banner", "replyUserBanner")
|
|
||||||
.leftJoinAndSelect("renote.user", "renoteUser")
|
|
||||||
.leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar")
|
|
||||||
.leftJoinAndSelect("renoteUser.banner", "renoteUserBanner");
|
|
||||||
|
|
||||||
generateVisibilityQuery(query, me);
|
|
||||||
if (me) generateMutedUserQuery(query, me);
|
|
||||||
if (me) generateBlockedUserQuery(query, me);
|
|
||||||
|
|
||||||
const notes: Note[] = await query.take(ps.limit).getMany();
|
|
||||||
|
|
||||||
return await Notes.packMany(notes, me);
|
|
||||||
} else if (sonic) {
|
|
||||||
let start = 0;
|
|
||||||
const chunkSize = 100;
|
|
||||||
|
|
||||||
// Use sonic to fetch and step through all search results that could match the requirements
|
|
||||||
const ids = [];
|
|
||||||
while (true) {
|
|
||||||
const results = await sonic.search.query(
|
|
||||||
sonic.collection,
|
|
||||||
sonic.bucket,
|
|
||||||
ps.query,
|
|
||||||
{
|
|
||||||
limit: chunkSize,
|
|
||||||
offset: start,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
start += chunkSize;
|
|
||||||
|
|
||||||
if (results.length === 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = results
|
|
||||||
.map((k) => JSON.parse(k))
|
|
||||||
.filter((key) => {
|
|
||||||
if (ps.userId && key.userId !== ps.userId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ps.channelId && key.channelId !== ps.channelId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ps.sinceId && key.id <= ps.sinceId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ps.untilId && key.id >= ps.untilId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
})
|
|
||||||
.map((key) => key.id);
|
|
||||||
|
|
||||||
ids.push(...res);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort all the results by note id DESC (newest first)
|
|
||||||
ids.sort((a, b) => b - a);
|
|
||||||
|
|
||||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
|
||||||
start = 0;
|
|
||||||
const found = [];
|
|
||||||
while (found.length < ps.limit && start < ids.length) {
|
|
||||||
const chunk = ids.slice(start, start + chunkSize);
|
|
||||||
const notes: Note[] = await Notes.find({
|
|
||||||
where: {
|
|
||||||
id: In(chunk),
|
|
||||||
},
|
|
||||||
order: {
|
|
||||||
id: "DESC",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// The notes are checked for visibility and muted/blocked users when packed
|
|
||||||
found.push(...(await Notes.packMany(notes, me)));
|
|
||||||
start += chunkSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we have more results than the limit, trim them
|
|
||||||
if (found.length > ps.limit) {
|
|
||||||
found.length = ps.limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
return found;
|
|
||||||
} else if (meilisearch) {
|
|
||||||
let start = 0;
|
|
||||||
const chunkSize = 100;
|
|
||||||
|
|
||||||
// Use meilisearch to fetch and step through all search results that could match the requirements
|
|
||||||
const ids = [];
|
|
||||||
while (true) {
|
|
||||||
const results = await meilisearch.search(ps.query, chunkSize, start, me);
|
|
||||||
|
|
||||||
start += chunkSize;
|
|
||||||
|
|
||||||
if (results.hits.length === 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = results.hits
|
|
||||||
.filter((key: MeilisearchNote) => {
|
|
||||||
if (ps.userId && key.userId !== ps.userId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ps.channelId && key.channelId !== ps.channelId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ps.sinceId && key.id <= ps.sinceId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ps.untilId && key.id >= ps.untilId) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
})
|
|
||||||
.map((key) => key.id);
|
|
||||||
|
|
||||||
ids.push(...res);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort all the results by note id DESC (newest first)
|
|
||||||
ids.sort((a, b) => b - a);
|
|
||||||
|
|
||||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
|
||||||
start = 0;
|
|
||||||
const found = [];
|
|
||||||
while (found.length < ps.limit && start < ids.length) {
|
|
||||||
const chunk = ids.slice(start, start + chunkSize);
|
|
||||||
const notes: Note[] = await Notes.find({
|
|
||||||
where: {
|
|
||||||
id: In(chunk),
|
|
||||||
},
|
|
||||||
order: {
|
|
||||||
id: "DESC",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// The notes are checked for visibility and muted/blocked users when packed
|
|
||||||
found.push(...(await Notes.packMany(notes, me)));
|
|
||||||
start += chunkSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we have more results than the limit, trim them
|
|
||||||
if (found.length > ps.limit) {
|
|
||||||
found.length = ps.limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
return found;
|
|
||||||
} else {
|
|
||||||
const userQuery =
|
|
||||||
ps.userId != null
|
|
||||||
? [
|
|
||||||
{
|
|
||||||
term: {
|
|
||||||
userId: ps.userId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const hostQuery =
|
|
||||||
ps.userId == null
|
|
||||||
? ps.host === null
|
|
||||||
? [
|
|
||||||
{
|
|
||||||
bool: {
|
|
||||||
must_not: {
|
|
||||||
exists: {
|
|
||||||
field: "userHost",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
: ps.host !== undefined
|
|
||||||
? [
|
|
||||||
{
|
|
||||||
term: {
|
|
||||||
userHost: ps.host,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
: []
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const result = await es.search({
|
|
||||||
index: config.elasticsearch.index || "misskey_note",
|
|
||||||
body: {
|
|
||||||
size: ps.limit,
|
|
||||||
from: ps.offset,
|
|
||||||
query: {
|
|
||||||
bool: {
|
|
||||||
must: [
|
|
||||||
{
|
|
||||||
simple_query_string: {
|
|
||||||
fields: ["text"],
|
|
||||||
query: ps.query.toLowerCase(),
|
|
||||||
default_operator: "and",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
...hostQuery,
|
|
||||||
...userQuery,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
|
||||||
{
|
|
||||||
_doc: "desc",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const hits = result.body.hits.hits.map((hit: any) => hit._id);
|
|
||||||
|
|
||||||
if (hits.length === 0) return [];
|
|
||||||
|
|
||||||
// Fetch found notes
|
|
||||||
const notes = await Notes.find({
|
|
||||||
where: {
|
|
||||||
id: In(hits),
|
|
||||||
},
|
|
||||||
order: {
|
|
||||||
id: -1,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return await Notes.packMany(notes, me);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
query
|
||||||
|
.andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` })
|
||||||
|
.innerJoinAndSelect("note.user", "user")
|
||||||
|
.leftJoinAndSelect("user.avatar", "avatar")
|
||||||
|
.leftJoinAndSelect("user.banner", "banner")
|
||||||
|
.leftJoinAndSelect("note.reply", "reply")
|
||||||
|
.leftJoinAndSelect("note.renote", "renote")
|
||||||
|
.leftJoinAndSelect("reply.user", "replyUser")
|
||||||
|
.leftJoinAndSelect("replyUser.avatar", "replyUserAvatar")
|
||||||
|
.leftJoinAndSelect("replyUser.banner", "replyUserBanner")
|
||||||
|
.leftJoinAndSelect("renote.user", "renoteUser")
|
||||||
|
.leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar")
|
||||||
|
.leftJoinAndSelect("renoteUser.banner", "renoteUserBanner");
|
||||||
|
|
||||||
|
generateVisibilityQuery(query, me);
|
||||||
|
if (me) generateMutedUserQuery(query, me);
|
||||||
|
if (me) generateBlockedUserQuery(query, me);
|
||||||
|
|
||||||
|
const notes: Note[] = await query.take(ps.limit).getMany();
|
||||||
|
|
||||||
|
return await Notes.packMany(notes, me);
|
||||||
});
|
});
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
import * as os from "node:os";
|
import * as os from "node:os";
|
||||||
import si from "systeminformation";
|
import si from "systeminformation";
|
||||||
import define from "../define.js";
|
import define from "../define.js";
|
||||||
import meilisearch from "@/db/meilisearch.js";
|
|
||||||
import { fetchMeta } from "@/misc/fetch-meta.js";
|
import { fetchMeta } from "@/misc/fetch-meta.js";
|
||||||
|
|
||||||
export const meta = {
|
export const meta = {
|
||||||
@ -63,15 +62,3 @@ export default define(meta, paramDef, async () => {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
async function meilisearchStatus() {
|
|
||||||
if (meilisearch) {
|
|
||||||
return meilisearch.serverStats();
|
|
||||||
} else {
|
|
||||||
return {
|
|
||||||
health: "unconfigured",
|
|
||||||
size: 0,
|
|
||||||
indexed_count: 0,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,6 +1,3 @@
|
|||||||
import es from "@/db/elasticsearch.js";
|
|
||||||
import sonic from "@/db/sonic.js";
|
|
||||||
import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js";
|
|
||||||
import { Followings, Hashtags, Notes, Users } from "@/models/index.js";
|
import { Followings, Hashtags, Notes, Users } from "@/models/index.js";
|
||||||
import { sqlLikeEscape } from "@/misc/sql-like-escape.js";
|
import { sqlLikeEscape } from "@/misc/sql-like-escape.js";
|
||||||
import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js";
|
import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js";
|
||||||
@ -9,7 +6,7 @@ import { generateBlockedUserQuery } from "@/server/api/common/generate-block-que
|
|||||||
import { Note } from "@/models/entities/note.js";
|
import { Note } from "@/models/entities/note.js";
|
||||||
import { PaginationHelpers } from "@/server/api/mastodon/helpers/pagination.js";
|
import { PaginationHelpers } from "@/server/api/mastodon/helpers/pagination.js";
|
||||||
import { ILocalUser, User } from "@/models/entities/user.js";
|
import { ILocalUser, User } from "@/models/entities/user.js";
|
||||||
import { Brackets, In, IsNull } from "typeorm";
|
import { Brackets, IsNull } from "typeorm";
|
||||||
import { awaitAll } from "@/prelude/await-all.js";
|
import { awaitAll } from "@/prelude/await-all.js";
|
||||||
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
|
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
|
||||||
import Resolver from "@/remote/activitypub/resolver.js";
|
import Resolver from "@/remote/activitypub/resolver.js";
|
||||||
@ -19,7 +16,6 @@ import { createPerson } from "@/remote/activitypub/models/person.js";
|
|||||||
import { UserConverter } from "@/server/api/mastodon/converters/user.js";
|
import { UserConverter } from "@/server/api/mastodon/converters/user.js";
|
||||||
import { resolveUser } from "@/remote/resolve-user.js";
|
import { resolveUser } from "@/remote/resolve-user.js";
|
||||||
import { createNote } from "@/remote/activitypub/models/note.js";
|
import { createNote } from "@/remote/activitypub/models/note.js";
|
||||||
import { getUser } from "@/server/api/common/getters.js";
|
|
||||||
import config from "@/config/index.js";
|
import config from "@/config/index.js";
|
||||||
import { logger, MastoContext } from "@/server/api/mastodon/index.js";
|
import { logger, MastoContext } from "@/server/api/mastodon/index.js";
|
||||||
|
|
||||||
@ -144,216 +140,6 @@ export class SearchHelpers {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try sonic search first, unless we have advanced filters
|
|
||||||
if (sonic && !accountId && !following) {
|
|
||||||
let start = offset ?? 0;
|
|
||||||
const chunkSize = 100;
|
|
||||||
|
|
||||||
// Use sonic to fetch and step through all search results that could match the requirements
|
|
||||||
const ids = [];
|
|
||||||
while (true) {
|
|
||||||
const results = await sonic.search.query(
|
|
||||||
sonic.collection,
|
|
||||||
sonic.bucket,
|
|
||||||
q,
|
|
||||||
{
|
|
||||||
limit: chunkSize,
|
|
||||||
offset: start,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
start += chunkSize;
|
|
||||||
|
|
||||||
if (results.length === 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = results
|
|
||||||
.map((k) => JSON.parse(k))
|
|
||||||
.filter((key) => {
|
|
||||||
if (minId && key.id < minId) return false;
|
|
||||||
if (maxId && key.id > maxId) return false;
|
|
||||||
return true;
|
|
||||||
})
|
|
||||||
.map((key) => key.id);
|
|
||||||
|
|
||||||
ids.push(...res);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort all the results by note id DESC (newest first)
|
|
||||||
ids.sort((a, b) => b - a);
|
|
||||||
|
|
||||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
|
||||||
start = 0;
|
|
||||||
const found = [];
|
|
||||||
while (found.length < limit && start < ids.length) {
|
|
||||||
const chunk = ids.slice(start, start + chunkSize);
|
|
||||||
|
|
||||||
const query = Notes.createQueryBuilder("note")
|
|
||||||
.where({ id: In(chunk) })
|
|
||||||
.orderBy({ id: "DESC" })
|
|
||||||
|
|
||||||
generateVisibilityQuery(query, user);
|
|
||||||
|
|
||||||
if (!accountId) {
|
|
||||||
generateMutedUserQuery(query, user);
|
|
||||||
generateBlockedUserQuery(query, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (following) {
|
|
||||||
const followingQuery = Followings.createQueryBuilder("following")
|
|
||||||
.select("following.followeeId")
|
|
||||||
.where("following.followerId = :followerId", { followerId: user.id });
|
|
||||||
|
|
||||||
query.andWhere(
|
|
||||||
new Brackets((qb) => {
|
|
||||||
qb.where(`note.userId IN (${followingQuery.getQuery()} UNION ALL VALUES (:meId))`, { meId: user.id });
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const notes: Note[] = await query.getMany();
|
|
||||||
|
|
||||||
found.push(...notes);
|
|
||||||
start += chunkSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we have more results than the limit, trim them
|
|
||||||
if (found.length > limit) {
|
|
||||||
found.length = limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
// Try meilisearch next
|
|
||||||
else if (meilisearch) {
|
|
||||||
let start = 0;
|
|
||||||
const chunkSize = 100;
|
|
||||||
|
|
||||||
// Use meilisearch to fetch and step through all search results that could match the requirements
|
|
||||||
const ids = [];
|
|
||||||
if (accountId) {
|
|
||||||
const acc = await getUser(accountId);
|
|
||||||
const append = acc.host !== null ? `from:${acc.usernameLower}@${acc.host} ` : `from:${acc.usernameLower}`;
|
|
||||||
q = append + q;
|
|
||||||
}
|
|
||||||
if (following) {
|
|
||||||
q = `filter:following ${q}`;
|
|
||||||
}
|
|
||||||
while (true) {
|
|
||||||
const results = await meilisearch.search(q, chunkSize, start, user);
|
|
||||||
|
|
||||||
start += chunkSize;
|
|
||||||
|
|
||||||
if (results.hits.length === 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
//TODO test this, it's the same logic the mk api uses but it seems, we need to make .hits already be a MeilisearchNote[] instead of forcing type checks to pass
|
|
||||||
const res = (results.hits as MeilisearchNote[])
|
|
||||||
.filter((key: MeilisearchNote) => {
|
|
||||||
if (accountId && key.userId !== accountId) return false;
|
|
||||||
if (minId && key.id < minId) return false;
|
|
||||||
if (maxId && key.id > maxId) return false;
|
|
||||||
return true;
|
|
||||||
})
|
|
||||||
.map((key) => key.id);
|
|
||||||
|
|
||||||
ids.push(...res);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort all the results by note id DESC (newest first)
|
|
||||||
//FIXME: fix this sort function (is it even necessary?)
|
|
||||||
//ids.sort((a, b) => b - a);
|
|
||||||
|
|
||||||
// Fetch the notes from the database until we have enough to satisfy the limit
|
|
||||||
start = 0;
|
|
||||||
const found = [];
|
|
||||||
while (found.length < limit && start < ids.length) {
|
|
||||||
const chunk = ids.slice(start, start + chunkSize);
|
|
||||||
|
|
||||||
const query = Notes.createQueryBuilder("note")
|
|
||||||
.where({ id: In(chunk) })
|
|
||||||
.orderBy({ id: "DESC" })
|
|
||||||
|
|
||||||
generateVisibilityQuery(query, user);
|
|
||||||
|
|
||||||
if (!accountId) {
|
|
||||||
generateMutedUserQuery(query, user);
|
|
||||||
generateBlockedUserQuery(query, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
const notes: Note[] = await query.getMany();
|
|
||||||
|
|
||||||
found.push(...notes);
|
|
||||||
start += chunkSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we have more results than the limit, trim them
|
|
||||||
if (found.length > limit) {
|
|
||||||
found.length = limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
return found;
|
|
||||||
} else if (es) {
|
|
||||||
const userQuery =
|
|
||||||
accountId != null
|
|
||||||
? [
|
|
||||||
{
|
|
||||||
term: {
|
|
||||||
userId: accountId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const result = await es.search({
|
|
||||||
index: config.elasticsearch.index || "misskey_note",
|
|
||||||
body: {
|
|
||||||
size: limit,
|
|
||||||
from: offset,
|
|
||||||
query: {
|
|
||||||
bool: {
|
|
||||||
must: [
|
|
||||||
{
|
|
||||||
simple_query_string: {
|
|
||||||
fields: ["text"],
|
|
||||||
query: q.toLowerCase(),
|
|
||||||
default_operator: "and",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
...userQuery,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
|
||||||
{
|
|
||||||
_doc: "desc",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const hits = result.body.hits.hits.map((hit: any) => hit._id);
|
|
||||||
|
|
||||||
if (hits.length === 0) return [];
|
|
||||||
|
|
||||||
// Fetch found notes
|
|
||||||
const notes = await Notes.find({
|
|
||||||
where: {
|
|
||||||
id: In(hits),
|
|
||||||
},
|
|
||||||
order: {
|
|
||||||
id: -1,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
//TODO: test this
|
|
||||||
//FIXME: implement pagination
|
|
||||||
return notes;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to database query
|
|
||||||
const query = PaginationHelpers.makePaginationQuery(
|
const query = PaginationHelpers.makePaginationQuery(
|
||||||
Notes.createQueryBuilder("note"),
|
Notes.createQueryBuilder("note"),
|
||||||
undefined,
|
undefined,
|
||||||
|
|||||||
@ -78,7 +78,7 @@ const nodeinfo2 = async () => {
|
|||||||
disableRecommendedTimeline: meta.disableRecommendedTimeline,
|
disableRecommendedTimeline: meta.disableRecommendedTimeline,
|
||||||
disableGlobalTimeline: meta.disableGlobalTimeline,
|
disableGlobalTimeline: meta.disableGlobalTimeline,
|
||||||
emailRequiredForSignup: meta.emailRequiredForSignup,
|
emailRequiredForSignup: meta.emailRequiredForSignup,
|
||||||
searchFilters: config.meilisearch ? true : false,
|
searchFilters: true,
|
||||||
postEditing: true,
|
postEditing: true,
|
||||||
postImports: meta.experimentalFeatures?.postImports || false,
|
postImports: meta.experimentalFeatures?.postImports || false,
|
||||||
enableHcaptcha: meta.enableHcaptcha,
|
enableHcaptcha: meta.enableHcaptcha,
|
||||||
|
|||||||
@ -1,6 +1,4 @@
|
|||||||
import * as mfm from "mfm-js";
|
import * as mfm from "mfm-js";
|
||||||
import es from "../../db/elasticsearch.js";
|
|
||||||
import sonic from "../../db/sonic.js";
|
|
||||||
import {
|
import {
|
||||||
publishMainStream,
|
publishMainStream,
|
||||||
publishNotesStream,
|
publishNotesStream,
|
||||||
@ -64,7 +62,6 @@ import type { UserProfile } from "@/models/entities/user-profile.js";
|
|||||||
import { db } from "@/db/postgre.js";
|
import { db } from "@/db/postgre.js";
|
||||||
import { getActiveWebhooks } from "@/misc/webhook-cache.js";
|
import { getActiveWebhooks } from "@/misc/webhook-cache.js";
|
||||||
import { shouldSilenceInstance } from "@/misc/should-block-instance.js";
|
import { shouldSilenceInstance } from "@/misc/should-block-instance.js";
|
||||||
import meilisearch from "../../db/meilisearch.js";
|
|
||||||
import { redisClient } from "@/db/redis.js";
|
import { redisClient } from "@/db/redis.js";
|
||||||
import { Mutex } from "redis-semaphore";
|
import { Mutex } from "redis-semaphore";
|
||||||
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
|
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
|
||||||
@ -655,9 +652,6 @@ export default async (
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Register to search database
|
|
||||||
await index(note, false);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
async function renderNoteOrRenoteActivity(data: Option, note: Note) {
|
async function renderNoteOrRenoteActivity(data: Option, note: Note) {
|
||||||
@ -803,40 +797,6 @@ async function insertNote(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function index(note: Note, reindexing: boolean): Promise<void> {
|
|
||||||
if (!note.text) return;
|
|
||||||
|
|
||||||
if (config.elasticsearch && es) {
|
|
||||||
es.index({
|
|
||||||
index: config.elasticsearch.index || "misskey_note",
|
|
||||||
id: note.id.toString(),
|
|
||||||
body: {
|
|
||||||
text: normalizeForSearch(note.text),
|
|
||||||
userId: note.userId,
|
|
||||||
userHost: note.userHost,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sonic) {
|
|
||||||
await sonic.ingest.push(
|
|
||||||
sonic.collection,
|
|
||||||
sonic.bucket,
|
|
||||||
JSON.stringify({
|
|
||||||
id: note.id,
|
|
||||||
userId: note.userId,
|
|
||||||
userHost: note.userHost,
|
|
||||||
channelId: note.channelId,
|
|
||||||
}),
|
|
||||||
note.text,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (meilisearch && !reindexing) {
|
|
||||||
await meilisearch.ingestNote(note);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function notifyToWatchersOfRenotee(
|
async function notifyToWatchersOfRenotee(
|
||||||
renote: Note,
|
renote: Note,
|
||||||
user: { id: User["id"] },
|
user: { id: User["id"] },
|
||||||
|
|||||||
@ -21,7 +21,6 @@ import {
|
|||||||
import { countSameRenotes } from "@/misc/count-same-renotes.js";
|
import { countSameRenotes } from "@/misc/count-same-renotes.js";
|
||||||
import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
|
import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
|
||||||
import { deliverToRelays } from "../relay.js";
|
import { deliverToRelays } from "../relay.js";
|
||||||
import meilisearch from "@/db/meilisearch.js";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 投稿を削除します。
|
* 投稿を削除します。
|
||||||
@ -123,10 +122,6 @@ export default async function (
|
|||||||
id: note.id,
|
id: note.id,
|
||||||
userId: user.id,
|
userId: user.id,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (meilisearch) {
|
|
||||||
await meilisearch.deleteNotes(note.id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function findCascadingNotes(note: Note) {
|
async function findCascadingNotes(note: Note) {
|
||||||
|
|||||||
@ -24,7 +24,7 @@ import { genId } from "@/misc/gen-id.js";
|
|||||||
import type { IPoll } from "@/models/entities/poll.js";
|
import type { IPoll } from "@/models/entities/poll.js";
|
||||||
import { deliverToRelays } from "../relay.js";
|
import { deliverToRelays } from "../relay.js";
|
||||||
import renderUpdate from "@/remote/activitypub/renderer/update.js";
|
import renderUpdate from "@/remote/activitypub/renderer/update.js";
|
||||||
import { extractMentionedUsers, index } from "@/services/note/create.js";
|
import { extractMentionedUsers } from "@/services/note/create.js";
|
||||||
import { normalizeForSearch } from "@/misc/normalize-for-search.js";
|
import { normalizeForSearch } from "@/misc/normalize-for-search.js";
|
||||||
|
|
||||||
type Option = {
|
type Option = {
|
||||||
@ -182,8 +182,6 @@ export default async function (
|
|||||||
note = await Notes.findOneByOrFail({ id: note.id });
|
note = await Notes.findOneByOrFail({ id: note.id });
|
||||||
|
|
||||||
if (publishing) {
|
if (publishing) {
|
||||||
index(note, true);
|
|
||||||
|
|
||||||
// Publish update event for the updated note details
|
// Publish update event for the updated note details
|
||||||
publishNoteStream(note.id, "updated", {
|
publishNoteStream(note.id, "updated", {
|
||||||
updatedAt: update.updatedAt,
|
updatedAt: update.updatedAt,
|
||||||
|
|||||||
@ -29,26 +29,6 @@
|
|||||||
<p>Used: {{ bytes(diskUsed, 1) }}</p>
|
<p>Used: {{ bytes(diskUsed, 1) }}</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="_panel">
|
|
||||||
<XPie class="pie" :value="meiliProgress" />
|
|
||||||
<div>
|
|
||||||
<p>
|
|
||||||
<i class="ph-file-search ph-bold ph-lg"></i>MeiliSearch
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
{{ i18n.ts._widgets.meiliStatus }}: {{ meiliAvailable }}
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
{{ i18n.ts._widgets.meiliSize }}:
|
|
||||||
{{ bytes(meiliTotalSize, 1) }}
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
{{ i18n.ts._widgets.meiliIndexCount }}:
|
|
||||||
{{ meiliIndexCount }}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@ -71,11 +51,6 @@ let memTotal: number = $ref(0);
|
|||||||
let memUsed: number = $ref(0);
|
let memUsed: number = $ref(0);
|
||||||
let memFree: number = $ref(0);
|
let memFree: number = $ref(0);
|
||||||
|
|
||||||
let meiliProgress: number = $ref(0);
|
|
||||||
let meiliTotalSize: number = $ref(0);
|
|
||||||
let meiliIndexCount: number = $ref(0);
|
|
||||||
let meiliAvailable: string = $ref("unavailable");
|
|
||||||
|
|
||||||
const diskUsage = $computed(() => meta.fs.used / meta.fs.total);
|
const diskUsage = $computed(() => meta.fs.used / meta.fs.total);
|
||||||
const diskTotal = $computed(() => meta.fs.total);
|
const diskTotal = $computed(() => meta.fs.total);
|
||||||
const diskUsed = $computed(() => meta.fs.used);
|
const diskUsed = $computed(() => meta.fs.used);
|
||||||
@ -88,11 +63,6 @@ function onStats(stats) {
|
|||||||
memTotal = stats.mem.total;
|
memTotal = stats.mem.total;
|
||||||
memUsed = stats.mem.active;
|
memUsed = stats.mem.active;
|
||||||
memFree = memTotal - memUsed;
|
memFree = memTotal - memUsed;
|
||||||
|
|
||||||
meiliTotalSize = stats.meilisearch.size;
|
|
||||||
meiliIndexCount = stats.meilisearch.indexed_count;
|
|
||||||
meiliAvailable = stats.meilisearch.health;
|
|
||||||
meiliProgress = meiliIndexCount / serverStats.notesCount;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const connection = stream.useChannel("serverStats");
|
const connection = stream.useChannel("serverStats");
|
||||||
|
|||||||
@ -38,13 +38,6 @@
|
|||||||
:connection="connection"
|
:connection="connection"
|
||||||
:meta="meta"
|
:meta="meta"
|
||||||
/>
|
/>
|
||||||
<XMeili
|
|
||||||
v-else-if="
|
|
||||||
instance.features.searchFilters && widgetProps.view === 5
|
|
||||||
"
|
|
||||||
:connection="connection"
|
|
||||||
:meta="meta"
|
|
||||||
/>
|
|
||||||
</div>
|
</div>
|
||||||
</MkContainer>
|
</MkContainer>
|
||||||
</template>
|
</template>
|
||||||
@ -62,7 +55,6 @@ import XNet from "./net.vue";
|
|||||||
import XCpu from "./cpu.vue";
|
import XCpu from "./cpu.vue";
|
||||||
import XMemory from "./mem.vue";
|
import XMemory from "./mem.vue";
|
||||||
import XDisk from "./disk.vue";
|
import XDisk from "./disk.vue";
|
||||||
import XMeili from "./meilisearch.vue";
|
|
||||||
import MkContainer from "@/components/MkContainer.vue";
|
import MkContainer from "@/components/MkContainer.vue";
|
||||||
import type { GetFormResultType } from "@/scripts/form";
|
import type { GetFormResultType } from "@/scripts/form";
|
||||||
import * as os from "@/os";
|
import * as os from "@/os";
|
||||||
|
|||||||
@ -1,85 +0,0 @@
|
|||||||
<template>
|
|
||||||
<div class="verusivbr">
|
|
||||||
<XPie
|
|
||||||
v-tooltip="i18n.ts.meiliIndexCount"
|
|
||||||
class="pie"
|
|
||||||
:value="progress"
|
|
||||||
:reverse="true"
|
|
||||||
/>
|
|
||||||
<div>
|
|
||||||
<p><i class="ph-file-search ph-bold ph-lg"></i>MeiliSearch</p>
|
|
||||||
<p>{{ i18n.ts._widgets.meiliStatus }}: {{ available }}</p>
|
|
||||||
<p>{{ i18n.ts._widgets.meiliSize }}: {{ bytes(totalSize, 1) }}</p>
|
|
||||||
<p>{{ i18n.ts._widgets.meiliIndexCount }}: {{ indexCount }}</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<br />
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script lang="ts" setup>
|
|
||||||
import { onBeforeUnmount, onMounted } from "vue";
|
|
||||||
import XPie from "./pie.vue";
|
|
||||||
import bytes from "@/filters/bytes";
|
|
||||||
import { i18n } from "@/i18n";
|
|
||||||
import * as os from "@/os";
|
|
||||||
|
|
||||||
const props = defineProps<{
|
|
||||||
connection: any;
|
|
||||||
meta: any;
|
|
||||||
}>();
|
|
||||||
|
|
||||||
let progress: number = $ref(0),
|
|
||||||
serverStats = $ref(null),
|
|
||||||
totalSize: number = $ref(0),
|
|
||||||
indexCount: number = $ref(0),
|
|
||||||
available: string = $ref("unavailable");
|
|
||||||
|
|
||||||
function onStats(stats) {
|
|
||||||
totalSize = stats.meilisearch.size;
|
|
||||||
indexCount = stats.meilisearch.indexed_count;
|
|
||||||
available = stats.meilisearch.health;
|
|
||||||
progress = indexCount / serverStats.notesCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
onMounted(() => {
|
|
||||||
os.api("stats", {}).then((res) => {
|
|
||||||
serverStats = res;
|
|
||||||
});
|
|
||||||
props.connection.on("stats", onStats);
|
|
||||||
});
|
|
||||||
|
|
||||||
onBeforeUnmount(() => {
|
|
||||||
props.connection.off("stats", onStats);
|
|
||||||
});
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style lang="scss" scoped>
|
|
||||||
.verusivbr {
|
|
||||||
display: flex;
|
|
||||||
padding: 16px;
|
|
||||||
|
|
||||||
> .pie {
|
|
||||||
height: 82px;
|
|
||||||
flex-shrink: 0;
|
|
||||||
margin-right: 16px;
|
|
||||||
}
|
|
||||||
|
|
||||||
> div {
|
|
||||||
flex: 1;
|
|
||||||
|
|
||||||
> p {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.8em;
|
|
||||||
|
|
||||||
&:first-child {
|
|
||||||
font-weight: bold;
|
|
||||||
margin-bottom: 4px;
|
|
||||||
|
|
||||||
> i {
|
|
||||||
margin-right: 4px;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
46
yarn.lock
46
yarn.lock
@ -983,18 +983,6 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"@elastic/elasticsearch@npm:7.17.0":
|
|
||||||
version: 7.17.0
|
|
||||||
resolution: "@elastic/elasticsearch@npm:7.17.0"
|
|
||||||
dependencies:
|
|
||||||
debug: "npm:^4.3.1"
|
|
||||||
hpagent: "npm:^0.1.1"
|
|
||||||
ms: "npm:^2.1.3"
|
|
||||||
secure-json-parse: "npm:^2.4.0"
|
|
||||||
checksum: d54330ce50b4951b7b9db15349413b4961040fb0b73a09d3f07cef5cb2873fd22af17307e07b6c8b1b1e0844e76e9aeb78ce1e01d67a940e3190763a875648be
|
|
||||||
languageName: node
|
|
||||||
linkType: hard
|
|
||||||
|
|
||||||
"@es-joy/jsdoccomment@npm:~0.39.4":
|
"@es-joy/jsdoccomment@npm:~0.39.4":
|
||||||
version: 0.39.4
|
version: 0.39.4
|
||||||
resolution: "@es-joy/jsdoccomment@npm:0.39.4"
|
resolution: "@es-joy/jsdoccomment@npm:0.39.4"
|
||||||
@ -5308,7 +5296,6 @@ __metadata:
|
|||||||
"@bull-board/koa": "npm:5.6.0"
|
"@bull-board/koa": "npm:5.6.0"
|
||||||
"@bull-board/ui": "npm:5.6.0"
|
"@bull-board/ui": "npm:5.6.0"
|
||||||
"@discordapp/twemoji": "npm:14.1.2"
|
"@discordapp/twemoji": "npm:14.1.2"
|
||||||
"@elastic/elasticsearch": "npm:7.17.0"
|
|
||||||
"@koa/cors": "npm:3.4.3"
|
"@koa/cors": "npm:3.4.3"
|
||||||
"@koa/multer": "npm:3.0.2"
|
"@koa/multer": "npm:3.0.2"
|
||||||
"@koa/router": "npm:9.0.1"
|
"@koa/router": "npm:9.0.1"
|
||||||
@ -5422,7 +5409,6 @@ __metadata:
|
|||||||
koa-send: "npm:5.0.1"
|
koa-send: "npm:5.0.1"
|
||||||
koa-slow: "npm:2.1.0"
|
koa-slow: "npm:2.1.0"
|
||||||
koa-views: "npm:7.0.2"
|
koa-views: "npm:7.0.2"
|
||||||
meilisearch: "npm:0.33.0"
|
|
||||||
mfm-js: "npm:0.23.3"
|
mfm-js: "npm:0.23.3"
|
||||||
mime-types: "npm:2.1.35"
|
mime-types: "npm:2.1.35"
|
||||||
mocha: "npm:10.2.0"
|
mocha: "npm:10.2.0"
|
||||||
@ -5457,7 +5443,6 @@ __metadata:
|
|||||||
seedrandom: "npm:^3.0.5"
|
seedrandom: "npm:^3.0.5"
|
||||||
semver: "npm:7.5.4"
|
semver: "npm:7.5.4"
|
||||||
sharp: "npm:0.32.1"
|
sharp: "npm:0.32.1"
|
||||||
sonic-channel: "npm:^1.3.1"
|
|
||||||
strict-event-emitter-types: "npm:2.0.0"
|
strict-event-emitter-types: "npm:2.0.0"
|
||||||
stringz: "npm:2.1.0"
|
stringz: "npm:2.1.0"
|
||||||
summaly: "npm:2.7.0"
|
summaly: "npm:2.7.0"
|
||||||
@ -7227,7 +7212,7 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"cross-fetch@npm:^3.0.4, cross-fetch@npm:^3.1.6":
|
"cross-fetch@npm:^3.0.4":
|
||||||
version: 3.1.8
|
version: 3.1.8
|
||||||
resolution: "cross-fetch@npm:3.1.8"
|
resolution: "cross-fetch@npm:3.1.8"
|
||||||
dependencies:
|
dependencies:
|
||||||
@ -7539,7 +7524,7 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"debug@npm:4, debug@npm:4.3.4, debug@npm:^4.0.0, debug@npm:^4.0.1, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4":
|
"debug@npm:4, debug@npm:4.3.4, debug@npm:^4.0.0, debug@npm:^4.0.1, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4":
|
||||||
version: 4.3.4
|
version: 4.3.4
|
||||||
resolution: "debug@npm:4.3.4"
|
resolution: "debug@npm:4.3.4"
|
||||||
dependencies:
|
dependencies:
|
||||||
@ -10989,7 +10974,7 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"hpagent@npm:0.1.2, hpagent@npm:^0.1.1":
|
"hpagent@npm:0.1.2":
|
||||||
version: 0.1.2
|
version: 0.1.2
|
||||||
resolution: "hpagent@npm:0.1.2"
|
resolution: "hpagent@npm:0.1.2"
|
||||||
checksum: bd033b3700bb523edc9a805f8683c71fddd622df901e73842b5e3357136ce062c2ddb2ab5e9f5b3d84e0977bfe439f5cdc51d755a11e99376eb95e4624312f0a
|
checksum: bd033b3700bb523edc9a805f8683c71fddd622df901e73842b5e3357136ce062c2ddb2ab5e9f5b3d84e0977bfe439f5cdc51d755a11e99376eb95e4624312f0a
|
||||||
@ -14397,15 +14382,6 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"meilisearch@npm:0.33.0":
|
|
||||||
version: 0.33.0
|
|
||||||
resolution: "meilisearch@npm:0.33.0"
|
|
||||||
dependencies:
|
|
||||||
cross-fetch: "npm:^3.1.6"
|
|
||||||
checksum: d2aff57b3d5f7eea8befe1c404b9afc12f72526836e99fca79ab03cf95fb163a01bc5014f610d5b6631c4a6339a4272da41e866208b2af92074b658bee53c645
|
|
||||||
languageName: node
|
|
||||||
linkType: hard
|
|
||||||
|
|
||||||
"meow@npm:^9.0.0":
|
"meow@npm:^9.0.0":
|
||||||
version: 9.0.0
|
version: 9.0.0
|
||||||
resolution: "meow@npm:9.0.0"
|
resolution: "meow@npm:9.0.0"
|
||||||
@ -14831,7 +14807,7 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1, ms@npm:^2.1.3":
|
"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1":
|
||||||
version: 2.1.3
|
version: 2.1.3
|
||||||
resolution: "ms@npm:2.1.3"
|
resolution: "ms@npm:2.1.3"
|
||||||
checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d
|
checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d
|
||||||
@ -18253,13 +18229,6 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"secure-json-parse@npm:^2.4.0":
|
|
||||||
version: 2.7.0
|
|
||||||
resolution: "secure-json-parse@npm:2.7.0"
|
|
||||||
checksum: 974386587060b6fc5b1ac06481b2f9dbbb0d63c860cc73dc7533f27835fdb67b0ef08762dbfef25625c15bc0a0c366899e00076cb0d556af06b71e22f1dede4c
|
|
||||||
languageName: node
|
|
||||||
linkType: hard
|
|
||||||
|
|
||||||
"seedrandom@npm:2.4.2":
|
"seedrandom@npm:2.4.2":
|
||||||
version: 2.4.2
|
version: 2.4.2
|
||||||
resolution: "seedrandom@npm:2.4.2"
|
resolution: "seedrandom@npm:2.4.2"
|
||||||
@ -18641,13 +18610,6 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"sonic-channel@npm:^1.3.1":
|
|
||||||
version: 1.3.1
|
|
||||||
resolution: "sonic-channel@npm:1.3.1"
|
|
||||||
checksum: ee849863a378d5cc631d87c1d184f697979c766edc30159dc2fe28cc5741dbf8caa587f4baabb5d0bee301829e15c629f36e8d40c75dcb22c705d98ffdc23731
|
|
||||||
languageName: node
|
|
||||||
linkType: hard
|
|
||||||
|
|
||||||
"sort-keys-length@npm:^1.0.0":
|
"sort-keys-length@npm:^1.0.0":
|
||||||
version: 1.0.1
|
version: 1.0.1
|
||||||
resolution: "sort-keys-length@npm:1.0.1"
|
resolution: "sort-keys-length@npm:1.0.1"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user