From 390c0c2839ce176e0de1a3e550108d838d6ccbd5 Mon Sep 17 00:00:00 2001 From: Syuilo Date: Sat, 8 Jul 2023 15:39:42 -0700 Subject: [PATCH 01/15] =?UTF-8?q?refactor:=20=E2=9A=A1=20antenna=20notes?= =?UTF-8?q?=20in=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kainoa Kanter --- .../migration/1680491187535-cleanup.js | 9 ++ packages/backend/src/models/index.ts | 1 - .../backend/src/models/repositories/user.ts | 32 +++---- .../server/api/endpoints/antennas/markread.ts | 20 ++--- .../server/api/endpoints/antennas/notes.ts | 31 +++++-- .../src/services/add-note-to-antenna.ts | 56 +++--------- packages/backend/src/services/note/read.ts | 87 +++++++++---------- 7 files changed, 113 insertions(+), 123 deletions(-) create mode 100644 packages/backend/migration/1680491187535-cleanup.js diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js new file mode 100644 index 000000000..021a368c0 --- /dev/null +++ b/packages/backend/migration/1680491187535-cleanup.js @@ -0,0 +1,9 @@ +export class cleanup1680491187535 { + name = "cleanup1680491187535"; + + async up(queryRunner) { + await queryRunner.query(`DROP TABLE "antenna_note" `); + } + + async down(queryRunner) {} +} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index cfc3b01c5..ca35b7708 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -123,7 +123,6 @@ export const ModerationLogs = ModerationLogRepository; export const Clips = ClipRepository; export const ClipNotes = db.getRepository(ClipNote); export const Antennas = AntennaRepository; -export const AntennaNotes = db.getRepository(AntennaNote); export const PromoNotes = db.getRepository(PromoNote); export const PromoReads = db.getRepository(PromoRead); export const Relays = RelayRepository; diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index 5ca36e3d3..1f9cd80fb 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -18,7 +18,6 @@ import { createPerson } from "@/remote/activitypub/models/person.js"; import { AnnouncementReads, Announcements, - AntennaNotes, Blockings, ChannelFollowings, DriveFiles, @@ -258,23 +257,24 @@ export const UserRepository = db.getRepository(User).extend({ }, async getHasUnreadAntenna(userId: User["id"]): Promise { - try { - const myAntennas = (await getAntennas()).filter( - (a) => a.userId === userId, - ); + // try { + // const myAntennas = (await getAntennas()).filter( + // (a) => a.userId === userId, + // ); - const unread = - myAntennas.length > 0 - ? await AntennaNotes.findOneBy({ - antennaId: In(myAntennas.map((x) => x.id)), - read: false, - }) - : null; + // const unread = + // myAntennas.length > 0 + // ? await AntennaNotes.findOneBy({ + // antennaId: In(myAntennas.map((x) => x.id)), + // read: false, + // }) + // : null; - return unread != null; - } catch (e) { - return false; - } + // return unread != null; + // } catch (e) { + // return false; + // } + return false; // TODO }, async getHasUnreadChannel(userId: User["id"]): Promise { diff --git a/packages/backend/src/server/api/endpoints/antennas/markread.ts b/packages/backend/src/server/api/endpoints/antennas/markread.ts index e29e13bbb..fedb48c91 100644 --- a/packages/backend/src/server/api/endpoints/antennas/markread.ts +++ b/packages/backend/src/server/api/endpoints/antennas/markread.ts @@ -1,5 +1,5 @@ import define from "../../define.js"; -import { Antennas, AntennaNotes } from "@/models/index.js"; +import { Antennas } from "@/models/index.js"; import { FindOptionsWhere } from "typeorm"; import { AntennaNote } from "@/models/entities/antenna-note.js"; @@ -29,15 +29,15 @@ export default define(meta, paramDef, async (ps, me) => { return null; } - await AntennaNotes.update( - { - antennaId: antenna.id, - read: false, - }, - { - read: true, - }, - ); + // await AntennaNotes.update( + // { + // antennaId: antenna.id, + // read: false, + // }, + // { + // read: true, + // }, + // ); return true; }); diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 29e2e085a..6ca71c08e 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -1,6 +1,8 @@ import define from "../../define.js"; import readNote from "@/services/note/read.js"; -import { Antennas, Notes, AntennaNotes } from "@/models/index.js"; +import { Antennas, Notes } from "@/models/index.js"; +import { redisClient } from "@/db/redis.js"; +import { genId } from "@/misc/gen-id.js"; import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { generateVisibilityQuery } from "../../common/generate-visibility-query.js"; import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; @@ -58,6 +60,26 @@ export default define(meta, paramDef, async (ps, user) => { throw new ApiError(meta.errors.noSuchAntenna); } + const noteIdsRes = await redisClient.xrevrange( + `antennaTimeline:${antenna.id}`, + ps.untilDate || "+", + "-", + "COUNT", + ps.limit + 1, + ); // untilIdに指定したものも含まれるため+1 + + if (noteIdsRes.length === 0) { + return []; + } + + const noteIds = noteIdsRes + .map((x) => x[1][1]) + .filter((x) => x !== ps.untilId); + + if (noteIds.length === 0) { + return []; + } + const query = makePaginationQuery( Notes.createQueryBuilder("note"), ps.sinceId, @@ -65,11 +87,7 @@ export default define(meta, paramDef, async (ps, user) => { ps.sinceDate, ps.untilDate, ) - .innerJoin( - AntennaNotes.metadata.targetName, - "antennaNote", - "antennaNote.noteId = note.id", - ) + .where("note.id IN (:...noteIds)", { noteIds: noteIds }) .innerJoinAndSelect("note.user", "user") .leftJoinAndSelect("user.avatar", "avatar") .leftJoinAndSelect("user.banner", "banner") @@ -81,7 +99,6 @@ export default define(meta, paramDef, async (ps, user) => { .leftJoinAndSelect("renote.user", "renoteUser") .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") - .andWhere("antennaNote.antennaId = :antennaId", { antennaId: antenna.id }) .andWhere("note.visibility != 'home'"); generateVisibilityQuery(query, user); diff --git a/packages/backend/src/services/add-note-to-antenna.ts b/packages/backend/src/services/add-note-to-antenna.ts index 38979acb4..131c0348c 100644 --- a/packages/backend/src/services/add-note-to-antenna.ts +++ b/packages/backend/src/services/add-note-to-antenna.ts @@ -1,9 +1,8 @@ import type { Antenna } from "@/models/entities/antenna.js"; import type { Note } from "@/models/entities/note.js"; -import { AntennaNotes, Mutings, Notes } from "@/models/index.js"; import { genId } from "@/misc/gen-id.js"; -import { isUserRelated } from "@/misc/is-user-related.js"; -import { publishAntennaStream, publishMainStream } from "@/services/stream.js"; +import { redisClient } from "@/db/redis.js"; +import { publishAntennaStream } from "@/services/stream.js"; import type { User } from "@/models/entities/user.js"; export async function addNoteToAntenna( @@ -14,48 +13,15 @@ export async function addNoteToAntenna( // 通知しない設定になっているか、自分自身の投稿なら既読にする const read = !antenna.notify || antenna.userId === noteUser.id; - AntennaNotes.insert({ - id: genId(), - antennaId: antenna.id, - noteId: note.id, - read: read, - }); + redisClient.xadd( + `antennaTimeline:${antenna.id}`, + "MAXLEN", + "~", + "200", + `${genId(note.createdAt)}-*`, + "note", + note.id, + ); publishAntennaStream(antenna.id, "note", note); - - if (!read) { - const mutings = await Mutings.find({ - where: { - muterId: antenna.userId, - }, - select: ["muteeId"], - }); - - // Copy - const _note: Note = { - ...note, - }; - - if (note.replyId != null) { - _note.reply = await Notes.findOneByOrFail({ id: note.replyId }); - } - if (note.renoteId != null) { - _note.renote = await Notes.findOneByOrFail({ id: note.renoteId }); - } - - if (isUserRelated(_note, new Set(mutings.map((x) => x.muteeId)))) { - return; - } - - // 2秒経っても既読にならなかったら通知 - setTimeout(async () => { - const unread = await AntennaNotes.findOneBy({ - antennaId: antenna.id, - read: false, - }); - if (unread) { - publishMainStream(antenna.userId, "unreadAntenna", antenna); - } - }, 2000); - } } diff --git a/packages/backend/src/services/note/read.ts b/packages/backend/src/services/note/read.ts index 53188f15f..2dcf49235 100644 --- a/packages/backend/src/services/note/read.ts +++ b/packages/backend/src/services/note/read.ts @@ -3,7 +3,6 @@ import type { Note } from "@/models/entities/note.js"; import type { User } from "@/models/entities/user.js"; import { NoteUnreads, - AntennaNotes, Users, Followings, ChannelFollowings, @@ -55,7 +54,7 @@ export default async function ( const readMentions: (Note | Packed<"Note">)[] = []; const readSpecifiedNotes: (Note | Packed<"Note">)[] = []; const readChannelNotes: (Note | Packed<"Note">)[] = []; - const readAntennaNotes: (Note | Packed<"Note">)[] = []; + // const readAntennaNotes: (Note | Packed<"Note">)[] = []; for (const note of notes) { if (note.mentions?.includes(userId)) { @@ -68,22 +67,22 @@ export default async function ( readChannelNotes.push(note); } - if (note.user != null) { - // たぶんnullになることは無いはずだけど一応 - for (const antenna of myAntennas) { - if ( - await checkHitAntenna( - antenna, - note, - note.user, - undefined, - Array.from(following), - ) - ) { - readAntennaNotes.push(note); - } - } - } + // if (note.user != null) { + // // たぶんnullになることは無いはずだけど一応 + // for (const antenna of myAntennas) { + // if ( + // await checkHitAntenna( + // antenna, + // note, + // note.user, + // undefined, + // Array.from(following), + // ) + // ) { + // readAntennaNotes.push(note); + // } + // } + // } } if ( @@ -141,33 +140,33 @@ export default async function ( }); } - if (readAntennaNotes.length > 0) { - await AntennaNotes.update( - { - antennaId: In(myAntennas.map((a) => a.id)), - noteId: In(readAntennaNotes.map((n) => n.id)), - }, - { - read: true, - }, - ); + // if (readAntennaNotes.length > 0) { + // await AntennaNotes.update( + // { + // antennaId: In(myAntennas.map((a) => a.id)), + // noteId: In(readAntennaNotes.map((n) => n.id)), + // }, + // { + // read: true, + // }, + // ); - // TODO: まとめてクエリしたい - for (const antenna of myAntennas) { - const count = await AntennaNotes.countBy({ - antennaId: antenna.id, - read: false, - }); + // // TODO: まとめてクエリしたい + // for (const antenna of myAntennas) { + // const count = await AntennaNotes.countBy({ + // antennaId: antenna.id, + // read: false, + // }); - if (count === 0) { - publishMainStream(userId, "readAntenna", antenna); - } - } + // if (count === 0) { + // publishMainStream(userId, "readAntenna", antenna); + // } + // } - Users.getHasUnreadAntenna(userId).then((unread) => { - if (!unread) { - publishMainStream(userId, "readAllAntennas"); - } - }); - } + // Users.getHasUnreadAntenna(userId).then((unread) => { + // if (!unread) { + // publishMainStream(userId, "readAllAntennas"); + // } + // }); + // } } From fb56e5110cdb80960a39a9b5306eb4f16de4a0d8 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 8 Jul 2023 15:59:47 -0700 Subject: [PATCH 02/15] fix --- packages/backend/src/db/postgre.ts | 2 - .../src/models/entities/antenna-note.ts | 50 ------------------- packages/backend/src/models/index.ts | 1 - .../server/api/endpoints/antennas/markread.ts | 1 - 4 files changed, 54 deletions(-) delete mode 100644 packages/backend/src/models/entities/antenna-note.ts diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 89b7a7bf6..10ea5b15f 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -58,7 +58,6 @@ import { AnnouncementRead } from "@/models/entities/announcement-read.js"; import { Clip } from "@/models/entities/clip.js"; import { ClipNote } from "@/models/entities/clip-note.js"; import { Antenna } from "@/models/entities/antenna.js"; -import { AntennaNote } from "@/models/entities/antenna-note.js"; import { PromoNote } from "@/models/entities/promo-note.js"; import { PromoRead } from "@/models/entities/promo-read.js"; import { Relay } from "@/models/entities/relay.js"; @@ -168,7 +167,6 @@ export const entities = [ Clip, ClipNote, Antenna, - AntennaNote, PromoNote, PromoRead, Relay, diff --git a/packages/backend/src/models/entities/antenna-note.ts b/packages/backend/src/models/entities/antenna-note.ts deleted file mode 100644 index fe982c19e..000000000 --- a/packages/backend/src/models/entities/antenna-note.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { - Entity, - Index, - JoinColumn, - Column, - ManyToOne, - PrimaryColumn, -} from "typeorm"; -import { Note } from "./note.js"; -import { Antenna } from "./antenna.js"; -import { id } from "../id.js"; - -@Entity() -@Index(["noteId", "antennaId"], { unique: true }) -export class AntennaNote { - @PrimaryColumn(id()) - public id: string; - - @Index() - @Column({ - ...id(), - comment: "The note ID.", - }) - public noteId: Note["id"]; - - @ManyToOne((type) => Note, { - onDelete: "CASCADE", - }) - @JoinColumn() - public note: Note | null; - - @Index() - @Column({ - ...id(), - comment: "The antenna ID.", - }) - public antennaId: Antenna["id"]; - - @ManyToOne((type) => Antenna, { - onDelete: "CASCADE", - }) - @JoinColumn() - public antenna: Antenna | null; - - @Index() - @Column("boolean", { - default: false, - }) - public read: boolean; -} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index ca35b7708..8782d5740 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -51,7 +51,6 @@ import { UsedUsername } from "./entities/used-username.js"; import { ClipRepository } from "./repositories/clip.js"; import { ClipNote } from "./entities/clip-note.js"; import { AntennaRepository } from "./repositories/antenna.js"; -import { AntennaNote } from "./entities/antenna-note.js"; import { PromoNote } from "./entities/promo-note.js"; import { PromoRead } from "./entities/promo-read.js"; import { EmojiRepository } from "./repositories/emoji.js"; diff --git a/packages/backend/src/server/api/endpoints/antennas/markread.ts b/packages/backend/src/server/api/endpoints/antennas/markread.ts index fedb48c91..db8e683e4 100644 --- a/packages/backend/src/server/api/endpoints/antennas/markread.ts +++ b/packages/backend/src/server/api/endpoints/antennas/markread.ts @@ -1,7 +1,6 @@ import define from "../../define.js"; import { Antennas } from "@/models/index.js"; import { FindOptionsWhere } from "typeorm"; -import { AntennaNote } from "@/models/entities/antenna-note.js"; export const meta = { tags: ["antennas", "account"], From 68440395348f2e393a6aceb6054648e06776ec6f Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 8 Jul 2023 16:10:12 -0700 Subject: [PATCH 03/15] typo --- packages/backend/src/server/api/endpoints/antennas/create.ts | 2 +- packages/backend/src/services/note/read.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index c2117d51a..ed1645012 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -30,7 +30,7 @@ export const meta = { id: "c3a5a51e-04d4-11ee-be56-0242ac120002", }, noKeywords: { - message: "No keywords", + message: "No keywords.", code: "NO_KEYWORDS", id: "aa975b74-1ddb-11ee-be56-0242ac120002", }, diff --git a/packages/backend/src/services/note/read.ts b/packages/backend/src/services/note/read.ts index 2dcf49235..5e61fe95d 100644 --- a/packages/backend/src/services/note/read.ts +++ b/packages/backend/src/services/note/read.ts @@ -50,7 +50,7 @@ export default async function ( ).map((x) => x.followeeId), ); - const myAntennas = (await getAntennas()).filter((a) => a.userId === userId); + // const myAntennas = (await getAntennas()).filter((a) => a.userId === userId); const readMentions: (Note | Packed<"Note">)[] = []; const readSpecifiedNotes: (Note | Packed<"Note">)[] = []; const readChannelNotes: (Note | Packed<"Note">)[] = []; From 1fa4632411eaf4dc1249b227fdcebb79cf31ec8f Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 8 Jul 2023 16:19:55 -0700 Subject: [PATCH 04/15] fix: :bug: server stats setting, meta fetching --- packages/backend/src/daemons/server-stats.ts | 6 ++++-- packages/client/src/pages/admin/settings.vue | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index ba7427876..42e19d813 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -21,8 +21,10 @@ export default function () { ev.emit(`serverStatsLog:${x.id}`, log.slice(0, x.length || 50)); }); - const meta = fetchMeta(); - if (!meta.enableServerMachineStats) return; + fetchMeta().then((meta) => { + if (!meta.enableServerMachineStats) return; + } + ); async function tick() { const cpu = await cpuUsage(); diff --git a/packages/client/src/pages/admin/settings.vue b/packages/client/src/pages/admin/settings.vue index 5108c0696..b03a98ae5 100644 --- a/packages/client/src/pages/admin/settings.vue +++ b/packages/client/src/pages/admin/settings.vue @@ -468,6 +468,7 @@ let enableIdenticonGeneration: boolean = $ref(false); async function init() { const meta = await os.api("admin/meta"); + if (!meta) throw new Error("No meta"); name = meta.name; description = meta.description; tosUrl = meta.tosUrl; From 2edb9ab8bd00f00fa8084cab550abb144f79bd10 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 01:39:33 -0400 Subject: [PATCH 05/15] copy existing posts in antenna to redis at migration --- .../migration/1680491187535-cleanup.js | 4 +- packages/backend/native-utils/Cargo.lock | 145 ++++++++++- packages/backend/native-utils/Cargo.toml | 4 +- .../native-utils/__test__/index.spec.mjs | 8 +- .../backend/native-utils/migration/Cargo.toml | 9 +- .../backend/native-utils/migration/src/lib.rs | 2 + .../m20230709_000510_move_antenna_to_cache.rs | 240 ++++++++++++++++++ .../native-utils/migration/src/main.rs | 81 +++++- .../native-utils/src/model/schema/app.rs | 8 +- packages/backend/native-utils/src/util/id.rs | 82 +++--- packages/backend/native-utils/tests/common.rs | 8 +- .../tests/model/repository/antenna.rs | 2 +- packages/backend/src/misc/gen-id.ts | 2 +- .../src/services/add-note-to-antenna.ts | 7 +- 14 files changed, 516 insertions(+), 86 deletions(-) create mode 100644 packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js index 021a368c0..671f7521e 100644 --- a/packages/backend/migration/1680491187535-cleanup.js +++ b/packages/backend/migration/1680491187535-cleanup.js @@ -1,9 +1,7 @@ export class cleanup1680491187535 { name = "cleanup1680491187535"; - async up(queryRunner) { - await queryRunner.query(`DROP TABLE "antenna_note" `); - } + async up(queryRunner) {} async down(queryRunner) {} } diff --git a/packages/backend/native-utils/Cargo.lock b/packages/backend/native-utils/Cargo.lock index 1a20b792c..e5f8af37a 100644 --- a/packages/backend/native-utils/Cargo.lock +++ b/packages/backend/native-utils/Cargo.lock @@ -458,6 +458,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "console" version = "0.15.7" @@ -486,6 +500,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -1278,11 +1302,14 @@ dependencies = [ "futures", "indicatif", "native-utils", + "redis", + "sea-orm", "sea-orm-migration", "serde", "serde_json", "serde_yaml", "tokio", + "url", "urlencoding", ] @@ -1509,6 +1536,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -1843,6 +1876,29 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.3", + "rustls-native-certs", + "ryu", + "sha1_smol", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2043,6 +2099,30 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19faa85ecb5197342b54f987b142fb3e30d0c90da40f80ef4fa9a726e6676ed" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -2052,6 +2132,16 @@ dependencies = [ "base64 0.21.2", ] +[[package]] +name = "rustls-webpki" +version = "0.101.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -2076,6 +2166,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "schemars" version = "0.8.12" @@ -2286,6 +2385,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -2370,6 +2492,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.6" @@ -2518,7 +2646,7 @@ dependencies = [ "percent-encoding", "rand", "rust_decimal", - "rustls", + "rustls 0.20.8", "rustls-pemfile", "serde", "serde_json", @@ -2564,7 +2692,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" dependencies = [ "once_cell", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -2778,11 +2906,21 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.3", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -2949,6 +3087,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/packages/backend/native-utils/Cargo.toml b/packages/backend/native-utils/Cargo.toml index f93180fe4..6f4dd9175 100644 --- a/packages/backend/native-utils/Cargo.toml +++ b/packages/backend/native-utils/Cargo.toml @@ -9,7 +9,7 @@ members = ["migration"] [features] default = [] noarray = [] -napi = ["dep:napi", "dep:napi-derive", "dep:radix_fmt"] +napi = ["dep:napi", "dep:napi-derive"] [lib] crate-type = ["cdylib", "lib"] @@ -31,11 +31,11 @@ serde_json = "1.0.96" thiserror = "1.0.40" tokio = { version = "1.28.1", features = ["full"] } utoipa = "3.3.0" +radix_fmt = "1.0.0" # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true } napi-derive = { version = "2.12.0", optional = true } -radix_fmt = { version = "1.0.0", optional = true } [dev-dependencies] pretty_assertions = "1.3.0" diff --git a/packages/backend/native-utils/__test__/index.spec.mjs b/packages/backend/native-utils/__test__/index.spec.mjs index 6e6a91858..9ff8ead6c 100644 --- a/packages/backend/native-utils/__test__/index.spec.mjs +++ b/packages/backend/native-utils/__test__/index.spec.mjs @@ -12,18 +12,18 @@ test("convert to mastodon id", (t) => { t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219"); t.is( convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId), - "3954607381600562394", + "2083785058661759970208986", ); t.is( convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId), - "3494513243013053824", + "5878598648988104013828532260828151168", ); }); test("create cuid2 with timestamp prefix", (t) => { nativeInitIdGenerator(16, ""); - t.not(nativeCreateId(BigInt(Date.now())), nativeCreateId(BigInt(Date.now()))); - t.is(nativeCreateId(BigInt(Date.now())).length, 16); + t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now())); + t.is(nativeCreateId(Date.now()).length, 16); }); test("create random string", (t) => { diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 7ed9fd5f0..1e5fa57c3 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -10,17 +10,20 @@ path = "src/lib.rs" [features] default = [] -convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] +convert = ["dep:indicatif"] [dependencies] serde_json = "1.0.96" -native-utils = { path = "../", optional = true } +native-utils = { path = "../" } indicatif = { version = "0.17.4", features = ["tokio"], optional = true } tokio = { version = "1.28.2", features = ["full"] } -futures = { version = "0.3.28", optional = true } +futures = "0.3.28" serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2" +redis = { version = "0.23.0", features = ["tokio-rustls-comp"] } +sea-orm = "0.11.3" +url = { version = "2.4.0", features = ["serde"] } [dependencies.sea-orm-migration] version = "0.11.0" diff --git a/packages/backend/native-utils/migration/src/lib.rs b/packages/backend/native-utils/migration/src/lib.rs index 94e2b08cc..5ad23f162 100644 --- a/packages/backend/native-utils/migration/src/lib.rs +++ b/packages/backend/native-utils/migration/src/lib.rs @@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*; mod m20230531_180824_drop_reversi; mod m20230627_185451_index_note_url; +mod m20230709_000510_move_antenna_to_cache; pub struct Migrator; @@ -11,6 +12,7 @@ impl MigratorTrait for Migrator { vec![ Box::new(m20230531_180824_drop_reversi::Migration), Box::new(m20230627_185451_index_note_url::Migration), + Box::new(m20230709_000510_move_antenna_to_cache::Migration), ] } } diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs new file mode 100644 index 000000000..7c38fd1bd --- /dev/null +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -0,0 +1,240 @@ +use redis::streams::StreamMaxlen; +use sea_orm::Statement; +use sea_orm_migration::prelude::*; +use std::env; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cache_url = env::var("CACHE_URL").unwrap(); + let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); + let copy_limit: i64 = match copy_limit.parse() { + Ok(limit) => limit, + Err(_) => 0, + }; + + if cache_url != "no" { + let prefix = env::var("CACHE_PREFIX").unwrap(); + + let db = manager.get_connection(); + let bk = manager.get_database_backend(); + + let count_stmt = + Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned()); + let total_num = db + .query_one(count_stmt) + .await? + .unwrap() + .try_get_by_index::(0)?; + let copy_limit = if copy_limit == 0 { + total_num + } else { + copy_limit + }; + println!( + "Copying {} out of {} entries in antenna_note.", + copy_limit, total_num + ); + + let stmt_base = Query::select() + .column((AntennaNote::Table, AntennaNote::Id)) + .column(AntennaNote::AntennaId) + .column(AntennaNote::NoteId) + .from(AntennaNote::Table) + .order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc) + .limit(1000) + .to_owned(); + + let mut stmt = stmt_base.clone(); + + let client = redis::Client::open(cache_url).unwrap(); + let mut redis_conn = client.get_connection().unwrap(); + + let mut remaining = total_num; + let mut pagination: i64 = 0; + + loop { + let res = db.query_all(bk.build(&stmt)).await?; + if res.len() == 0 { + break; + } + let val: Vec<(String, String, String)> = res + .iter() + .filter_map(|q| q.try_get_many_by_index().ok()) + .collect(); + + remaining -= val.len() as i64; + if remaining <= copy_limit { + let mut pipe = redis::pipe(); + for v in &val { + pipe.xadd_maxlen( + format!("{}:antennaTimeline:{}", prefix, v.1), + StreamMaxlen::Approx(200), + "*", + &[("note", v.2.to_owned())], + ) + .ignore(); + } + pipe.query::<()>(&mut redis_conn).unwrap(); + } + + let copied = total_num - remaining; + let copied = std::cmp::min(copied, total_num); + pagination += 1; + if pagination % 100 == 0 { + println!( + "Migrating antenna [{:.2}%]", + (copied as f64 / total_num as f64) * 100_f64, + ); + } + + if let Some((last_id, _, _)) = val.last() { + stmt = stmt_base + .clone() + .and_where( + Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()), + ) + .to_owned(); + } else { + break; + } + } + } + println!("Migrating antenna [100.00%]"); + + manager + .drop_table( + Table::drop() + .table(AntennaNote::Table) + .if_exists() + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(AntennaNote::Table) + .if_not_exists() + .col( + ColumnDef::new(AntennaNote::Id) + .string_len(32) + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(AntennaNote::NoteId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::AntennaId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::Read) + .boolean() + .default(false) + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_0d775946662d2575dfd2068a5f") + .table(AntennaNote::Table) + .col(AntennaNote::AntennaId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_bd0397be22147e17210940e125") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_335a0bf3f904406f9ef3dd51c2") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .col(AntennaNote::AntennaId) + .unique() + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_9937ea48d7ae97ffb4f3f063a4") + .table(AntennaNote::Table) + .col(AntennaNote::Read) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_0d775946662d2575dfd2068a5f5") + .from(AntennaNote::Table, AntennaNote::AntennaId) + .to(Antenna::Table, Antenna::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_bd0397be22147e17210940e125b") + .from(AntennaNote::Table, AntennaNote::NoteId) + .to(Note::Table, Note::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum AntennaNote { + Table, + Id, + #[iden = "noteId"] + NoteId, + #[iden = "antennaId"] + AntennaId, + Read, +} + +#[derive(Iden)] +enum Antenna { + Table, + Id, +} + +#[derive(Iden)] +enum Note { + Table, + Id, +} diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index f0f761f65..fb9920b67 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -5,6 +5,10 @@ use urlencoding::encode; use sea_orm_migration::prelude::*; +const DB_URL_ENV: &str = "DATABASE_URL"; +const CACHE_URL_ENV: &str = "CACHE_URL"; +const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX"; + #[cfg(feature = "convert")] mod vec_to_json; @@ -15,17 +19,48 @@ async fn main() { .expect("Failed to open '.config/default.yml'"); let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml"); - env::set_var( - "DATABASE_URL", - format!( - "postgres://{}:{}@{}:{}/{}", - config.db.user, - encode(&config.db.pass), - config.db.host, - config.db.port, - config.db.db, - ), - ); + if env::var_os(DB_URL_ENV).is_none() { + env::set_var( + DB_URL_ENV, + format!( + "postgres://{}:{}@{}:{}/{}", + config.db.user, + encode(&config.db.pass), + config.db.host, + config.db.port, + config.db.db, + ), + ); + }; + + if env::var_os(CACHE_URL_ENV).is_none() { + let redis_conf = match config.cache_server { + None => config.redis, + Some(conf) => conf, + }; + let redis_proto = match redis_conf.tls { + None => "redis", + Some(_) => "rediss", + }; + let redis_uri_userpass = match redis_conf.user { + None => "".to_string(), + Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()), + }; + let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port); + let redis_uri = format!( + "{}://{}{}", + redis_proto, redis_uri_userpass, redis_uri_hostport + ); + env::set_var(CACHE_URL_ENV, redis_uri); + env::set_var( + CACHE_PREFIX_ENV, + if redis_conf.prefix.is_empty() { + config.url.host_str().unwrap() + } else { + &redis_conf.prefix + }, + ); + } cli::run_cli(migration::Migrator).await; @@ -36,7 +71,10 @@ async fn main() { #[derive(Debug, PartialEq, Deserialize)] #[serde(rename = "camelCase")] pub struct Config { + pub url: url::Url, pub db: DbConfig, + pub redis: RedisConfig, + pub cache_server: Option, } #[derive(Debug, PartialEq, Deserialize)] @@ -48,3 +86,24 @@ pub struct DbConfig { pub user: String, pub pass: String, } + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct RedisConfig { + pub host: String, + pub port: u32, + pub user: Option, + pub pass: Option, + pub tls: Option, + #[serde(default)] + pub db: u32, + #[serde(default)] + pub prefix: String, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct TlsConfig { + pub host: String, + pub reject_unauthorized: bool, +} diff --git a/packages/backend/native-utils/src/model/schema/app.rs b/packages/backend/native-utils/src/model/schema/app.rs index 682b82ec0..d13e9ef36 100644 --- a/packages/backend/native-utils/src/model/schema/app.rs +++ b/packages/backend/native-utils/src/model/schema/app.rs @@ -105,9 +105,9 @@ mod unit_test { #[test] fn app_valid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), "name": "Test App", "secret": gen_string(24), "callbackUrl": "urn:ietf:wg:oauth:2.0:oob", @@ -119,9 +119,9 @@ mod unit_test { #[test] fn app_invalid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), // "name" is required "name": null, // "permission" must be one of the app permissions diff --git a/packages/backend/native-utils/src/util/id.rs b/packages/backend/native-utils/src/util/id.rs index d922518f9..b18637fdb 100644 --- a/packages/backend/native-utils/src/util/id.rs +++ b/packages/backend/native-utils/src/util/id.rs @@ -1,7 +1,10 @@ //! ID generation utility based on [cuid2] use cfg_if::cfg_if; +use chrono::Utc; use once_cell::sync::OnceCell; +use radix_fmt::radix_36; +use std::cmp; use crate::impl_into_napi_error; @@ -14,47 +17,56 @@ impl_into_napi_error!(ErrorUninitialized); static FINGERPRINT: OnceCell = OnceCell::new(); static GENERATOR: OnceCell = OnceCell::new(); +const TIME_2000: i64 = 946_684_800_000; +const TIMESTAMP_LENGTH: u16 = 8; + /// Initializes Cuid2 generator. Must be called before any [create_id]. -pub fn init_id(length: u16, fingerprint: impl Into) { - FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint.into(), cuid2::create_id())); +pub fn init_id<'a>(length: u16, fingerprint: &'a str) { + FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id())); GENERATOR.get_or_init(move || { cuid2::CuidConstructor::new() - .with_length(length) + // length to pass shoule be greater than or equal to 8. + .with_length(cmp::max(length - TIMESTAMP_LENGTH, 8)) .with_fingerprinter(|| FINGERPRINT.get().unwrap().clone()) }); } /// Returns Cuid2 with the length specified by [init_id]. Must be called after /// [init_id], otherwise returns [ErrorUninitialized]. -pub fn create_id() -> Result { +/// The current timestamp via [chrono::Utc] is used if `date_num` is `0`. +pub fn create_id(date_num: i64) -> Result { match GENERATOR.get() { None => Err(ErrorUninitialized), - Some(gen) => Ok(gen.create_id()), + Some(gen) => { + let date_num = if date_num > 0 { + date_num + } else { + Utc::now().timestamp_millis() + }; + let time = cmp::max(date_num - TIME_2000, 0); + Ok(format!( + "{:0>8}{}", + radix_36(time).to_string(), + gen.create_id() + )) + } } } cfg_if! { if #[cfg(feature = "napi")] { - use radix_fmt::radix_36; - use std::cmp; - use napi::bindgen_prelude::BigInt; use napi_derive::napi; - const TIME_2000: u64 = 946_684_800_000; - const TIMESTAMP_LENGTH: u16 = 8; - /// Calls [init_id] inside. Must be called before [native_create_id]. #[napi] pub fn native_init_id_generator(length: u16, fingerprint: String) { - // length to pass init_id shoule be greater than or equal to 8. - init_id(cmp::max(length - TIMESTAMP_LENGTH, 8), fingerprint); + init_id(length, &fingerprint); } /// Generates #[napi] - pub fn native_create_id(date_num: BigInt) -> String { - let time = cmp::max(date_num.get_u64().1 - TIME_2000, 0); - format!("{:0>8}{}", radix_36(time).to_string(), create_id().unwrap()) + pub fn native_create_id(date_num: i64) -> String { + create_id(date_num).unwrap() } } } @@ -62,37 +74,17 @@ cfg_if! { #[cfg(test)] mod unit_test { use crate::util::id; - use cfg_if::cfg_if; use pretty_assertions::{assert_eq, assert_ne}; use std::thread; - cfg_if! { - if #[cfg(feature = "napi")] { - use chrono::Utc; - - #[test] - fn can_generate_aid_compat_ids() { - id::native_init_id_generator(20, "".to_string()); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_eq!(id1.len(), 20); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - let id2 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_ne!(id1, id2); - let id1 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - let id2 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } else { - #[test] - fn can_generate_unique_ids() { - assert_eq!(id::create_id(), Err(id::ErrorUninitialized)); - id::init_id(12, ""); - assert_eq!(id::create_id().unwrap().len(), 12); - assert_ne!(id::create_id().unwrap(), id::create_id().unwrap()); - let id1 = thread::spawn(|| id::create_id().unwrap()); - let id2 = thread::spawn(|| id::create_id().unwrap()); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } + #[test] + fn can_generate_unique_ids() { + assert_eq!(id::create_id(0), Err(id::ErrorUninitialized)); + id::init_id(16, ""); + assert_eq!(id::create_id(0).unwrap().len(), 16); + assert_ne!(id::create_id(0).unwrap(), id::create_id(0).unwrap()); + let id1 = thread::spawn(|| id::create_id(0).unwrap()); + let id2 = thread::spawn(|| id::create_id(0).unwrap()); + assert_ne!(id1.join().unwrap(), id2.join().unwrap()); } } diff --git a/packages/backend/native-utils/tests/common.rs b/packages/backend/native-utils/tests/common.rs index 186e862bd..31c6ef053 100644 --- a/packages/backend/native-utils/tests/common.rs +++ b/packages/backend/native-utils/tests/common.rs @@ -139,11 +139,11 @@ async fn cleanup() { } async fn setup_model(db: &DbConn) { - init_id(12, ""); + init_id(16, ""); db.transaction::<_, (), DbErr>(|txn| { Box::pin(async move { - let user_id = create_id().unwrap(); + let user_id = create_id(0).unwrap(); let name = "Alice"; let user_model = entity::user::Model { id: user_id.to_owned(), @@ -161,7 +161,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let antenna_model = entity::antenna::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), user_id: user_id.to_owned(), name: "Alice Antenna".to_string(), @@ -186,7 +186,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let note_model = entity::note::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), text: Some("Testing 123".to_string()), user_id: user_id.to_owned(), diff --git a/packages/backend/native-utils/tests/model/repository/antenna.rs b/packages/backend/native-utils/tests/model/repository/antenna.rs index 3bda2ca18..02ef8f5be 100644 --- a/packages/backend/native-utils/tests/model/repository/antenna.rs +++ b/packages/backend/native-utils/tests/model/repository/antenna.rs @@ -95,7 +95,7 @@ mod int_test { .unwrap() .expect("note not found"); let antenna_note = antenna_note::Model { - id: util::id::create_id().unwrap(), + id: util::id::create_id(0).unwrap(), antenna_id: alice_antenna.id.to_owned(), note_id: note_model.id.to_owned(), read: false, diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts index ea0d414e7..580c39c3c 100644 --- a/packages/backend/src/misc/gen-id.ts +++ b/packages/backend/src/misc/gen-id.ts @@ -17,5 +17,5 @@ nativeInitIdGenerator(length, fingerprint); * Ref: https://github.com/paralleldrive/cuid2#parameterized-length */ export function genId(date?: Date): string { - return nativeCreateId(BigInt((date ?? new Date()).getTime())); + return nativeCreateId((date ?? new Date()).getTime()); } diff --git a/packages/backend/src/services/add-note-to-antenna.ts b/packages/backend/src/services/add-note-to-antenna.ts index 131c0348c..8d6d3e84d 100644 --- a/packages/backend/src/services/add-note-to-antenna.ts +++ b/packages/backend/src/services/add-note-to-antenna.ts @@ -8,17 +8,14 @@ import type { User } from "@/models/entities/user.js"; export async function addNoteToAntenna( antenna: Antenna, note: Note, - noteUser: { id: User["id"] }, + _noteUser: { id: User["id"] }, ) { - // 通知しない設定になっているか、自分自身の投稿なら既読にする - const read = !antenna.notify || antenna.userId === noteUser.id; - redisClient.xadd( `antennaTimeline:${antenna.id}`, "MAXLEN", "~", "200", - `${genId(note.createdAt)}-*`, + "*", "note", note.id, ); From 634559d68ae383eecc8861655c4c5a999754d3b7 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 02:17:06 -0400 Subject: [PATCH 06/15] fix: renew entity models --- packages/backend/native-utils/src/model/entity.rs | 1 - .../native-utils/src/model/entity/announcement.rs | 4 ++++ .../backend/native-utils/src/model/entity/antenna.rs | 8 -------- .../backend/native-utils/src/model/entity/meta.rs | 4 ++++ .../backend/native-utils/src/model/entity/note.rs | 8 -------- .../backend/native-utils/src/model/entity/prelude.rs | 1 - .../native-utils/src/model/repository/antenna.rs | 12 +++--------- 7 files changed, 11 insertions(+), 27 deletions(-) diff --git a/packages/backend/native-utils/src/model/entity.rs b/packages/backend/native-utils/src/model/entity.rs index d71057fde..ccf3f0060 100644 --- a/packages/backend/native-utils/src/model/entity.rs +++ b/packages/backend/native-utils/src/model/entity.rs @@ -8,7 +8,6 @@ pub mod ad; pub mod announcement; pub mod announcement_read; pub mod antenna; -pub mod antenna_note; pub mod app; pub mod attestation_challenge; pub mod auth_session; diff --git a/packages/backend/native-utils/src/model/entity/announcement.rs b/packages/backend/native-utils/src/model/entity/announcement.rs index e8a2a28aa..5cdb690d2 100644 --- a/packages/backend/native-utils/src/model/entity/announcement.rs +++ b/packages/backend/native-utils/src/model/entity/announcement.rs @@ -15,6 +15,10 @@ pub struct Model { pub image_url: Option, #[sea_orm(column_name = "updatedAt")] pub updated_at: Option, + #[sea_orm(column_name = "showPopup")] + pub show_popup: bool, + #[sea_orm(column_name = "isGoodNews")] + pub is_good_news: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/native-utils/src/model/entity/antenna.rs b/packages/backend/native-utils/src/model/entity/antenna.rs index 85bdfbfea..9d5a64f82 100644 --- a/packages/backend/native-utils/src/model/entity/antenna.rs +++ b/packages/backend/native-utils/src/model/entity/antenna.rs @@ -37,8 +37,6 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::antenna_note::Entity")] - AntennaNote, #[sea_orm( belongs_to = "super::user::Entity", from = "Column::UserId", @@ -65,12 +63,6 @@ pub enum Relation { UserList, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::AntennaNote.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::User.def() diff --git a/packages/backend/native-utils/src/model/entity/meta.rs b/packages/backend/native-utils/src/model/entity/meta.rs index 2c0dc315c..ac43a7842 100644 --- a/packages/backend/native-utils/src/model/entity/meta.rs +++ b/packages/backend/native-utils/src/model/entity/meta.rs @@ -189,6 +189,10 @@ pub struct Model { pub silenced_hosts: StringVec, #[sea_orm(column_name = "experimentalFeatures", column_type = "JsonBinary")] pub experimental_features: Json, + #[sea_orm(column_name = "enableServerMachineStats")] + pub enable_server_machine_stats: bool, + #[sea_orm(column_name = "enableIdenticonGeneration")] + pub enable_identicon_generation: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/native-utils/src/model/entity/note.rs b/packages/backend/native-utils/src/model/entity/note.rs index 077841e48..5c97a20cf 100644 --- a/packages/backend/native-utils/src/model/entity/note.rs +++ b/packages/backend/native-utils/src/model/entity/note.rs @@ -67,8 +67,6 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::antenna_note::Entity")] - AntennaNote, #[sea_orm( belongs_to = "super::channel::Entity", from = "Column::ChannelId", @@ -131,12 +129,6 @@ pub enum Relation { UserNotePining, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::AntennaNote.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Channel.def() diff --git a/packages/backend/native-utils/src/model/entity/prelude.rs b/packages/backend/native-utils/src/model/entity/prelude.rs index 8be696cb4..a859a9cc3 100644 --- a/packages/backend/native-utils/src/model/entity/prelude.rs +++ b/packages/backend/native-utils/src/model/entity/prelude.rs @@ -6,7 +6,6 @@ pub use super::ad::Entity as Ad; pub use super::announcement::Entity as Announcement; pub use super::announcement_read::Entity as AnnouncementRead; pub use super::antenna::Entity as Antenna; -pub use super::antenna_note::Entity as AntennaNote; pub use super::app::Entity as App; pub use super::attestation_challenge::Entity as AttestationChallenge; pub use super::auth_session::Entity as AuthSession; diff --git a/packages/backend/native-utils/src/model/repository/antenna.rs b/packages/backend/native-utils/src/model/repository/antenna.rs index 7c614b954..36c641cb0 100644 --- a/packages/backend/native-utils/src/model/repository/antenna.rs +++ b/packages/backend/native-utils/src/model/repository/antenna.rs @@ -1,9 +1,9 @@ use async_trait::async_trait; use cfg_if::cfg_if; -use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use sea_orm::EntityTrait; use crate::database; -use crate::model::entity::{antenna, antenna_note, user_group_joining}; +use crate::model::entity::{antenna, user_group_joining}; use crate::model::error::Error; use crate::model::schema::Antenna; @@ -14,12 +14,6 @@ use super::Repository; impl Repository for antenna::Model { async fn pack(self) -> Result { let db = database::get_database()?; - let has_unread_note = antenna_note::Entity::find() - .filter(antenna_note::Column::AntennaId.eq(self.id.to_owned())) - .filter(antenna_note::Column::Read.eq(false)) - .one(db) - .await? - .is_some(); let user_group_joining = match self.user_group_joining_id { None => None, Some(id) => user_group_joining::Entity::find_by_id(id).one(db).await?, @@ -52,7 +46,7 @@ impl Repository for antenna::Model { notify: self.notify, with_replies: self.with_replies, with_file: self.with_file, - has_unread_note, + has_unread_note: false, }) } From c81cef151783b9379ff5801d4b821f6673a5b2d6 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 02:31:50 -0400 Subject: [PATCH 07/15] add environment variable to skip copying antenna --- .../src/m20230709_000510_move_antenna_to_cache.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs index 7c38fd1bd..d285678c9 100644 --- a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -10,13 +10,16 @@ pub struct Migration; impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let cache_url = env::var("CACHE_URL").unwrap(); + let skip_copy = env::var("ANTENNA_MIGRATION_SKIP").unwrap_or_default(); let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); let copy_limit: i64 = match copy_limit.parse() { Ok(limit) => limit, Err(_) => 0, }; - if cache_url != "no" { + if skip_copy == "true" { + println!("Skipped antenna migration"); + } else { let prefix = env::var("CACHE_PREFIX").unwrap(); let db = manager.get_connection(); @@ -102,8 +105,9 @@ impl MigrationTrait for Migration { break; } } + + println!("Migrating antenna [100.00%]"); } - println!("Migrating antenna [100.00%]"); manager .drop_table( From 19abeb662a755117da08009e4cf3f0f2c9109002 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 02:51:54 -0400 Subject: [PATCH 08/15] add environment variable of read size --- .../src/m20230709_000510_move_antenna_to_cache.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs index d285678c9..a514e9b8e 100644 --- a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -11,7 +11,11 @@ impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let cache_url = env::var("CACHE_URL").unwrap(); let skip_copy = env::var("ANTENNA_MIGRATION_SKIP").unwrap_or_default(); - let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); + let copy_limit = env::var("ANTENNA_MIGRATION_COPY_LIMIT").unwrap_or_default(); + let read_limit: u64 = env::var("ANTENNA_MIGRATION_READ_LIMIT") + .unwrap_or("10000".to_string()) + .parse() + .unwrap(); let copy_limit: i64 = match copy_limit.parse() { Ok(limit) => limit, Err(_) => 0, @@ -48,7 +52,7 @@ impl MigrationTrait for Migration { .column(AntennaNote::NoteId) .from(AntennaNote::Table) .order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc) - .limit(1000) + .limit(read_limit) .to_owned(); let mut stmt = stmt_base.clone(); @@ -87,7 +91,7 @@ impl MigrationTrait for Migration { let copied = total_num - remaining; let copied = std::cmp::min(copied, total_num); pagination += 1; - if pagination % 100 == 0 { + if pagination % 10 == 0 { println!( "Migrating antenna [{:.2}%]", (copied as f64 / total_num as f64) * 100_f64, From 626bbf2ee2f09b2dd7927e5d387f1c7219b3869b Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:23:25 -0400 Subject: [PATCH 09/15] fix config parser --- packages/backend/native-utils/migration/src/main.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index fb9920b67..a7b649c51 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -69,7 +69,7 @@ async fn main() { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] +#[serde(rename_all = "camelCase")] pub struct Config { pub url: url::Url, pub db: DbConfig, @@ -78,7 +78,6 @@ pub struct Config { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] pub struct DbConfig { pub host: String, pub port: u32, @@ -88,7 +87,6 @@ pub struct DbConfig { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] pub struct RedisConfig { pub host: String, pub port: u32, @@ -102,7 +100,7 @@ pub struct RedisConfig { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] +#[serde(rename_all = "camelCase")] pub struct TlsConfig { pub host: String, pub reject_unauthorized: bool, From 0fb7449dcbc1af86587d61374bcdeb77414b1b23 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:43:44 -0400 Subject: [PATCH 10/15] remove unnecessary migration file --- packages/backend/migration/1680491187535-cleanup.js | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 packages/backend/migration/1680491187535-cleanup.js diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js deleted file mode 100644 index 671f7521e..000000000 --- a/packages/backend/migration/1680491187535-cleanup.js +++ /dev/null @@ -1,7 +0,0 @@ -export class cleanup1680491187535 { - name = "cleanup1680491187535"; - - async up(queryRunner) {} - - async down(queryRunner) {} -} From 2ed170215418e010575128a6823d3d0f20256f42 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:48:46 -0400 Subject: [PATCH 11/15] make unused dependencies optional --- .../backend/native-utils/migration/Cargo.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 1e5fa57c3..9bf793e04 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -10,14 +10,14 @@ path = "src/lib.rs" [features] default = [] -convert = ["dep:indicatif"] +convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] [dependencies] serde_json = "1.0.96" -native-utils = { path = "../" } +native-utils = { path = "../", optional = true } indicatif = { version = "0.17.4", features = ["tokio"], optional = true } tokio = { version = "1.28.2", features = ["full"] } -futures = "0.3.28" +futures = { version = "0.3.28", optional = true } serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2" @@ -28,10 +28,10 @@ url = { version = "2.4.0", features = ["serde"] } [dependencies.sea-orm-migration] version = "0.11.0" features = [ - # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. - # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. - # e.g. - "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature - "sqlx-postgres", # `DATABASE_DRIVER` feature + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + # e.g. + "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature + "sqlx-postgres", # `DATABASE_DRIVER` feature "sqlx-sqlite", ] From 9ea5fa80f8524a0751217c375094391df28b6eac Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:54:28 -0400 Subject: [PATCH 12/15] fix: add db to redis uri --- packages/backend/native-utils/migration/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index a7b649c51..896f1ed59 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -44,12 +44,12 @@ async fn main() { }; let redis_uri_userpass = match redis_conf.user { None => "".to_string(), - Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()), + Some(user) => format!("{}:{}@", user, encode(&redis_conf.pass.unwrap_or_default())), }; let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port); let redis_uri = format!( - "{}://{}{}", - redis_proto, redis_uri_userpass, redis_uri_hostport + "{}://{}{}/{}", + redis_proto, redis_uri_userpass, redis_uri_hostport, redis_conf.db ); env::set_var(CACHE_URL_ENV, redis_uri); env::set_var( From aabb2822e039c95b97b5792275baad98d4469b40 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 13 Jul 2023 18:19:08 -0400 Subject: [PATCH 13/15] refactor: use copy_limit if greater than 0 --- .../migration/src/m20230709_000510_move_antenna_to_cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs index a514e9b8e..83bd0448a 100644 --- a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -36,10 +36,10 @@ impl MigrationTrait for Migration { .await? .unwrap() .try_get_by_index::(0)?; - let copy_limit = if copy_limit == 0 { - total_num - } else { + let copy_limit = if copy_limit > 0 { copy_limit + } else { + total_num }; println!( "Copying {} out of {} entries in antenna_note.", From 80cd7b434915aaf0c97c733ae8063d42f8bb88f8 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 17 Jul 2023 01:48:53 -0400 Subject: [PATCH 14/15] fix: use hostname as prefix instead of host --- packages/backend/src/config/load.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/config/load.ts b/packages/backend/src/config/load.ts index fa9878955..df163a8ac 100644 --- a/packages/backend/src/config/load.ts +++ b/packages/backend/src/config/load.ts @@ -54,9 +54,9 @@ export default function load() { mixin.userAgent = `Calckey/${meta.version} (${config.url})`; mixin.clientEntry = clientManifest["src/init.ts"]; - if (!config.redis.prefix) config.redis.prefix = mixin.host; + if (!config.redis.prefix) config.redis.prefix = mixin.hostname; if (config.cacheServer && !config.cacheServer.prefix) - config.cacheServer.prefix = mixin.host; + config.cacheServer.prefix = mixin.hostname; return Object.assign(config, mixin); } From b7d68da285825a3ff551736c13ab1ce4ce84933b Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Mon, 17 Jul 2023 09:50:28 -0700 Subject: [PATCH 15/15] hide mark read button --- packages/client/src/pages/antenna-timeline.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/client/src/pages/antenna-timeline.vue b/packages/client/src/pages/antenna-timeline.vue index 825dbb96a..011e44ce6 100644 --- a/packages/client/src/pages/antenna-timeline.vue +++ b/packages/client/src/pages/antenna-timeline.vue @@ -100,11 +100,11 @@ const headerActions = $computed(() => text: i18n.ts.settings, handler: settings, }, - { - icon: "ph-check ph-bold ph-lg", - text: i18n.ts.markAllAsRead, - handler: markRead, - }, + // { + // icon: "ph-check ph-bold ph-lg", + // text: i18n.ts.markAllAsRead, + // handler: markRead, + // }, ] : [], );