feat: auto nsfw detection (#8840)

* feat: auto nsfw detection

* ✌️

* Update ja-JP.yml

* Update ja-JP.yml

* ポルノ判定のしきい値を高めに

* エラーハンドリングちゃんとした

* Update ja-JP.yml

* 感度設定を強化

* refactor

* feat: add video support for auto nsfw detection

* rename: image -> media

* .js

* fix: add missing error handling

* fix: use valid pathname instead of using filename due to invalid usage

* perf(nsfw-detection): decode frames

* disable detection of video for some reasons

* perf(nsfw-detection): streamify detection process for video

* disable disallowUploadWhenPredictedAsPorn option

* fix(nsfw-detection): improve reliability

* fix(nsfw-detection): use Math.ceil instead of Math.round

* perf(nsfw-detection): delete tmp frames after used

* fix(nsfw-detection): FSWatcher does not emit ready event

* perf(nsfw-detection): skip black frames

* refactor: strip exists check

* Update package.json

* めっちゃ変えた

* lint

* Update COPYING

* オプションで動画解析できるように

* Update yarn.lock

* Update CHANGELOG.md

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

CHANGELOG.md

@@ -11,6 +11,9 @@ You should also include the user name that made the change.
 
 ## 12.x.x (unreleased)
 
+### Known issues
+- 現在arm64環境ではインストールに失敗します。これは次のバージョンで修正される予定です。
+
 ### Changes
 - ハイライトがみつけるに統合されました
 - カスタム絵文字ページはインスタンス情報ページに統合されました
@@ -18,6 +21,7 @@ You should also include the user name that made the change.
 
 ### Improvements
 - Server: Allow GET method for some endpoints @syuilo
+- Server: Auto NSFW detection @syuilo
 - Server: Add rate limit to i/notifications @tamaina
 - Client: Improve control panel @syuilo
 - Client: Show warning in control panel when there is an unresolved abuse report @syuilo

COPYING

@@ -1,5 +1,5 @@
 Unless otherwise stated this repository is
-Copyright © 2014-2020 syuilo and contributers
+Copyright © 2014-2022 syuilo and contributers
 
 And is distributed under The GNU Affero General Public License Version 3, you should have received a copy of the license file as LICENSE.
 
@@ -13,3 +13,7 @@ https://github.com/muan/emojilib/blob/master/LICENSE
 RsaSignature2017 implementation by Transmute Industries Inc
 License: MIT
 https://github.com/transmute-industries/RsaSignature2017/blob/master/LICENSE
+
+Machine learning model for sensitive images by Infinite Red, Inc.
+License: MIT
+https://github.com/infinitered/nsfwjs/blob/master/LICENSE

locales/ja-JP.yml

@@ -877,6 +877,24 @@ type: "タイプ"
 speed: "速度"
 slow: "遅い"
 fast: "速い"
+sensitiveMediaDetection: "センシティブなメディアの検出"
+localOnly: "ローカルのみ"
+remoteOnly: "リモートのみ"
+failedToUpload: "アップロード失敗"
+cannotUploadBecauseInappropriate: "不適切な内容を含む可能性があると判定されたためアップロードできません。"
+cannotUploadBecauseNoFreeSpace: "ドライブの空き容量が無いためアップロードできません。"
+beta: "ベータ"
+enableAutoSensitive: "自動NSFW判定"
+enableAutoSensitiveDescription: "利用可能な場合は、機械学習を利用して自動でメディアにNSFWフラグを設定します。この機能をオフにしても、インスタンスによっては自動で設定されることがあります。"
+
+_sensitiveMediaDetection:
+  description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てることができます。サーバーの負荷が少し増えます。"
+  sensitivity: "検出感度"
+  sensitivityDescription: "感度を低くすると、誤検知(偽陽性)が減ります。感度を高くすると、検知漏れ(偽陰性)が減ります。"
+  setSensitiveFlagAutomatically: "NSFWフラグを設定する"
+  setSensitiveFlagAutomaticallyDescription: "この設定をオフにしても内部的に判定結果は保持されます。"
+  analyzeVideos: "動画の解析を有効化"
+  analyzeVideosDescription: "静止画に加えて動画も解析するようにします。サーバーの負荷が少し増えます。"
 
 _emailUnavailable:
   used: "既に使用されています"

packages/backend/migration/1655368940105-nsfw-detection.js

@@ -0,0 +1,23 @@
+export class nsfwDetection1655368940105 {
+    name = 'nsfwDetection1655368940105'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "drive_file" ADD "forceIsSensitive" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`ALTER TABLE "drive_file" ADD "predictedIsSensitive" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`COMMENT ON COLUMN "drive_file"."predictedIsSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetection_enum" AS ENUM('none', 'all', 'local', 'remote')`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetection" "public"."meta_sensitiveimagedetection_enum" NOT NULL DEFAULT 'none'`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "forceIsSensitiveWhenPredicted" boolean NOT NULL DEFAULT true`);
+        await queryRunner.query(`CREATE INDEX "IDX_fc2d74a6d7d8b11292a851d8f8" ON "drive_file" ("predictedIsSensitive") `);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`DROP INDEX "public"."IDX_fc2d74a6d7d8b11292a851d8f8"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "forceIsSensitiveWhenPredicted"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetection"`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetection_enum"`);
+        await queryRunner.query(`COMMENT ON COLUMN "drive_file"."predictedIsSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
+        await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "predictedIsSensitive"`);
+        await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "forceIsSensitive"`);
+    }
+}

packages/backend/migration/1655371960534-nsfw-detection-2.js

@@ -0,0 +1,15 @@
+export class nsfwDetection21655371960534 {
+    name = 'nsfwDetection21655371960534'
+
+    async up(queryRunner) {
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" AS ENUM('medium', 'low', 'high')`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetectionSensitivity" "public"."meta_sensitiveimagedetectionsensitivity_enum" NOT NULL DEFAULT 'medium'`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "disallowUploadWhenPredictedAsPorn" boolean NOT NULL DEFAULT false`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "disallowUploadWhenPredictedAsPorn"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetectionSensitivity"`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum"`);
+    }
+}

packages/backend/migration/1655388169582-nsfw-detection-3.js

@@ -0,0 +1,21 @@
+export class nsfwDetection31655388169582 {
+    name = 'nsfwDetection31655388169582'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" RENAME TO "meta_sensitiveimagedetectionsensitivity_enum_old"`);
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" AS ENUM('medium', 'low', 'high', 'veryLow', 'veryHigh')`);
+        await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" DROP DEFAULT`);
+        await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" USING "sensitiveImageDetectionSensitivity"::"text"::"public"."meta_sensitiveimagedetectionsensitivity_enum"`);
+        await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" SET DEFAULT 'medium'`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old"`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old" AS ENUM('medium', 'low', 'high')`);
+        await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" DROP DEFAULT`);
+        await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old" USING "sensitiveImageDetectionSensitivity"::"text"::"public"."meta_sensitiveimagedetectionsensitivity_enum_old"`);
+        await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" SET DEFAULT 'medium'`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum"`);
+        await queryRunner.query(`ALTER TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old" RENAME TO "meta_sensitiveimagedetectionsensitivity_enum"`);
+    }
+}

packages/backend/migration/1655393015659-nsfw-detection-4.js

@@ -0,0 +1,25 @@
+export class nsfwDetection41655393015659 {
+    name = 'nsfwDetection41655393015659'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetection"`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetection_enum"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetectionSensitivity"`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum"`);
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitivemediadetection_enum" AS ENUM('none', 'all', 'local', 'remote')`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveMediaDetection" "public"."meta_sensitivemediadetection_enum" NOT NULL DEFAULT 'none'`);
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitivemediadetectionsensitivity_enum" AS ENUM('medium', 'low', 'high', 'veryLow', 'veryHigh')`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveMediaDetectionSensitivity" "public"."meta_sensitivemediadetectionsensitivity_enum" NOT NULL DEFAULT 'medium'`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveMediaDetectionSensitivity"`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitivemediadetectionsensitivity_enum"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveMediaDetection"`);
+        await queryRunner.query(`DROP TYPE "public"."meta_sensitivemediadetection_enum"`);
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" AS ENUM('medium', 'low', 'high', 'veryLow', 'veryHigh')`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetectionSensitivity" "public"."meta_sensitiveimagedetectionsensitivity_enum" NOT NULL DEFAULT 'medium'`);
+        await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetection_enum" AS ENUM('none', 'all', 'local', 'remote')`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetection" "public"."meta_sensitiveimagedetection_enum" NOT NULL DEFAULT 'none'`);
+    }
+}

packages/backend/migration/1656251734807-nsfw-detection-5.js

@@ -0,0 +1,33 @@
+export class nsfwDetection51656251734807 {
+    name = 'nsfwDetection51656251734807'
+
+    async up(queryRunner) {
+        await queryRunner.query(`DROP INDEX "public"."IDX_fc2d74a6d7d8b11292a851d8f8"`);
+        await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "forceIsSensitive"`);
+        await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "predictedIsSensitive"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "forceIsSensitiveWhenPredicted"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "disallowUploadWhenPredictedAsPorn"`);
+        await queryRunner.query(`ALTER TABLE "drive_file" ADD "maybeSensitive" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`COMMENT ON COLUMN "drive_file"."maybeSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
+        await queryRunner.query(`ALTER TABLE "drive_file" ADD "maybePorn" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "setSensitiveFlagAutomatically" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`ALTER TABLE "user_profile" ADD "autoSensitive" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`CREATE INDEX "IDX_3b33dff77bb64b23c88151d23e" ON "drive_file" ("maybeSensitive") `);
+        await queryRunner.query(`CREATE INDEX "IDX_8bdcd3dd2bddb78014999a16ce" ON "drive_file" ("maybePorn") `);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`DROP INDEX "public"."IDX_8bdcd3dd2bddb78014999a16ce"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_3b33dff77bb64b23c88151d23e"`);
+        await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "autoSensitive"`);
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "setSensitiveFlagAutomatically"`);
+        await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "maybePorn"`);
+        await queryRunner.query(`COMMENT ON COLUMN "drive_file"."maybeSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
+        await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "maybeSensitive"`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "disallowUploadWhenPredictedAsPorn" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "forceIsSensitiveWhenPredicted" boolean NOT NULL DEFAULT true`);
+        await queryRunner.query(`ALTER TABLE "drive_file" ADD "predictedIsSensitive" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`ALTER TABLE "drive_file" ADD "forceIsSensitive" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`CREATE INDEX "IDX_fc2d74a6d7d8b11292a851d8f8" ON "drive_file" ("predictedIsSensitive") `);
+    }
+}

packages/backend/migration/1656408772602-nsfw-detection-6.js

@@ -0,0 +1,11 @@
+export class nsfwDetection61656408772602 {
+    name = 'nsfwDetection61656408772602'
+
+    async up(queryRunner) {
+				await queryRunner.query(`ALTER TABLE "meta" ADD "enableSensitiveMediaDetectionForVideos" boolean NOT NULL DEFAULT false`);
+    }
+
+    async down(queryRunner) {
+			await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableSensitiveMediaDetectionForVideos"`);
+    }
+}

packages/backend/nsfw-model/group1-shard5of6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:994515b0f73407d71740301e9877511b086609dbdc7ead1b31bcb82b3397ae74
+size 4194304

packages/backend/package.json

@@ -23,6 +23,7 @@
 		"@peertube/http-signature": "1.6.0",
 		"@sinonjs/fake-timers": "9.1.2",
 		"@syuilo/aiscript": "0.11.1",
+		"@tensorflow/tfjs-node": "3.18.0",
 		"abort-controller": "3.0.0",
 		"ajv": "8.11.0",
 		"archiver": "5.3.1",
@@ -36,6 +37,7 @@
 		"cbor": "8.1.0",
 		"chalk": "5.0.1",
 		"chalk-template": "0.4.0",
+		"chokidar": "3.3.1",
 		"cli-highlight": "2.1.11",
 		"color-convert": "2.0.1",
 		"content-disposition": "0.5.4",
@@ -74,6 +76,7 @@
 		"nested-property": "4.0.0",
 		"node-fetch": "3.2.6",
 		"nodemailer": "6.7.6",
+		"nsfwjs": "2.4.1",
 		"os-utils": "0.0.14",
 		"parse5": "7.0.0",
 		"pg": "8.7.3",

packages/backend/src/misc/get-file-info.ts

@@ -1,12 +1,18 @@
 import * as fs from 'node:fs';
 import * as crypto from 'node:crypto';
+import { join } from 'node:path';
 import * as stream from 'node:stream';
 import * as util from 'node:util';
+import { FSWatcher } from 'chokidar';
 import { fileTypeFromFile } from 'file-type';
+import FFmpeg from 'fluent-ffmpeg';
 import isSvg from 'is-svg';
 import probeImageSize from 'probe-image-size';
+import { type predictionType } from 'nsfwjs';
 import sharp from 'sharp';
 import { encode } from 'blurhash';
+import { detectSensitive } from '@/services/detect-sensitive.js';
+import { createTempDir } from './create-temp.js';
 
 const pipeline = util.promisify(stream.pipeline);
 
@@ -21,6 +27,8 @@ export type FileInfo = {
 	height?: number;
 	orientation?: number;
 	blurhash?: string;
+	sensitive: boolean;
+	porn: boolean;
 	warnings: string[];
 };
 
@@ -37,7 +45,12 @@ const TYPE_SVG = {
 /**
  * Get file information
  */
-export async function getFileInfo(path: string): Promise<FileInfo> {
+export async function getFileInfo(path: string, opts: {
+	skipSensitiveDetection: boolean;
+	sensitiveThreshold?: number;
+	sensitiveThresholdForPorn?: number;
+	enableSensitiveMediaDetectionForVideos?: boolean;
+}): Promise<FileInfo> {
 	const warnings = [] as string[];
 
 	const size = await getFileSize(path);
@@ -58,7 +71,7 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
 
 		// うまく判定できない画像は octet-stream にする
 		if (!imageSize) {
-			warnings.push(`cannot detect image dimensions`);
+			warnings.push('cannot detect image dimensions');
 			type = TYPE_OCTET_STREAM;
 		} else if (imageSize.wUnits === 'px') {
 			width = imageSize.width;
@@ -67,7 +80,7 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
 
 			// 制限を超えている画像は octet-stream にする
 			if (imageSize.width > 16383 || imageSize.height > 16383) {
-				warnings.push(`image dimensions exceeds limits`);
+				warnings.push('image dimensions exceeds limits');
 				type = TYPE_OCTET_STREAM;
 			}
 		} else {
@@ -84,6 +97,19 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
 		});
 	}
 
+	let sensitive = false;
+	let porn = false;
+
+	if (!opts.skipSensitiveDetection) {
+		[sensitive, porn] = await detectSensitivity(
+			path,
+			type.mime,
+			opts.sensitiveThreshold ?? 0.5,
+			opts.sensitiveThresholdForPorn ?? 0.75,
+			opts.enableSensitiveMediaDetectionForVideos ?? false,
+		);
+	}
+
 	return {
 		size,
 		md5,
@@ -92,10 +118,150 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
 		height,
 		orientation,
 		blurhash,
+		sensitive,
+		porn,
 		warnings,
 	};
 }
 
+async function detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
+	let sensitive = false;
+	let porn = false;
+
+	function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] {
+		let sensitive = false;
+		let porn = false;
+
+		if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
+		if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
+		if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
+
+		if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
+
+		return [sensitive, porn];
+	}
+
+	if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) {
+		const result = await detectSensitive(source);
+		if (result) {
+			[sensitive, porn] = judgePrediction(result);
+		}
+	} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
+		const [outDir, disposeOutDir] = await createTempDir();
+		try {
+			const command = FFmpeg()
+				.input(source)
+				.inputOptions([
+					'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする（そうなるとは限らない）
+					'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする（そうなるとは限らない）
+				])
+				.noAudio()
+				.videoFilters([
+					{
+						filter: 'select', // フレームのフィルタリング
+						options: {
+							e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタする（VP9 とかはデコードしてみないとわからないっぽい）
+						},
+					},
+					{
+						filter: 'blackframe', // 暗いフレームの検出
+						options: {
+							amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
+						},
+					},
+					{
+						filter: 'metadata',
+						options: {
+							mode: 'select', // フレーム選択モード
+							key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率（前のフィルタからのメタデータを参照する）
+							value: '50',
+							function: 'less', // 50% 未満のフレームを選択する（50% 以上暗部があるフレームだと誤検知を招くかもしれないので）
+						},
+					},
+					{
+						filter: 'scale',
+						options: {
+							w: 299,
+							h: 299,
+						},
+					},
+				])
+				.format('image2')
+				.output(join(outDir, '%d.png'))
+				.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
+			const results: ReturnType<typeof judgePrediction>[] = [];
+			let frameIndex = 0;
+			let targetIndex = 0;
+			let nextIndex = 1;
+			for await (const path of asyncIterateFrames(outDir, command)) {
+				try {
+					const index = frameIndex++;
+					if (index !== targetIndex) {
+						continue;
+					}
+					targetIndex = nextIndex;
+					nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける
+					const result = await detectSensitive(path);
+					if (result) {
+						results.push(judgePrediction(result));
+					}
+				} finally {
+					fs.promises.unlink(path);
+				}
+			}
+			sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
+			porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
+		} finally {
+			disposeOutDir();
+		}
+	}
+
+	return [sensitive, porn];
+}
+
+async function* asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {
+	const watcher = new FSWatcher({
+		cwd,
+		disableGlobbing: true,
+	});
+	let finished = false;
+	command.once('end', () => {
+		finished = true;
+		watcher.close();
+	});
+	command.run();
+	for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
+		const current = `${i}.png`;
+		const next = `${i + 1}.png`;
+		const framePath = join(cwd, current);
+		if (await exists(join(cwd, next))) {
+			yield framePath;
+		} else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
+			watcher.add(next);
+			await new Promise<void>((resolve, reject) => {
+				watcher.on('add', function onAdd(path) {
+					if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
+						watcher.unwatch(current);
+						watcher.off('add', onAdd);
+						resolve();
+					}
+				});
+				command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
+				command.once('error', reject);
+			});
+			yield framePath;
+		} else if (await exists(framePath)) {
+			yield framePath;
+		} else {
+			return;
+		}
+	}
+}
+
+function exists(path: string): Promise<boolean> {
+	return fs.promises.access(path).then(() => true, () => false);
+}
+
 /**
  * Detect MIME Type and extension
  */

packages/backend/src/models/entities/drive-file.ts

@@ -156,6 +156,19 @@ export class DriveFile {
 	})
 	public isSensitive: boolean;
 
+	@Index()
+	@Column('boolean', {
+		default: false,
+		comment: 'Whether the DriveFile is NSFW. (predict)',
+	})
+	public maybeSensitive: boolean;
+
+	@Index()
+	@Column('boolean', {
+		default: false,
+	})
+	public maybePorn: boolean;
+
 	/**
 	 * 外部の(信頼されていない)URLへの直リンクか否か
 	 */

packages/backend/src/models/entities/meta.ts

@@ -188,6 +188,28 @@ export class Meta {
 	})
 	public recaptchaSecretKey: string | null;
 
+	@Column('enum', {
+		enum: ['none', 'all', 'local', 'remote'],
+		default: 'none',
+	})
+	public sensitiveMediaDetection: 'none' | 'all' | 'local' | 'remote';
+
+	@Column('enum', {
+		enum: ['medium', 'low', 'high', 'veryLow', 'veryHigh'],
+		default: 'medium',
+	})
+	public sensitiveMediaDetectionSensitivity: 'medium' | 'low' | 'high' | 'veryLow' | 'veryHigh';
+
+	@Column('boolean', {
+		default: false,
+	})
+	public setSensitiveFlagAutomatically: boolean;
+
+	@Column('boolean', {
+		default: false,
+	})
+	public enableSensitiveMediaDetectionForVideos: boolean;
+
 	@Column('integer', {
 		default: 1024,
 		comment: 'Drive capacity of a local user (MB)',

packages/backend/src/models/entities/user-profile.ts

@@ -152,6 +152,11 @@ export class UserProfile {
 	})
 	public alwaysMarkNsfw: boolean;
 
+	@Column('boolean', {
+		default: false,
+	})
+	public autoSensitive: boolean;
+
 	@Column('boolean', {
 		default: false,
 	})

packages/backend/src/models/repositories/user.ts

@@ -360,6 +360,7 @@ export const UserRepository = db.getRepository(User).extend({
 				injectFeaturedNote: profile!.injectFeaturedNote,
 				receiveAnnouncementEmail: profile!.receiveAnnouncementEmail,
 				alwaysMarkNsfw: profile!.alwaysMarkNsfw,
+				autoSensitive: profile!.autoSensitive,
 				carefulBot: profile!.carefulBot,
 				autoAcceptFollowed: profile!.autoAcceptFollowed,
 				noCrawle: profile!.noCrawle,

packages/backend/src/models/schema/user.ts

@@ -161,19 +161,19 @@ export const packedUserDetailedNotMeOnlySchema = {
 			type: 'array',
 			nullable: false, optional: false,
 			items: {
-					type: 'object',
-					nullable: false, optional: false,
-					properties: {
-						name: {
-							type: 'string',
-							nullable: false, optional: false,
-						},
-						value: {
-							type: 'string',
-							nullable: false, optional: false,
-						},
+				type: 'object',
+				nullable: false, optional: false,
+				properties: {
+					name: {
+						type: 'string',
+						nullable: false, optional: false,
 					},
-					maxLength: 4,
+					value: {
+						type: 'string',
+						nullable: false, optional: false,
+					},
+				},
+				maxLength: 4,
 			},
 		},
 		followersCount: {
@@ -292,6 +292,10 @@ export const packedMeDetailedOnlySchema = {
 			type: 'boolean',
 			nullable: true, optional: false,
 		},
+		autoSensitive: {
+			type: 'boolean',
+			nullable: true, optional: false,
+		},
 		carefulBot: {
 			type: 'boolean',
 			nullable: true, optional: false,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -195,6 +195,22 @@ export const meta = {
 				type: 'string',
 				optional: true, nullable: true,
 			},
+			sensitiveMediaDetection: {
+				type: 'string',
+				optional: true, nullable: false,
+			},
+			sensitiveMediaDetectionSensitivity: {
+				type: 'string',
+				optional: true, nullable: false,
+			},
+			setSensitiveFlagAutomatically: {
+				type: 'boolean',
+				optional: true, nullable: false,
+			},
+			enableSensitiveMediaDetectionForVideos: {
+				type: 'boolean',
+				optional: true, nullable: false,
+			},
 			proxyAccountId: {
 				type: 'string',
 				optional: true, nullable: true,
@@ -370,6 +386,10 @@ export default define(meta, paramDef, async (ps, me) => {
 		blockedHosts: instance.blockedHosts,
 		hcaptchaSecretKey: instance.hcaptchaSecretKey,
 		recaptchaSecretKey: instance.recaptchaSecretKey,
+		sensitiveMediaDetection: instance.sensitiveMediaDetection,
+		sensitiveMediaDetectionSensitivity: instance.sensitiveMediaDetectionSensitivity,
+		setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
+		enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
 		proxyAccountId: instance.proxyAccountId,
 		twitterConsumerKey: instance.twitterConsumerKey,
 		twitterConsumerSecret: instance.twitterConsumerSecret,

packages/backend/src/server/api/endpoints/admin/show-user.ts

@@ -58,6 +58,7 @@ export default define(meta, paramDef, async (ps, me) => {
 		autoAcceptFollowed: profile.autoAcceptFollowed,
 		noCrawle: profile.noCrawle,
 		alwaysMarkNsfw: profile.alwaysMarkNsfw,
+		autoSensitive: profile.autoSensitive,
 		carefulBot: profile.carefulBot,
 		injectFeaturedNote: profile.injectFeaturedNote,
 		receiveAnnouncementEmail: profile.receiveAnnouncementEmail,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -48,6 +48,10 @@ export const paramDef = {
 		enableRecaptcha: { type: 'boolean' },
 		recaptchaSiteKey: { type: 'string', nullable: true },
 		recaptchaSecretKey: { type: 'string', nullable: true },
+		sensitiveMediaDetection: { type: 'string', enum: ['none', 'all', 'local', 'remote'] },
+		sensitiveMediaDetectionSensitivity: { type: 'string', enum: ['medium', 'low', 'high', 'veryLow', 'veryHigh'] },
+		setSensitiveFlagAutomatically: { type: 'boolean' },
+		enableSensitiveMediaDetectionForVideos: { type: 'boolean' },
 		proxyAccountId: { type: 'string', format: 'misskey:id', nullable: true },
 		maintainerName: { type: 'string', nullable: true },
 		maintainerEmail: { type: 'string', nullable: true },
@@ -213,6 +217,22 @@ export default define(meta, paramDef, async (ps, me) => {
 		set.recaptchaSecretKey = ps.recaptchaSecretKey;
 	}
 
+	if (ps.sensitiveMediaDetection !== undefined) {
+		set.sensitiveMediaDetection = ps.sensitiveMediaDetection;
+	}
+
+	if (ps.sensitiveMediaDetectionSensitivity !== undefined) {
+		set.sensitiveMediaDetectionSensitivity = ps.sensitiveMediaDetectionSensitivity;
+	}
+
+	if (ps.setSensitiveFlagAutomatically !== undefined) {
+		set.setSensitiveFlagAutomatically = ps.setSensitiveFlagAutomatically;
+	}
+
+	if (ps.enableSensitiveMediaDetectionForVideos !== undefined) {
+		set.enableSensitiveMediaDetectionForVideos = ps.enableSensitiveMediaDetectionForVideos;
+	}
+
 	if (ps.proxyAccountId !== undefined) {
 		set.proxyAccountId = ps.proxyAccountId;
 	}

packages/backend/src/server/api/endpoints/drive/files/create.ts

@@ -2,6 +2,7 @@ import ms from 'ms';
 import { addFile } from '@/services/drive/add-file.js';
 import { DriveFiles } from '@/models/index.js';
 import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
+import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { fetchMeta } from '@/misc/fetch-meta.js';
 import define from '../../../define.js';
 import { apiLogger } from '../../../logger.js';
@@ -35,6 +36,18 @@ export const meta = {
 			code: 'INVALID_FILE_NAME',
 			id: 'f449b209-0c60-4e51-84d5-29486263bfd4',
 		},
+
+		inappropriate: {
+			message: 'Cannot upload the file because it has been determined that it possibly contains inappropriate content.',
+			code: 'INAPPROPRIATE',
+			id: 'bec5bd69-fba3-43c9-b4fb-2894b66ad5d2',
+		},
+
+		noFreeSpace: {
+			message: 'Cannot upload the file because you have no free space of drive.',
+			code: 'NO_FREE_SPACE',
+			id: 'd08dbc37-a6a9-463a-8c47-96c32ab5f064',
+		},
 	},
 } as const;
 
@@ -87,6 +100,10 @@ export default define(meta, paramDef, async (ps, user, _, file, cleanup, ip, hea
 		if (e instanceof Error || typeof e === 'string') {
 			apiLogger.error(e);
 		}
+		if (e instanceof IdentifiableError) {
+			if (e.id === '282f77bf-5816-4f72-9264-aa14d8261a21') throw new ApiError(meta.errors.inappropriate);
+			if (e.id === 'c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6') throw new ApiError(meta.errors.noFreeSpace);
+		}
 		throw new ApiError();
 	} finally {
 		cleanup!();

packages/backend/src/server/api/endpoints/drive/files/update.ts

@@ -1,8 +1,8 @@
 import { publishDriveStream } from '@/services/stream.js';
-import define from '../../../define.js';
-import { ApiError } from '../../../error.js';
 import { DriveFiles, DriveFolders, Users } from '@/models/index.js';
 import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
+import define from '../../../define.js';
+import { ApiError } from '../../../error.js';
 
 export const meta = {
 	tags: ['drive'],

packages/backend/src/server/api/endpoints/i/update.ts

@@ -3,17 +3,17 @@ import * as mfm from 'mfm-js';
 import { publishMainStream, publishUserEvent } from '@/services/stream.js';
 import acceptAllFollowRequests from '@/services/following/requests/accept-all.js';
 import { publishToFollowers } from '@/services/i/update.js';
-import define from '../../define.js';
 import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
 import { extractHashtags } from '@/misc/extract-hashtags.js';
 import { updateUsertags } from '@/services/update-hashtag.js';
-import { ApiError } from '../../error.js';
 import { Users, DriveFiles, UserProfiles, Pages } from '@/models/index.js';
 import { User } from '@/models/entities/user.js';
 import { UserProfile } from '@/models/entities/user-profile.js';
 import { notificationTypes } from '@/types.js';
 import { normalizeForSearch } from '@/misc/normalize-for-search.js';
 import { langmap } from '@/misc/langmap.js';
+import { ApiError } from '../../error.js';
+import define from '../../define.js';
 
 export const meta = {
 	tags: ['account'],
@@ -57,7 +57,7 @@ export const meta = {
 			message: 'Invalid Regular Expression.',
 			code: 'INVALID_REGEXP',
 			id: '0d786918-10df-41cd-8f33-8dec7d9a89a5',
-		}
+		},
 	},
 
 	res: {
@@ -77,7 +77,8 @@ export const paramDef = {
 		lang: { type: 'string', enum: [null, ...Object.keys(langmap)], nullable: true },
 		avatarId: { type: 'string', format: 'misskey:id', nullable: true },
 		bannerId: { type: 'string', format: 'misskey:id', nullable: true },
-		fields: { type: 'array',
+		fields: {
+			type: 'array',
 			minItems: 0,
 			maxItems: 16,
 			items: {
@@ -102,6 +103,7 @@ export const paramDef = {
 		injectFeaturedNote: { type: 'boolean' },
 		receiveAnnouncementEmail: { type: 'boolean' },
 		alwaysMarkNsfw: { type: 'boolean' },
+		autoSensitive: { type: 'boolean' },
 		ffVisibility: { type: 'string', enum: ['public', 'followers', 'private'] },
 		pinnedPageId: { type: 'array', items: {
 			type: 'string', format: 'misskey:id',
@@ -168,6 +170,7 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 	if (typeof ps.injectFeaturedNote === 'boolean') profileUpdates.injectFeaturedNote = ps.injectFeaturedNote;
 	if (typeof ps.receiveAnnouncementEmail === 'boolean') profileUpdates.receiveAnnouncementEmail = ps.receiveAnnouncementEmail;
 	if (typeof ps.alwaysMarkNsfw === 'boolean') profileUpdates.alwaysMarkNsfw = ps.alwaysMarkNsfw;
+	if (typeof ps.autoSensitive === 'boolean') profileUpdates.autoSensitive = ps.autoSensitive;
 	if (ps.emailNotificationTypes !== undefined) profileUpdates.emailNotificationTypes = ps.emailNotificationTypes;
 
 	if (ps.avatarId) {

packages/backend/src/services/detect-sensitive.ts

@@ -0,0 +1,28 @@
+import * as fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname } from 'node:path';
+import * as nsfw from 'nsfwjs';
+import * as tf from '@tensorflow/tfjs-node';
+
+const _filename = fileURLToPath(import.meta.url);
+const _dirname = dirname(_filename);
+
+let model: nsfw.NSFWJS;
+
+export async function detectSensitive(path: string): Promise<nsfw.predictionType[] | null> {
+	try {
+		if (model == null) model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
+
+		const buffer = await fs.promises.readFile(path);
+		const image = await tf.node.decodeImage(buffer, 3) as tf.Tensor3D;
+		try {
+			const predictions = await model.classify(image);
+			return predictions;
+		} finally {
+			image.dispose();
+		}
+	} catch (err) {
+		console.error(err);
+		return null;
+	}
+}

packages/backend/src/services/drive/add-file.ts

@@ -16,6 +16,7 @@ import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/i
 import { genId } from '@/misc/gen-id.js';
 import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
+import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { getS3 } from './s3.js';
 import { InternalStorage } from './internal-storage.js';
 import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
@@ -349,9 +350,31 @@ export async function addFile({
 	requestIp = null,
 	requestHeaders = null,
 }: AddFileArgs): Promise<DriveFile> {
-	const info = await getFileInfo(path);
+	let skipNsfwCheck = false;
+	const instance = await fetchMeta();
+	if (user == null) skipNsfwCheck = true;
+	if (instance.sensitiveMediaDetection === 'none') skipNsfwCheck = true;
+	if (user && instance.sensitiveMediaDetection === 'local' && Users.isRemoteUser(user)) skipNsfwCheck = true;
+	if (user && instance.sensitiveMediaDetection === 'remote' && Users.isLocalUser(user)) skipNsfwCheck = true;
+
+	const info = await getFileInfo(path, {
+		skipSensitiveDetection: skipNsfwCheck,
+		sensitiveThreshold: // 感度が高いほどしきい値は低くすることになる
+			instance.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 0.1 :
+			instance.sensitiveMediaDetectionSensitivity === 'high' ? 0.3 :
+			instance.sensitiveMediaDetectionSensitivity === 'low' ? 0.7 :
+			instance.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0.9 :
+			0.5,
+		sensitiveThresholdForPorn: 0.75,
+		enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
+	});
 	logger.info(`${JSON.stringify(info)}`);
 
+	// 現状 false positive が多すぎて実用に耐えない
+	//if (info.porn && instance.disallowUploadWhenPredictedAsPorn) {
+	//	throw new IdentifiableError('282f77bf-5816-4f72-9264-aa14d8261a21', 'Detected as porn.');
+	//}
+
 	// detect name
 	const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : 'untitled');
 
@@ -387,7 +410,7 @@ export async function addFile({
 		// If usage limit exceeded
 		if (usage + info.size > driveCapacity) {
 			if (Users.isLocalUser(user)) {
-				throw new Error('no-free-space');
+				throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.');
 			} else {
 				// (アバターまたはバナーを含まず)最も古いファイルを削除する
 				deleteOldFile(await Users.findOneByOrFail({ id: user.id }) as IRemoteUser);
@@ -441,6 +464,8 @@ export async function addFile({
 	file.isLink = isLink;
 	file.requestIp = requestIp;
 	file.requestHeaders = requestHeaders;
+	file.maybeSensitive = info.sensitive;
+	file.maybePorn = info.porn;
 	file.isSensitive = user
 		? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
 		(sensitive !== null && sensitive !== undefined)
@@ -448,6 +473,9 @@ export async function addFile({
 			: false
 		: false;
 
+	if (info.sensitive && profile!.autoSensitive) file.isSensitive = true;
+	if (info.sensitive && instance.setSensitiveFlagAutomatically) file.isSensitive = true;
+
 	if (url !== null) {
 		file.src = url;
 

packages/backend/test/get-file-info.ts

@@ -10,9 +10,10 @@ const _dirname = dirname(_filename);
 describe('Get file info', () => {
 	it('Empty file', async (async () => {
 		const path = `${_dirname}/resources/emptyfile`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 0,
 			md5: 'd41d8cd98f00b204e9800998ecf8427e',
@@ -28,9 +29,10 @@ describe('Get file info', () => {
 
 	it('Generic JPEG', async (async () => {
 		const path = `${_dirname}/resources/Lenna.jpg`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 25360,
 			md5: '091b3f259662aa31e2ffef4519951168',
@@ -46,9 +48,10 @@ describe('Get file info', () => {
 
 	it('Generic APNG', async (async () => {
 		const path = `${_dirname}/resources/anime.png`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 1868,
 			md5: '08189c607bea3b952704676bb3c979e0',
@@ -64,9 +67,10 @@ describe('Get file info', () => {
 
 	it('Generic AGIF', async (async () => {
 		const path = `${_dirname}/resources/anime.gif`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 2248,
 			md5: '32c47a11555675d9267aee1a86571e7e',
@@ -82,9 +86,10 @@ describe('Get file info', () => {
 
 	it('PNG with alpha', async (async () => {
 		const path = `${_dirname}/resources/with-alpha.png`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 3772,
 			md5: 'f73535c3e1e27508885b69b10cf6e991',
@@ -100,9 +105,10 @@ describe('Get file info', () => {
 
 	it('Generic SVG', async (async () => {
 		const path = `${_dirname}/resources/image.svg`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 505,
 			md5: 'b6f52b4b021e7b92cdd04509c7267965',
@@ -119,9 +125,10 @@ describe('Get file info', () => {
 	it('SVG with XML definition', async (async () => {
 		// https://github.com/misskey-dev/misskey/issues/4413
 		const path = `${_dirname}/resources/with-xml-def.svg`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 544,
 			md5: '4b7a346cde9ccbeb267e812567e33397',
@@ -137,9 +144,10 @@ describe('Get file info', () => {
 
 	it('Dimension limit', async (async () => {
 		const path = `${_dirname}/resources/25000x25000.png`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 75933,
 			md5: '268c5dde99e17cf8fe09f1ab3f97df56',
@@ -155,9 +163,10 @@ describe('Get file info', () => {
 
 	it('Rotate JPEG', async (async () => {
 		const path = `${_dirname}/resources/rotate.jpg`;
-		const info = await getFileInfo(path) as any;
+		const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
 		delete info.warnings;
 		delete info.blurhash;
+		delete info.sensitive;
 		assert.deepStrictEqual(info, {
 			size: 12624,
 			md5: '68d5b2d8d1d1acbbce99203e3ec3857e',

packages/client/src/pages/admin/security.vue

@@ -14,6 +14,49 @@
 					<XBotProtection/>
 				</FormFolder>
 
+				<FormFolder class="_formBlock">
+					<template #icon><i class="fas fa-eye-slash"></i></template>
+					<template #label>{{ i18n.ts.sensitiveMediaDetection }}</template>
+					<template v-if="sensitiveMediaDetection === 'all'" #suffix>{{ i18n.ts.all }}</template>
+					<template v-else-if="sensitiveMediaDetection === 'local'" #suffix>{{ i18n.ts.localOnly }}</template>
+					<template v-else-if="sensitiveMediaDetection === 'remote'" #suffix>{{ i18n.ts.remoteOnly }}</template>
+					<template v-else #suffix>{{ i18n.ts.none }}</template>
+
+					<div class="_formRoot">
+						<span class="_formBlock">{{ i18n.ts._sensitiveMediaDetection.description }}</span>
+
+						<FormRadios v-model="sensitiveMediaDetection" class="_formBlock">
+							<option value="none">{{ i18n.ts.none }}</option>
+							<option value="all">{{ i18n.ts.all }}</option>
+							<option value="local">{{ i18n.ts.localOnly }}</option>
+							<option value="remote">{{ i18n.ts.remoteOnly }}</option>
+						</FormRadios>
+
+						<FormRange v-model="sensitiveMediaDetectionSensitivity" :min="0" :max="4" :step="1" :text-converter="(v) => `${v + 1}`" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.sensitivity }}</template>
+							<template #caption>{{ i18n.ts._sensitiveMediaDetection.sensitivityDescription }}</template>
+						</FormRange>
+
+						<FormSwitch v-model="enableSensitiveMediaDetectionForVideos" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.analyzeVideos }}<span class="_beta">{{ i18n.ts.beta }}</span></template>
+							<template #caption>{{ i18n.ts._sensitiveMediaDetection.analyzeVideosDescription }}</template>
+						</FormSwitch>
+
+						<FormSwitch v-model="setSensitiveFlagAutomatically" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.setSensitiveFlagAutomatically }} ({{ i18n.ts.notRecommended }})</template>
+							<template #caption>{{ i18n.ts._sensitiveMediaDetection.setSensitiveFlagAutomaticallyDescription }}</template>
+						</FormSwitch>
+
+						<!-- 現状 false positive が多すぎて実用に耐えない
+						<FormSwitch v-model="disallowUploadWhenPredictedAsPorn" class="_formBlock">
+							<template #label>{{ i18n.ts._sensitiveMediaDetection.disallowUploadWhenPredictedAsPorn }}</template>
+						</FormSwitch>
+						-->
+
+						<FormButton primary class="_formBlock" @click="save"><i class="fas fa-save"></i> {{ i18n.ts.save }}</FormButton>
+					</div>
+				</FormFolder>
+
 				<FormFolder class="_formBlock">
 					<template #label>Log IP address</template>
 					<template v-if="enableIpLogging" #suffix>Enabled</template>
@@ -49,10 +92,11 @@ import { } from 'vue';
 import XBotProtection from './bot-protection.vue';
 import XHeader from './_header_.vue';
 import FormFolder from '@/components/form/folder.vue';
+import FormRadios from '@/components/form/radios.vue';
 import FormSwitch from '@/components/form/switch.vue';
 import FormInfo from '@/components/ui/info.vue';
 import FormSuspense from '@/components/form/suspense.vue';
-import FormSection from '@/components/form/section.vue';
+import FormRange from '@/components/form/range.vue';
 import FormInput from '@/components/form/input.vue';
 import FormButton from '@/components/ui/button.vue';
 import * as os from '@/os';
@@ -63,6 +107,10 @@ import { definePageMetadata } from '@/scripts/page-metadata';
 let summalyProxy: string = $ref('');
 let enableHcaptcha: boolean = $ref(false);
 let enableRecaptcha: boolean = $ref(false);
+let sensitiveMediaDetection: string = $ref('none');
+let sensitiveMediaDetectionSensitivity: number = $ref(0);
+let setSensitiveFlagAutomatically: boolean = $ref(false);
+let enableSensitiveMediaDetectionForVideos: boolean = $ref(false);
 let enableIpLogging: boolean = $ref(false);
 
 async function init() {
@@ -70,12 +118,31 @@ async function init() {
 	summalyProxy = meta.summalyProxy;
 	enableHcaptcha = meta.enableHcaptcha;
 	enableRecaptcha = meta.enableRecaptcha;
+	sensitiveMediaDetection = meta.sensitiveMediaDetection;
+	sensitiveMediaDetectionSensitivity =
+		meta.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0 :
+		meta.sensitiveMediaDetectionSensitivity === 'low' ? 1 :
+		meta.sensitiveMediaDetectionSensitivity === 'medium' ? 2 :
+		meta.sensitiveMediaDetectionSensitivity === 'high' ? 3 :
+		meta.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 4 : 0;
+	setSensitiveFlagAutomatically = meta.setSensitiveFlagAutomatically;
+	enableSensitiveMediaDetectionForVideos = meta.enableSensitiveMediaDetectionForVideos;
 	enableIpLogging = meta.enableIpLogging;
 }
 
 function save() {
 	os.apiWithDialog('admin/update-meta', {
 		summalyProxy,
+		sensitiveMediaDetection,
+		sensitiveMediaDetectionSensitivity:
+			sensitiveMediaDetectionSensitivity === 0 ? 'veryLow' :
+			sensitiveMediaDetectionSensitivity === 1 ? 'low' :
+			sensitiveMediaDetectionSensitivity === 2 ? 'medium' :
+			sensitiveMediaDetectionSensitivity === 3 ? 'high' :
+			sensitiveMediaDetectionSensitivity === 4 ? 'veryHigh' :
+			0,
+		setSensitiveFlagAutomatically,
+		enableSensitiveMediaDetectionForVideos,
 		enableIpLogging,
 	}).then(() => {
 		fetchInstance();

packages/client/src/pages/settings/drive.vue

@@ -28,7 +28,17 @@
 			<template #suffix>{{ uploadFolder ? uploadFolder.name : '-' }}</template>
 			<template #suffixIcon><i class="fas fa-folder-open"></i></template>
 		</FormLink>
-		<FormSwitch v-model="keepOriginalUploading" class="_formBlock">{{ i18n.ts.keepOriginalUploading }}<template #caption>{{ i18n.ts.keepOriginalUploadingDescription }}</template></FormSwitch>
+		<FormSwitch v-model="keepOriginalUploading" class="_formBlock">
+			<template #label>{{ i18n.ts.keepOriginalUploading }}</template>
+			<template #caption>{{ i18n.ts.keepOriginalUploadingDescription }}</template>
+		</FormSwitch>
+		<FormSwitch v-model="alwaysMarkNsfw" class="_formBlock" @update:modelValue="saveProfile()">
+			<template #label>{{ i18n.ts.alwaysMarkSensitive }}</template>
+		</FormSwitch>
+		<FormSwitch v-model="autoSensitive" class="_formBlock" @update:modelValue="saveProfile()">
+			<template #label>{{ i18n.ts.enableAutoSensitive }}<span class="_beta">{{ i18n.ts.beta }}</span></template>
+			<template #caption>{{ i18n.ts.enableAutoSensitiveDescription }}</template>
+		</FormSwitch>
 	</FormSection>
 </div>
 </template>
@@ -47,11 +57,14 @@ import { defaultStore } from '@/store';
 import MkChart from '@/components/chart.vue';
 import { i18n } from '@/i18n';
 import { definePageMetadata } from '@/scripts/page-metadata';
+import { $i } from '@/account';
 
 const fetching = ref(true);
 const usage = ref<any>(null);
 const capacity = ref<any>(null);
 const uploadFolder = ref<any>(null);
+let alwaysMarkNsfw = $ref($i.alwaysMarkNsfw);
+let autoSensitive = $ref($i.autoSensitive);
 
 const meterStyle = computed(() => {
 	return {
@@ -94,6 +107,13 @@ function chooseUploadFolder() {
 	});
 }
 
+function saveProfile() {
+	os.api('i/update', {
+		alwaysMarkNsfw: !!alwaysMarkNsfw,
+		autoSensitive: !!autoSensitive,
+	});
+}
+
 const headerActions = $computed(() => []);
 
 const headerTabs = $computed(() => []);

packages/client/src/pages/settings/profile.vue

@@ -56,8 +56,6 @@
 	<FormSwitch v-model="profile.isCat" class="_formBlock">{{ i18n.ts.flagAsCat }}<template #caption>{{ i18n.ts.flagAsCatDescription }}</template></FormSwitch>
 	<FormSwitch v-model="profile.showTimelineReplies" class="_formBlock">{{ i18n.ts.flagShowTimelineReplies }}<template #caption>{{ i18n.ts.flagShowTimelineRepliesDescription }} {{ i18n.ts.reflectMayTakeTime }}</template></FormSwitch>
 	<FormSwitch v-model="profile.isBot" class="_formBlock">{{ i18n.ts.flagAsBot }}<template #caption>{{ i18n.ts.flagAsBotDescription }}</template></FormSwitch>
-
-	<FormSwitch v-model="profile.alwaysMarkNsfw" class="_formBlock">{{ i18n.ts.alwaysMarkSensitive }}</FormSwitch>
 </div>
 </template>
 
@@ -88,7 +86,6 @@ const profile = reactive({
 	isBot: $i.isBot,
 	isCat: $i.isCat,
 	showTimelineReplies: $i.showTimelineReplies,
-	alwaysMarkNsfw: $i.alwaysMarkNsfw,
 });
 
 watch(() => profile, () => {
@@ -126,7 +123,6 @@ function save() {
 		isBot: !!profile.isBot,
 		isCat: !!profile.isCat,
 		showTimelineReplies: !!profile.showTimelineReplies,
-		alwaysMarkNsfw: !!profile.alwaysMarkNsfw,
 	});
 }
 

packages/client/src/scripts/select-file.ts

@@ -1,9 +1,9 @@
 import { ref } from 'vue';
+import { DriveFile } from 'misskey-js/built/entities';
 import * as os from '@/os';
 import { stream } from '@/stream';
 import { i18n } from '@/i18n';
 import { defaultStore } from '@/store';
-import { DriveFile } from 'misskey-js/built/entities';
 import { uploadFile } from '@/scripts/upload';
 
 function select(src: any, label: string | null, multiple: boolean): Promise<DriveFile | DriveFile[]> {
@@ -20,10 +20,7 @@ function select(src: any, label: string | null, multiple: boolean): Promise<Driv
 				Promise.all(promises).then(driveFiles => {
 					res(multiple ? driveFiles : driveFiles[0]);
 				}).catch(err => {
-					os.alert({
-						type: 'error',
-						text: err
-					});
+					// アップロードのエラーは uploadFile 内でハンドリングされているためアラートダイアログを出したりはしてはいけない
 				});
 
 				// 一応廃棄
@@ -47,7 +44,7 @@ function select(src: any, label: string | null, multiple: boolean): Promise<Driv
 			os.inputText({
 				title: i18n.ts.uploadFromUrl,
 				type: 'url',
-				placeholder: i18n.ts.uploadFromUrlDescription
+				placeholder: i18n.ts.uploadFromUrlDescription,
 			}).then(({ canceled, result: url }) => {
 				if (canceled) return;
 
@@ -64,35 +61,35 @@ function select(src: any, label: string | null, multiple: boolean): Promise<Driv
 				os.api('drive/files/upload-from-url', {
 					url: url,
 					folderId: defaultStore.state.uploadFolder,
-					marker
+					marker,
 				});
 
 				os.alert({
 					title: i18n.ts.uploadFromUrlRequested,
-					text: i18n.ts.uploadFromUrlMayTakeTime
+					text: i18n.ts.uploadFromUrlMayTakeTime,
 				});
 			});
 		};
 
 		os.popupMenu([label ? {
 			text: label,
-			type: 'label'
+			type: 'label',
 		} : undefined, {
 			type: 'switch',
 			text: i18n.ts.keepOriginalUploading,
-			ref: keepOriginal
+			ref: keepOriginal,
 		}, {
 			text: i18n.ts.upload,
 			icon: 'fas fa-upload',
-			action: chooseFileFromPc
+			action: chooseFileFromPc,
 		}, {
 			text: i18n.ts.fromDrive,
 			icon: 'fas fa-cloud',
-			action: chooseFileFromDrive
+			action: chooseFileFromDrive,
 		}, {
 			text: i18n.ts.fromUrl,
 			icon: 'fas fa-link',
-			action: chooseFileFromUrl
+			action: chooseFileFromUrl,
 		}], src);
 	});
 }

packages/client/src/scripts/upload.ts

@@ -5,6 +5,7 @@ import { defaultStore } from '@/store';
 import { apiUrl } from '@/config';
 import { $i } from '@/account';
 import { alert } from '@/os';
+import { i18n } from '@/i18n';
 
 type Uploading = {
 	id: string;
@@ -80,14 +81,37 @@ export function uploadFile(
 			xhr.open('POST', apiUrl + '/drive/files/create', true);
 			xhr.onload = (ev) => {
 				if (xhr.status !== 200 || ev.target == null || ev.target.response == null) {
-					// TODO: 消すのではなくて再送できるようにしたい
+					// TODO: 消すのではなくて(ネットワーク的なエラーなら)再送できるようにしたい
 					uploads.value = uploads.value.filter(x => x.id !== id);
 
-					alert({
-						type: 'error',
-						title: 'Failed to upload',
-						text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}`,
-					});
+					if (ev.target?.response) {
+						const res = JSON.parse(ev.target.response);
+						if (res.error?.id === 'bec5bd69-fba3-43c9-b4fb-2894b66ad5d2') {
+							alert({
+								type: 'error',
+								title: i18n.ts.failedToUpload,
+								text: i18n.ts.cannotUploadBecauseInappropriate,
+							});
+						} else if (res.error?.id === 'd08dbc37-a6a9-463a-8c47-96c32ab5f064') {
+							alert({
+								type: 'error',
+								title: i18n.ts.failedToUpload,
+								text: i18n.ts.cannotUploadBecauseNoFreeSpace,
+							});
+						} else {
+							alert({
+								type: 'error',
+								title: i18n.ts.failedToUpload,
+								text: `${res.error?.message}\n${res.error?.code}\n${res.error?.id}`,
+							});
+						}
+					} else {
+						alert({
+							type: 'error',
+							title: 'Failed to upload',
+							text: `${JSON.stringify(ev.target?.response)}, ${JSON.stringify(xhr.response)}`,
+						});
+					}
 
 					reject();
 					return;

packages/client/src/style.scss

@@ -399,6 +399,16 @@ hr {
 	}
 }
 
+._beta {
+	margin-left: 0.7em;
+	font-size: 65%;
+	padding: 2px 3px;
+	color: var(--accent);
+	border: solid 1px var(--accent);
+	border-radius: 4px;
+	vertical-align: top;
+}
+
 ._table {
 	> ._row {
 		display: flex;