From 05854c53ad50136eebbe1bc83a3598ff42d443e3 Mon Sep 17 00:00:00 2001
From: ThatOneCalculator <kainoa@t1c.dev>
Date: Mon, 8 Aug 2022 15:40:06 -0700
Subject: [PATCH] Start work on OCR

---
 CALCKEY.md                                    |   1 +
 locales/en-US.yml                             |   1 +
 locales/ja-JP.yml                             |   1 +
 package.json                                  |   2 +-
 packages/client/package.json                  |   1 +
 .../src/components/gallery-post-preview.vue   |   2 +-
 .../client/src/components/media-caption.vue   |  84 +++++++-----
 yarn.lock                                     | 126 +++++++++++++++++-
 8 files changed, 181 insertions(+), 37 deletions(-)

diff --git a/CALCKEY.md b/CALCKEY.md
index f977f35ab..1e4a23316 100644
--- a/CALCKEY.md
+++ b/CALCKEY.md
@@ -20,6 +20,7 @@
 
 ## Work in progress
 
+- OCR image/video captioning
 - Recommended instances timeline
 - Admin custom CSS
 - Improve accesibility score
diff --git a/locales/en-US.yml b/locales/en-US.yml
index c5c43bf1a..d7e8c224b 100644
--- a/locales/en-US.yml
+++ b/locales/en-US.yml
@@ -911,6 +911,7 @@ customSplashIconsDescription: "URLs for custom splash screen icons separated by
 showUpdates: "Show a popup when Calckey updates"
 recommendedInstances: "Recommended instances"
 recommendedInstancesDescription: "Recommended instances seperated by line breaks to appear in the recommended timeline. Do NOT add `https://`, ONLY the domain."
+caption: "Auto Caption"
 
 _sensitiveMediaDetection:
   description: "Reduces the effort of server moderation through automatically recognizing NSFW media via Machine Learning. This will slightly increase the load on the server."
diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
index c80ca8e9b..ac7154de2 100644
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@@ -911,6 +911,7 @@ customSplashIconsDescription: "ユーザがページをロード/リロードす
 showUpdates: "Calckeyの更新時にポップアップを表示する"
 recommendedInstances: "推奨インスタンス"
 recommendedInstancesDescription: "推奨タイムラインに表示するために改行で区切られた推奨インスタンス。`https：//`を追加しないでください。ドメインのみを追加してください。"
+caption: "自動キャプション"
 
 _sensitiveMediaDetection:
   description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てることができます。サーバーの負荷が少し増えます。"
diff --git a/package.json b/package.json
index 5fda2f7c8..fa3d8dc80 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "misskey",
-	"version": "12.118.1-calc.rc.3",
+	"version": "12.118.1-calc.rc.4",
 	"codename": "aqua",
 	"repository": {
 		"type": "git",
diff --git a/packages/client/package.json b/packages/client/package.json
index 664d0f7d5..d76eafd5d 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -47,6 +47,7 @@
 		"strict-event-emitter-types": "2.0.0",
 		"stringz": "2.1.0",
 		"syuilo-password-strength": "0.0.1",
+		"tesseract.js": "^2.1.5",
 		"textarea-caret": "3.1.0",
 		"three": "0.143.0",
 		"throttle-debounce": "5.0.0",
diff --git a/packages/client/src/components/gallery-post-preview.vue b/packages/client/src/components/gallery-post-preview.vue
index 824590297..082ac669d 100644
--- a/packages/client/src/components/gallery-post-preview.vue
+++ b/packages/client/src/components/gallery-post-preview.vue
@@ -31,7 +31,7 @@ export default defineComponent({
 		},
 	},
 	methods: {
-		userName
+		userName,
 	}
 });
 </script>
diff --git a/packages/client/src/components/media-caption.vue b/packages/client/src/components/media-caption.vue
index feed3854f..b6285902d 100644
--- a/packages/client/src/components/media-caption.vue
+++ b/packages/client/src/components/media-caption.vue
@@ -1,40 +1,46 @@
 <template>
-	<MkModal ref="modal" @click="done(true)" @closed="$emit('closed')">
-		<div class="container">
-			<div class="fullwidth top-caption">
-				<div class="mk-dialog">
-					<header>
-						<Mfm v-if="title" class="title" :text="title"/>
-						<span class="text-count" :class="{ over: remainingLength < 0 }">{{ remainingLength }}</span>
-					</header>
-					<textarea v-model="inputValue" autofocus :placeholder="input.placeholder" @keydown="onInputKeydown"></textarea>
-					<div v-if="(showOkButton || showCancelButton)" class="buttons">
-						<MkButton inline primary :disabled="remainingLength < 0" @click="ok">{{ $ts.ok }}</MkButton>
-						<MkButton inline @click="cancel" >{{ $ts.cancel }}</MkButton>
-					</div>
+<MkModal ref="modal" @click="done(true)" @closed="$emit('closed')">
+	<div class="container">
+		<div class="fullwidth top-caption">
+			<div class="mk-dialog">
+				<header>
+					<Mfm v-if="title" class="title" :text="title"/>
+					<span class="text-count" :class="{ over: remainingLength < 0 }">{{ remainingLength }}</span>
+				</header>
+				<textarea v-model="inputValue" autofocus :placeholder="input.placeholder" @keydown="onInputKeydown"></textarea>
+				<div v-if="(showOkButton || showCaptionButton || showCancelButton)" class="buttons">
+					<MkButton inline primary :disabled="remainingLength < 0" @click="ok">{{ $ts.ok }}</MkButton>
+					<MkButton inline @click="caption" >{{ $ts.caption }}</MkButton>
+					<MkButton inline @click="cancel" >{{ $ts.cancel }}</MkButton>
 				</div>
 			</div>
-			<div class="hdrwpsaf fullwidth">
-				<header>{{ image.name }}</header>
-				<img :src="image.url" :alt="image.comment" :title="image.comment" @click="$refs.modal.close()"/>
-				<footer>
-					<span>{{ image.type }}</span>
-					<span>{{ bytes(image.size) }}</span>
-					<span v-if="image.properties && image.properties.width">{{ number(image.properties.width) }}px × {{ number(image.properties.height) }}px</span>
-				</footer>
-			</div>
 		</div>
-	</MkModal>
+		<div class="hdrwpsaf fullwidth">
+			<header>{{ image.name }}</header>
+			<img id="imgtocaption" :src="image.url" :alt="image.comment" :title="image.comment" @click="$refs.modal.close()"/>
+			<footer>
+				<span>{{ image.type }}</span>
+				<span>{{ bytes(image.size) }}</span>
+				<span v-if="image.properties && image.properties.width">{{ number(image.properties.width) }}px × {{ number(image.properties.height) }}px</span>
+			</footer>
+		</div>
+	</div>
+</MkModal>
 </template>
 
 <script lang="ts">
 import { defineComponent } from 'vue';
 import { length } from 'stringz';
+import { createWorker, PSM, OEM } from 'tesseract.js';
 import MkModal from '@/components/ui/modal.vue';
 import MkButton from '@/components/ui/button.vue';
 import bytes from '@/filters/bytes';
 import number from '@/filters/number';
 
+const worker = createWorker({
+	logger: m => console.log(m),
+});
+
 export default defineComponent({
 	components: {
 		MkModal,
@@ -48,22 +54,26 @@ export default defineComponent({
 		},
 		title: {
 			type: String,
-			required: false
+			required: false,
 		},
 		input: {
-			required: true
+			required: true,
 		},
 		showOkButton: {
 			type: Boolean,
-			default: true
+			default: true,
+		},
+		showCaptionButton: {
+			type: Boolean,
+			default: true,
 		},
 		showCancelButton: {
 			type: Boolean,
-			default: true
+			default: true,
 		},
 		cancelableByBgClick: {
 			type: Boolean,
-			default: true
+			default: true,
 		},
 	},
 
@@ -93,6 +103,8 @@ export default defineComponent({
 	methods: {
 		bytes,
 		number,
+		PSM,
+		OEM,
 
 		done(canceled, result?) {
 			this.$emit('done', { canceled, result });
@@ -130,8 +142,20 @@ export default defineComponent({
 					this.ok();
 				}
 			}
-		}
-	}
+		},
+
+		caption: async () => {
+			const img = document.getElementById('imgtocaption');
+			await worker.load();
+			await worker.loadLanguage('eng');
+			await worker.initialize('eng', OEM.LSTM_ONLY);
+			await worker.setParameters({
+				tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
+			});
+			const { data: { text } } = await worker.recognize(img);
+			console.log(text);
+		},
+	},
 });
 </script>
 
diff --git a/yarn.lock b/yarn.lock
index bc60047b5..c239f1bee 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3210,6 +3210,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"blueimp-load-image@npm:^3.0.0":
+  version: 3.0.0
+  resolution: "blueimp-load-image@npm:3.0.0"
+  checksum: 19a1e9a76ea87e15c1624c3ed30536f094252f0a65a193239f54ec0c25fe480b5140c962ac41a66f35a2d19b58d9da26803287b5a37b78d82d4b23328b09bf4a
+  languageName: node
+  linkType: hard
+
 "blurhash@npm:1.1.5":
   version: 1.1.5
   resolution: "blurhash@npm:1.1.5"
@@ -3217,6 +3224,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"bmp-js@npm:^0.1.0":
+  version: 0.1.0
+  resolution: "bmp-js@npm:0.1.0"
+  checksum: 2f6cf7eeabae2aa50eb768122f59e9752caa97248028cb8b5cf0d9db7cf8fb3a60262aeb2c6889dd21357ab061b2fb318f21f20d2b24963ba36ead8e264c6654
+  languageName: node
+  linkType: hard
+
 "bn.js@npm:^4.0.0":
   version: 4.12.0
   resolution: "bn.js@npm:4.12.0"
@@ -3950,6 +3964,7 @@ __metadata:
     strict-event-emitter-types: 2.0.0
     stringz: 2.1.0
     syuilo-password-strength: 0.0.1
+    tesseract.js: ^2.1.5
     textarea-caret: 3.1.0
     three: 0.143.0
     throttle-debounce: 5.0.0
@@ -4229,6 +4244,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"colors@npm:^1.4.0":
+  version: 1.4.0
+  resolution: "colors@npm:1.4.0"
+  checksum: 98aa2c2418ad87dedf25d781be69dc5fc5908e279d9d30c34d8b702e586a0474605b3a189511482b9d5ed0d20c867515d22749537f7bc546256c6014f3ebdcec
+  languageName: node
+  linkType: hard
+
 "colors@npm:~1.1.2":
   version: 1.1.2
   resolution: "colors@npm:1.1.2"
@@ -6385,6 +6407,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"file-type@npm:^12.4.1":
+  version: 12.4.2
+  resolution: "file-type@npm:12.4.2"
+  checksum: 67c8d7f8f032fd8cf4d14016d96567d20eeb7bf3524915f2c5d79337ca4e5338032d373a5fe827610eaf4ab7eb80629ff868331a66f63d1f9e9cc4c433e3f047
+  languageName: node
+  linkType: hard
+
 "filelist@npm:^1.0.1":
   version: 1.0.4
   resolution: "filelist@npm:1.0.4"
@@ -7036,7 +7065,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"glob@npm:^7.1.1, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.2.0":
+"glob@npm:^7.1.1, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6, glob@npm:^7.2.0":
   version: 7.2.3
   resolution: "glob@npm:7.2.3"
   dependencies:
@@ -7708,6 +7737,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"idb-keyval@npm:^3.2.0":
+  version: 3.2.0
+  resolution: "idb-keyval@npm:3.2.0"
+  checksum: 3270010af049fa7f07b0381818daa7f16d3704aaba9d2585e531908f09130f6d0772464b2b48c23676dd1e51e5245585da5ce56f2c74dfd4f9961d1e0d0f5909
+  languageName: node
+  linkType: hard
+
 "ieee754@npm:1.1.13":
   version: 1.1.13
   resolution: "ieee754@npm:1.1.13"
@@ -8098,6 +8134,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"is-electron@npm:^2.2.0":
+  version: 2.2.1
+  resolution: "is-electron@npm:2.2.1"
+  checksum: 06e569aa933a737d418489bb9ca081af62eceb714d4c3d553ad2497610e35494be6dddd010c4e29890c7dd9d0481c2e3e1e9097af9d19df1c52dd5be747d80a0
+  languageName: node
+  linkType: hard
+
 "is-expression@npm:^4.0.0":
   version: 4.0.0
   resolution: "is-expression@npm:4.0.0"
@@ -8411,6 +8454,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"is-url@npm:^1.2.4":
+  version: 1.2.4
+  resolution: "is-url@npm:1.2.4"
+  checksum: 100e74b3b1feab87a43ef7653736e88d997eb7bd32e71fd3ebc413e58c1cbe56269699c776aaea84244b0567f2a7d68dfaa512a062293ed2f9fdecb394148432
+  languageName: node
+  linkType: hard
+
 "is-utf8@npm:^0.2.0, is-utf8@npm:^0.2.1":
   version: 0.2.1
   resolution: "is-utf8@npm:0.2.1"
@@ -8536,6 +8586,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"jpeg-autorotate@npm:^7.1.1":
+  version: 7.1.1
+  resolution: "jpeg-autorotate@npm:7.1.1"
+  dependencies:
+    colors: ^1.4.0
+    glob: ^7.1.6
+    jpeg-js: ^0.4.2
+    piexifjs: ^1.0.6
+    yargs-parser: ^20.2.1
+  bin:
+    jpeg-autorotate: src/cli.js
+  checksum: fdd6898ed287e074e26667e503731846ddb3e3042d5aee70f3f6ae86051b8d493bfa0906f9df19a4e5fb0df6253ff059bf8967cda5fb3e6f1c8e8e2d7a58a3ee
+  languageName: node
+  linkType: hard
+
 "jpeg-js@npm:^0.3.2":
   version: 0.3.7
   resolution: "jpeg-js@npm:0.3.7"
@@ -8543,7 +8608,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"jpeg-js@npm:^0.4.1":
+"jpeg-js@npm:^0.4.1, jpeg-js@npm:^0.4.2":
   version: 0.4.4
   resolution: "jpeg-js@npm:0.4.4"
   checksum: bd7cb61aa8df40a9ee2c2106839c3df6054891e56cfc22c0ac581402e06c6295f962a4754b0b2ac50a401789131b1c6dc9df8d24400f1352168be1894833c590
@@ -10316,7 +10381,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:~2.6.1":
+"node-fetch@npm:^2.6.0, node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:~2.6.1":
   version: 2.6.7
   resolution: "node-fetch@npm:2.6.7"
   dependencies:
@@ -10728,6 +10793,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"opencollective-postinstall@npm:^2.0.2":
+  version: 2.0.3
+  resolution: "opencollective-postinstall@npm:2.0.3"
+  bin:
+    opencollective-postinstall: index.js
+  checksum: 0a68c5cef135e46d11e665d5077398285d1ce5311c948e8327b435791c409744d4a6bb9c55bd6507fb5f2ef34b0ad920565adcdaf974cbdae701aead6f32b396
+  languageName: node
+  linkType: hard
+
 "opentype.js@npm:^0.4.3":
   version: 0.4.11
   resolution: "opentype.js@npm:0.4.11"
@@ -11262,6 +11336,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"piexifjs@npm:^1.0.6":
+  version: 1.0.6
+  resolution: "piexifjs@npm:1.0.6"
+  checksum: 137caf5ca36c0ad98cfd445c0ec40737f7589469e750ac417a2019b940cf6534e9834f3f01dc6a85a6d6c127c3eb5b5dc750812c800e6567392b67da94a755bd
+  languageName: node
+  linkType: hard
+
 "pify@npm:^2.0.0, pify@npm:^2.2.0":
   version: 2.3.0
   resolution: "pify@npm:2.3.0"
@@ -12412,7 +12493,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"regenerator-runtime@npm:^0.13.4, regenerator-runtime@npm:^0.13.5":
+"regenerator-runtime@npm:^0.13.3, regenerator-runtime@npm:^0.13.4, regenerator-runtime@npm:^0.13.5":
   version: 0.13.9
   resolution: "regenerator-runtime@npm:0.13.9"
   checksum: 65ed455fe5afd799e2897baf691ca21c2772e1a969d19bb0c4695757c2d96249eb74ee3553ea34a91062b2a676beedf630b4c1551cc6299afb937be1426ec55e
@@ -13895,6 +13976,34 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tesseract.js-core@npm:^2.2.0":
+  version: 2.2.0
+  resolution: "tesseract.js-core@npm:2.2.0"
+  checksum: 7587514a6bd51540ba7bbe1fd6349a83c5865261db43bee1625b07c877ac6949b1bf7e0a9a84b6925f8acfb34287f9c28dfd3f3d1bbaa49abc9cbb6cb7927389
+  languageName: node
+  linkType: hard
+
+"tesseract.js@npm:^2.1.5":
+  version: 2.1.5
+  resolution: "tesseract.js@npm:2.1.5"
+  dependencies:
+    blueimp-load-image: ^3.0.0
+    bmp-js: ^0.1.0
+    file-type: ^12.4.1
+    idb-keyval: ^3.2.0
+    is-electron: ^2.2.0
+    is-url: ^1.2.4
+    jpeg-autorotate: ^7.1.1
+    node-fetch: ^2.6.0
+    opencollective-postinstall: ^2.0.2
+    regenerator-runtime: ^0.13.3
+    resolve-url: ^0.2.1
+    tesseract.js-core: ^2.2.0
+    zlibjs: ^0.3.1
+  checksum: a28a2edae162ab5db97a9337dbce1ea96f3c303f6976560784d9f774be6fec15b05f04b8776fe6b9d12471324ee0c94fe25efd2d991fdb859ad66d24e8d66fb0
+  languageName: node
+  linkType: hard
+
 "text-table@npm:^0.2.0":
   version: 0.2.0
   resolution: "text-table@npm:0.2.0"
@@ -15433,7 +15542,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"yargs-parser@npm:^20.2.2":
+"yargs-parser@npm:^20.2.1, yargs-parser@npm:^20.2.2":
   version: 20.2.9
   resolution: "yargs-parser@npm:20.2.9"
   checksum: 8bb69015f2b0ff9e17b2c8e6bfe224ab463dd00ca211eece72a4cd8a906224d2703fb8a326d36fdd0e68701e201b2a60ed7cf81ce0fd9b3799f9fe7745977ae3
@@ -15573,3 +15682,10 @@ __metadata:
   checksum: 4a73da856738b0634700b52f4ab3fe0bf0a532bea6820ad962d0bda0163d2d5525df4859f89a7238e204a378384e12551985049790c1894c3ac191866e85887f
   languageName: node
   linkType: hard
+
+"zlibjs@npm:^0.3.1":
+  version: 0.3.1
+  resolution: "zlibjs@npm:0.3.1"
+  checksum: 91f3d28bb5925ac71565a1fd4d44303b6b876ed483c9c192864393811151b399f29d917c9118a5aca5a541d9dfc7c199460f86b489ea1f77b7217e0edc887c89
+  languageName: node
+  linkType: hard