From 28b92d8c16b63f9fc5741f8f64c5034cfd31cfe3 Mon Sep 17 00:00:00 2001
From: CGsama <CGsama@outlook.com>
Date: Sat, 15 Jul 2023 23:02:00 -0400
Subject: [PATCH] handle import mastodon package

---
 .../backend/src/misc/process-masto-notes.ts   | 58 +++++++++++++++++++
 .../queue/processors/db/import-masto-post.ts  | 33 ++++++-----
 .../src/queue/processors/db/import-posts.ts   | 17 ++++++
 3 files changed, 95 insertions(+), 13 deletions(-)
 create mode 100644 packages/backend/src/misc/process-masto-notes.ts

diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts
new file mode 100644
index 000000000..1310748d2
--- /dev/null
+++ b/packages/backend/src/misc/process-masto-notes.ts
@@ -0,0 +1,58 @@
+import * as fs from "node:fs";
+import Logger from "@/services/logger.js";
+import { createTemp, createTempDir } from "./create-temp.js";
+import { downloadUrl } from "./download-url.js";
+import { addFile } from "@/services/drive/add-file.js";
+import { exec } from "node:child_process";
+import { Users } from "@/models/index.js";
+
+const logger = new Logger("download-text-file");
+
+export async function processMastoNotes(
+	url: string,
+	uid: string,
+): Promise<any> {
+	// Create temp file
+	const [path, cleanup] = await createTemp();
+
+	const [unzipPath, unzipCleanup] = await createTempDir();
+
+	logger.info(`Temp file is ${path}`);
+
+	try {
+		// write content at URL to temp file
+		await downloadUrl(url, path);
+		return await processMastoFile(path, unzipPath, uid);
+	} finally {
+		cleanup();
+		unzipCleanup();
+	}
+}
+
+function processMastoFile(fn: string, dir: string, uid: string) {
+	return new Promise(async (resolve, reject) => {
+		const user = await Users.findOneBy({ id: uid });
+		exec(
+			`tar -xf ${fn} -C ${dir}`,
+			async (error: any, stdout: string, stderr: string) => {
+				if (error) {
+					reject(error);
+				}
+				const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
+				for (const note of outbox.orderedItems) {
+					for (const attachment of note.object.attachment) {
+						const url = attachment.url.replace("..", "");
+						try {
+							const fpath = `${dir}${url}`;
+							const driveFile = await addFile({ user: user, path: fpath });
+							attachment.driveFile = driveFile;
+						} catch (e) {
+							logger.error(`Skipped adding file to drive: ${url}`);
+						}
+					}
+				}
+				resolve(outbox);
+			},
+		);
+	});
+}
diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts
index 05166b085..1d18008a0 100644
--- a/packages/backend/src/queue/processors/db/import-masto-post.ts
+++ b/packages/backend/src/queue/processors/db/import-masto-post.ts
@@ -45,19 +45,26 @@ export async function importMastoPost(
 		throw e;
 	}
 	job.progress(80);
-	const urls = post.object.attachment
-		.map((x: any) => x.url)
-		.filter((x: String) => x.startsWith("http"));
-	const files: DriveFile[] = [];
-	for (const url of urls) {
-		try {
-			const file = await uploadFromUrl({
-				url: url,
-				user: user,
-			});
-			files.push(file);
-		} catch (e) {
-			logger.error(`Skipped adding file to drive: ${url}`);
+
+	let files: DriveFile[] = (post.object.attachment || [])
+		.map((x: any) => x?.driveFile)
+		.filter((x: any) => x);
+
+	if (files.length == 0) {
+		const urls = post.object.attachment
+			.map((x: any) => x.url)
+			.filter((x: String) => x.startsWith("http"));
+		files = [];
+		for (const url of urls) {
+			try {
+				const file = await uploadFromUrl({
+					url: url,
+					user: user,
+				});
+				files.push(file);
+			} catch (e) {
+				logger.error(`Skipped adding file to drive: ${url}`);
+			}
 		}
 	}
 
diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts
index f92a5f710..54061f8bf 100644
--- a/packages/backend/src/queue/processors/db/import-posts.ts
+++ b/packages/backend/src/queue/processors/db/import-posts.ts
@@ -1,4 +1,5 @@
 import { downloadTextFile } from "@/misc/download-text-file.js";
+import { processMastoNotes } from "@/misc/process-masto-notes.js";
 import { Users, DriveFiles } from "@/models/index.js";
 import type { DbUserImportPostsJobData } from "@/queue/types.js";
 import { queueLogger } from "../../logger.js";
@@ -30,6 +31,22 @@ export async function importPosts(
 		return;
 	}
 
+	if (file.name.endsWith("tar.gz")) {
+		try {
+			logger.info("Parsing animal style posts in package");
+			const outbox = await processMastoNotes(file.url, job.data.user.id);
+			for (const post of outbox.orderedItems) {
+				createImportMastoPostJob(job.data.user, post, job.data.signatureCheck);
+			}
+		} catch (e) {
+			// handle error
+			logger.warn(`Error reading: ${e}`);
+		}
+		logger.succ("Imported");
+		done();
+		return;
+	}
+
 	const json = await downloadTextFile(file.url);
 
 	try {