handle import mastodon package

packages/backend/src/misc/process-masto-notes.ts

@@ -0,0 +1,58 @@
+import * as fs from "node:fs";
+import Logger from "@/services/logger.js";
+import { createTemp, createTempDir } from "./create-temp.js";
+import { downloadUrl } from "./download-url.js";
+import { addFile } from "@/services/drive/add-file.js";
+import { exec } from "node:child_process";
+import { Users } from "@/models/index.js";
+
+const logger = new Logger("download-text-file");
+
+export async function processMastoNotes(
+	url: string,
+	uid: string,
+): Promise<any> {
+	// Create temp file
+	const [path, cleanup] = await createTemp();
+
+	const [unzipPath, unzipCleanup] = await createTempDir();
+
+	logger.info(`Temp file is ${path}`);
+
+	try {
+		// write content at URL to temp file
+		await downloadUrl(url, path);
+		return await processMastoFile(path, unzipPath, uid);
+	} finally {
+		cleanup();
+		unzipCleanup();
+	}
+}
+
+function processMastoFile(fn: string, dir: string, uid: string) {
+	return new Promise(async (resolve, reject) => {
+		const user = await Users.findOneBy({ id: uid });
+		exec(
+			`tar -xf ${fn} -C ${dir}`,
+			async (error: any, stdout: string, stderr: string) => {
+				if (error) {
+					reject(error);
+				}
+				const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
+				for (const note of outbox.orderedItems) {
+					for (const attachment of note.object.attachment) {
+						const url = attachment.url.replace("..", "");
+						try {
+							const fpath = `${dir}${url}`;
+							const driveFile = await addFile({ user: user, path: fpath });
+							attachment.driveFile = driveFile;
+						} catch (e) {
+							logger.error(`Skipped adding file to drive: ${url}`);
+						}
+					}
+				}
+				resolve(outbox);
+			},
+		);
+	});
+}

packages/backend/src/queue/processors/db/import-masto-post.ts

@@ -45,19 +45,26 @@ export async function importMastoPost(
 		throw e;
 	}
 	job.progress(80);
-	const urls = post.object.attachment
+
-		.map((x: any) => x.url)
+	let files: DriveFile[] = (post.object.attachment || [])
-		.filter((x: String) => x.startsWith("http"));
+		.map((x: any) => x?.driveFile)
-	const files: DriveFile[] = [];
+		.filter((x: any) => x);
-	for (const url of urls) {
+
-		try {
+	if (files.length == 0) {
-			const file = await uploadFromUrl({
+		const urls = post.object.attachment
-				url: url,
+			.map((x: any) => x.url)
-				user: user,
+			.filter((x: String) => x.startsWith("http"));
-			});
+		files = [];
-			files.push(file);
+		for (const url of urls) {
-		} catch (e) {
+			try {
-			logger.error(`Skipped adding file to drive: ${url}`);
+				const file = await uploadFromUrl({
+					url: url,
+					user: user,
+				});
+				files.push(file);
+			} catch (e) {
+				logger.error(`Skipped adding file to drive: ${url}`);
+			}
 		}
 	}
 

packages/backend/src/queue/processors/db/import-posts.ts

@@ -1,4 +1,5 @@
 import { downloadTextFile } from "@/misc/download-text-file.js";
+import { processMastoNotes } from "@/misc/process-masto-notes.js";
 import { Users, DriveFiles } from "@/models/index.js";
 import type { DbUserImportPostsJobData } from "@/queue/types.js";
 import { queueLogger } from "../../logger.js";
@@ -30,6 +31,22 @@ export async function importPosts(
 		return;
 	}
 
+	if (file.name.endsWith("tar.gz")) {
+		try {
+			logger.info("Parsing animal style posts in package");
+			const outbox = await processMastoNotes(file.url, job.data.user.id);
+			for (const post of outbox.orderedItems) {
+				createImportMastoPostJob(job.data.user, post, job.data.signatureCheck);
+			}
+		} catch (e) {
+			// handle error
+			logger.warn(`Error reading: ${e}`);
+		}
+		logger.succ("Imported");
+		done();
+		return;
+	}
+
 	const json = await downloadTextFile(file.url);
 
 	try {