161sh/trashposs
1
0
Fork 0
mirror of https://iceshrimp.dev/crimekillz/trashposs synced 2024-11-22 17:03:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

use gunzip-maybe tar-stream to replace exec

Browse Source
This commit is contained in:
CGsama 2023-07-16 01:41:57 -04:00
parent 28b92d8c16
commit 9339868636
2 changed files with 67 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend/package.json
View File

@ -65,6 +65,7 @@
"file-type": "17.1.6", "file-type": "17.1.6",
"fluent-ffmpeg": "2.1.2", "fluent-ffmpeg": "2.1.2",
"got": "12.5.3", "got": "12.5.3",
"gunzip-maybe": "^1.4.2",
"hpagent": "0.1.2", "hpagent": "0.1.2",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"ip-cidr": "3.1.0", "ip-cidr": "3.1.0",
@ -125,6 +126,7 @@
"summaly": "2.7.0", "summaly": "2.7.0",
"syslog-pro": "1.0.0", "syslog-pro": "1.0.0",
"systeminformation": "5.17.17", "systeminformation": "5.17.17",
"tar-stream": "^3.1.6",
"tesseract.js": "^3.0.3", "tesseract.js": "^3.0.3",
"tinycolor2": "1.5.2", "tinycolor2": "1.5.2",
"tmp": "0.2.1", "tmp": "0.2.1",

87
packages/backend/src/misc/process-masto-notes.ts
View File

@ -3,10 +3,11 @@ import Logger from "@/services/logger.js";
import { createTemp, createTempDir } from "./create-temp.js"; import { createTemp, createTempDir } from "./create-temp.js";
import { downloadUrl } from "./download-url.js"; import { downloadUrl } from "./download-url.js";
import { addFile } from "@/services/drive/add-file.js"; import { addFile } from "@/services/drive/add-file.js";
import { exec } from "node:child_process";
import { Users } from "@/models/index.js"; import { Users } from "@/models/index.js";
import * as tar from 'tar-stream';
import gunzip from "gunzip-maybe";
const logger = new Logger("download-text-file"); const logger = new Logger("process-masto-notes");
export async function processMastoNotes( export async function processMastoNotes(
url: string, url: string,
@ -32,27 +33,69 @@ export async function processMastoNotes(
function processMastoFile(fn: string, dir: string, uid: string) { function processMastoFile(fn: string, dir: string, uid: string) {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
const user = await Users.findOneBy({ id: uid }); const user = await Users.findOneBy({ id: uid });
exec( try{
`tar -xf ${fn} -C ${dir}`, logger.info(`Start unzip ${fn}`);
async (error: any, stdout: string, stderr: string) => { await unzipTarGz(fn, dir);
if (error) { logger.info(`Unzip to ${dir}`);
reject(error); const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`));
} for (const note of outbox.orderedItems) {
const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); for (const attachment of note.object.attachment) {
for (const note of outbox.orderedItems) { const url = attachment.url.replace("..", "");
for (const attachment of note.object.attachment) { try {
const url = attachment.url.replace("..", ""); const fpath = `${dir}${url}`;
try { const driveFile = await addFile({ user: user, path: fpath });
const fpath = `${dir}${url}`; attachment.driveFile = driveFile;
const driveFile = await addFile({ user: user, path: fpath }); } catch (e) {
attachment.driveFile = driveFile; logger.error(`Skipped adding file to drive: ${url}`);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
} }
} }
resolve(outbox); }
}, resolve(outbox);
); }catch(e){
logger.error(`Error on extract masto note package: ${fn}`);
reject(e);
}
});
}
function createFileDir(fn: string){
if(!fs.existsSync(fn)){
fs.mkdirSync(fn, {recursive: true});
fs.rmdirSync(fn);
}
}
function unzipTarGz(fn: string, dir: string){
return new Promise(async (resolve, reject) => {
const onErr = (err: any) => {
logger.error(`pipe broken: ${err}`);
reject();
}
try{
const extract = tar.extract().on('error', onErr);
dir = dir.endsWith("/") ? dir : dir + "/";
const ls: string[] = [];
extract.on('entry', function (header: any, stream: any, next: any) {
try{
ls.push(dir + header.name);
createFileDir(dir + header.name);
stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr);
next();
}catch(e){
logger.error(`create dir error:${e}`);
reject();
}
});
extract.on('finish', function () {
resolve(ls);
});
fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr);
}catch(e){
logger.error(`unzipTarGz error: ${e}`);
reject();
}
}); });
} }