From e05b0d0e97a0a00c72327ba11b8724c0da3678ae Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 1 Jun 2023 20:53:13 +0200 Subject: [PATCH 1/5] Import emojis without meta.json --- .../processors/db/import-custom-emojis.ts | 127 +++++++++++++----- 1 file changed, 91 insertions(+), 36 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index e2454405f..77eb130a5 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -11,6 +11,7 @@ import { addFile } from "@/services/drive/add-file.js"; import { genId } from "@/misc/gen-id.js"; import { db } from "@/db/postgre.js"; import probeImageSize from "probe-image-size"; +import * as path from "path"; const logger = queueLogger.createSubLogger("import-custom-emojis"); @@ -29,11 +30,11 @@ export async function importCustomEmojis( return; } - const [path, cleanup] = await createTempDir(); + const [tempPath, cleanup] = await createTempDir(); - logger.info(`Temp dir is ${path}`); + logger.info(`Temp dir is ${tempPath}`); - const destPath = `${path}/emojis.zip`; + const destPath = `${tempPath}/emojis.zip`; try { fs.writeFileSync(destPath, "", "binary"); @@ -46,44 +47,98 @@ export async function importCustomEmojis( throw e; } - const outputPath = `${path}/emojis`; + const outputPath = `${tempPath}/emojis`; const unzipStream = fs.createReadStream(destPath); const zip = new AdmZip(destPath); zip.extractAllToAsync(outputPath, true, false, async (error) => { if (error) throw error; - const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8"); - const meta = JSON.parse(metaRaw); - for (const record of meta.emojis) { - if (!record.downloaded) continue; - const emojiInfo = record.emoji; - const emojiPath = `${outputPath}/${record.fileName}`; - await Emojis.delete({ - name: emojiInfo.name, - }); - const driveFile = await addFile({ - user: null, - path: emojiPath, - name: record.fileName, - force: true, - }); - const file = fs.createReadStream(emojiPath); - const size = await probeImageSize(file); - file.destroy(); - await Emojis.insert({ - id: genId(), - updatedAt: new Date(), - name: emojiInfo.name, - category: emojiInfo.category, - host: null, - aliases: emojiInfo.aliases, - originalUrl: driveFile.url, - publicUrl: driveFile.webpublicUrl ?? driveFile.url, - type: driveFile.webpublicType ?? driveFile.type, - license: emojiInfo.license, - width: size.width || null, - height: size.height || null, - }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); + if (fs.existsSync(`${outputPath}/meta.json`)) { + logger.info("starting emoji import with metadata") + const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8"); + const meta = JSON.parse(metaRaw); + + for (const record of meta.emojis) { + if (!record.downloaded) continue; + const emojiInfo = record.emoji; + const emojiPath = `${outputPath}/${record.fileName}`; + await Emojis.delete({ + name: emojiInfo.name, + }); + const driveFile = await addFile({ + user: null, + path: emojiPath, + name: record.fileName, + force: true, + }); + const file = fs.createReadStream(emojiPath); + const size = await probeImageSize(file); + file.destroy(); + await Emojis.insert({ + id: genId(), + updatedAt: new Date(), + name: emojiInfo.name, + category: emojiInfo.category, + host: null, + aliases: emojiInfo.aliases, + originalUrl: driveFile.url, + publicUrl: driveFile.webpublicUrl ?? driveFile.url, + type: driveFile.webpublicType ?? driveFile.type, + license: emojiInfo.license, + width: size.width || null, + height: size.height || null, + }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); + } + } else { + logger.info("starting emoji import without metadata") + // Since we lack metadata, we import into a randomized category name instead + let categoryName = genId(); + + let containedEmojis = fs.readdirSync(outputPath); + + // Filter directories + containedEmojis = containedEmojis.filter( + (emoji) => !fs.lstatSync(`${outputPath}/${emoji}`).isDirectory(), + ); + + // Filter out accidental JSON files + containedEmojis = containedEmojis.filter((emoji) => + emoji.match(/\.(json)$/i), + ); + + // only go one layer deep, recursion would be weird + containedEmojis.filter((emoji) => !emoji.includes("/")); + + for (const emojiPath of containedEmojis) { + // strip extension and get filename to use as name + const name = path.basename(emojiPath, path.extname(emojiPath)); + await Emojis.delete({ + name: name, + }); + const driveFile = await addFile({ + user: null, + path: emojiPath, + name: path.basename(emojiPath), + force: true, + }); + const file = fs.createReadStream(emojiPath); + const size = await probeImageSize(file); + file.destroy(); + await Emojis.insert({ + id: genId(), + updatedAt: new Date(), + name: name, + category: categoryName, + host: null, + aliases: [], + originalUrl: driveFile.url, + publicUrl: driveFile.webpublicUrl ?? driveFile.url, + type: driveFile.webpublicType ?? driveFile.type, + license: null, + width: size.width || null, + height: size.height || null, + }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); + } } await db.queryResultCache!.remove(["meta_emojis"]); From 6146ca270a6e99160d799dcd0f8ef74e3bf41132 Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 1 Jun 2023 21:01:15 +0200 Subject: [PATCH 2/5] more logging --- .../backend/src/queue/processors/db/import-custom-emojis.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 77eb130a5..d80ff988b 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -112,6 +112,8 @@ export async function importCustomEmojis( for (const emojiPath of containedEmojis) { // strip extension and get filename to use as name const name = path.basename(emojiPath, path.extname(emojiPath)); + logger.info(`importing ${name}`) + await Emojis.delete({ name: name, }); @@ -124,6 +126,8 @@ export async function importCustomEmojis( const file = fs.createReadStream(emojiPath); const size = await probeImageSize(file); file.destroy(); + logger.info(`emoji size: ${size.width}x${size.height}`) + await Emojis.insert({ id: genId(), updatedAt: new Date(), From e705b240058da574895077dbeab4be7f1f3961ea Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 1 Jun 2023 21:12:03 +0200 Subject: [PATCH 3/5] me when I lack a brain --- .../backend/src/queue/processors/db/import-custom-emojis.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index d80ff988b..7dbb525d3 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -106,9 +106,6 @@ export async function importCustomEmojis( emoji.match(/\.(json)$/i), ); - // only go one layer deep, recursion would be weird - containedEmojis.filter((emoji) => !emoji.includes("/")); - for (const emojiPath of containedEmojis) { // strip extension and get filename to use as name const name = path.basename(emojiPath, path.extname(emojiPath)); From 2c2532b87d302dae2d2d0e0b3d92ec1a98e4e45b Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 1 Jun 2023 21:30:23 +0200 Subject: [PATCH 4/5] missing inversion + use proper emoji path --- .../queue/processors/db/import-custom-emojis.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 7dbb525d3..a40f2c5cb 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -96,19 +96,16 @@ export async function importCustomEmojis( let containedEmojis = fs.readdirSync(outputPath); - // Filter directories - containedEmojis = containedEmojis.filter( - (emoji) => !fs.lstatSync(`${outputPath}/${emoji}`).isDirectory(), - ); - // Filter out accidental JSON files containedEmojis = containedEmojis.filter((emoji) => - emoji.match(/\.(json)$/i), + !emoji.match(/\.(json)$/i), ); - for (const emojiPath of containedEmojis) { + for (const emojiFilename of containedEmojis) { // strip extension and get filename to use as name - const name = path.basename(emojiPath, path.extname(emojiPath)); + const name = path.basename(emojiFilename, path.extname(emojiFilename)); + const emojiPath = `${outputPath}/${emojiFilename}`; + logger.info(`importing ${name}`) await Emojis.delete({ @@ -117,7 +114,7 @@ export async function importCustomEmojis( const driveFile = await addFile({ user: null, path: emojiPath, - name: path.basename(emojiPath), + name: path.basename(emojiFilename), force: true, }); const file = fs.createReadStream(emojiPath); From cc2f5b516fd0e21e3d63905cc96d3d84369261f8 Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Thu, 1 Jun 2023 21:38:37 +0200 Subject: [PATCH 5/5] formatter --- .../src/queue/processors/db/import-custom-emojis.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index a40f2c5cb..9e8b3b174 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -54,7 +54,7 @@ export async function importCustomEmojis( if (error) throw error; if (fs.existsSync(`${outputPath}/meta.json`)) { - logger.info("starting emoji import with metadata") + logger.info("starting emoji import with metadata"); const metaRaw = fs.readFileSync(`${outputPath}/meta.json`, "utf-8"); const meta = JSON.parse(metaRaw); @@ -90,15 +90,15 @@ export async function importCustomEmojis( }).then((x) => Emojis.findOneByOrFail(x.identifiers[0])); } } else { - logger.info("starting emoji import without metadata") + logger.info("starting emoji import without metadata"); // Since we lack metadata, we import into a randomized category name instead let categoryName = genId(); let containedEmojis = fs.readdirSync(outputPath); // Filter out accidental JSON files - containedEmojis = containedEmojis.filter((emoji) => - !emoji.match(/\.(json)$/i), + containedEmojis = containedEmojis.filter( + (emoji) => !emoji.match(/\.(json)$/i), ); for (const emojiFilename of containedEmojis) { @@ -106,7 +106,7 @@ export async function importCustomEmojis( const name = path.basename(emojiFilename, path.extname(emojiFilename)); const emojiPath = `${outputPath}/${emojiFilename}`; - logger.info(`importing ${name}`) + logger.info(`importing ${name}`); await Emojis.delete({ name: name, @@ -120,7 +120,7 @@ export async function importCustomEmojis( const file = fs.createReadStream(emojiPath); const size = await probeImageSize(file); file.destroy(); - logger.info(`emoji size: ${size.width}x${size.height}`) + logger.info(`emoji size: ${size.width}x${size.height}`); await Emojis.insert({ id: genId(),