From e389529beea37a3f18e1d090e3274870a6e1e422 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Thu, 21 Mar 2019 04:50:44 +0900 Subject: [PATCH] Fix #4546 (#4548) * Refactor download * emoji type --- src/misc/create-temp.ts | 10 +++ src/misc/detect-mine.ts | 31 +++++++ src/misc/detect-url-mine.ts | 15 ++++ src/misc/donwload-url.ts | 61 +++++++++++++ src/misc/download-text-file.ts | 76 +++-------------- src/models/emoji.ts | 1 + src/remote/activitypub/renderer/emoji.ts | 2 +- src/server/api/endpoints/admin/emoji/add.ts | 6 +- .../api/endpoints/admin/emoji/update.ts | 6 +- src/server/proxy/proxy-media.ts | 85 ++----------------- src/services/drive/add-file.ts | 32 +------ src/services/drive/upload-from-url.ts | 63 +------------- 12 files changed, 151 insertions(+), 237 deletions(-) create mode 100644 src/misc/create-temp.ts create mode 100644 src/misc/detect-mine.ts create mode 100644 src/misc/detect-url-mine.ts create mode 100644 src/misc/donwload-url.ts diff --git a/src/misc/create-temp.ts b/src/misc/create-temp.ts new file mode 100644 index 000000000..04604cf7d --- /dev/null +++ b/src/misc/create-temp.ts @@ -0,0 +1,10 @@ +import * as tmp from 'tmp'; + +export function createTemp(): Promise<[string, any]> { + return new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); +} diff --git a/src/misc/detect-mine.ts b/src/misc/detect-mine.ts new file mode 100644 index 000000000..bbf49efc1 --- /dev/null +++ b/src/misc/detect-mine.ts @@ -0,0 +1,31 @@ +import * as fs from 'fs'; +import fileType from 'file-type'; +import checkSvg from '../misc/check-svg'; + +export async function detectMine(path: string) { + return new Promise<[string, string]>((res, rej) => { + const readable = fs.createReadStream(path); + readable + .on('error', rej) + .once('data', (buffer: Buffer) => { + readable.destroy(); + const type = fileType(buffer); + if (type) { + if (type.mime == 'application/xml' && checkSvg(path)) { + res(['image/svg+xml', 'svg']); + } else { + res([type.mime, type.ext]); + } + } else if (checkSvg(path)) { + res(['image/svg+xml', 'svg']); + } else { + // 種類が同定できなかったら application/octet-stream にする + res(['application/octet-stream', null]); + } + }) + .on('end', () => { + // maybe 0 bytes + res(['application/octet-stream', null]); + }); + }); +} diff --git a/src/misc/detect-url-mine.ts b/src/misc/detect-url-mine.ts new file mode 100644 index 000000000..eef64cfc5 --- /dev/null +++ b/src/misc/detect-url-mine.ts @@ -0,0 +1,15 @@ +import { createTemp } from './create-temp'; +import { downloadUrl } from './donwload-url'; +import { detectMine } from './detect-mine'; + +export async function detectUrlMine(url: string) { + const [path, cleanup] = await createTemp(); + + try { + await downloadUrl(url, path); + const [type] = await detectMine(path); + return type; + } finally { + cleanup(); + } +} diff --git a/src/misc/donwload-url.ts b/src/misc/donwload-url.ts new file mode 100644 index 000000000..0dd4e4ef5 --- /dev/null +++ b/src/misc/donwload-url.ts @@ -0,0 +1,61 @@ +import * as fs from 'fs'; +import * as URL from 'url'; +import * as request from 'request'; +import config from '../config'; +import chalk from 'chalk'; +import Logger from '../services/logger'; + +export async function downloadUrl(url: string, path: string) { + const logger = new Logger('download-url'); + + await new Promise((res, rej) => { + logger.info(`Downloading ${chalk.cyan(url)} ...`); + + const writable = fs.createWriteStream(path); + + writable.on('finish', () => { + logger.succ(`Download finished: ${chalk.cyan(url)}`); + res(); + }); + + writable.on('error', error => { + logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, { + url: url, + e: error + }); + rej(error); + }); + + const requestUrl = URL.parse(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url; + + const req = request({ + url: requestUrl, + proxy: config.proxy, + timeout: 10 * 1000, + headers: { + 'User-Agent': config.userAgent + } + }); + + req.pipe(writable); + + req.on('response', response => { + if (response.statusCode !== 200) { + logger.error(`Got ${response.statusCode} (${url})`); + writable.close(); + rej(response.statusCode); + } + }); + + req.on('error', error => { + logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, { + url: url, + e: error + }); + writable.close(); + rej(error); + }); + + logger.succ(`Downloaded to: ${path}`); + }); +} diff --git a/src/misc/download-text-file.ts b/src/misc/download-text-file.ts index 60c9d83da..f73286e9b 100644 --- a/src/misc/download-text-file.ts +++ b/src/misc/download-text-file.ts @@ -1,79 +1,25 @@ -import * as tmp from 'tmp'; import * as fs from 'fs'; import * as util from 'util'; -import chalk from 'chalk'; -import * as request from 'request'; import Logger from '../services/logger'; -import config from '../config'; +import { createTemp } from './create-temp'; +import { downloadUrl } from './donwload-url'; const logger = new Logger('download-text-file'); export async function downloadTextFile(url: string): Promise { // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); logger.info(`Temp file is ${path}`); - // write content at URL to temp file - await new Promise((res, rej) => { - logger.info(`Downloading ${chalk.cyan(url)} ...`); + try { + // write content at URL to temp file + await downloadUrl(url, path); - const writable = fs.createWriteStream(path); + const text = await util.promisify(fs.readFile)(path, 'utf8'); - writable.on('finish', () => { - logger.succ(`Download finished: ${chalk.cyan(url)}`); - res(); - }); - - writable.on('error', error => { - logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, { - url: url, - e: error - }); - rej(error); - }); - - const requestUrl = new URL(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url; - - const req = request({ - url: requestUrl, - proxy: config.proxy, - timeout: 10 * 1000, - headers: { - 'User-Agent': config.userAgent - } - }); - - req.pipe(writable); - - req.on('response', response => { - if (response.statusCode !== 200) { - logger.error(`Got ${response.statusCode} (${url})`); - writable.close(); - rej(response.statusCode); - } - }); - - req.on('error', error => { - logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, { - url: url, - e: error - }); - writable.close(); - rej(error); - }); - }); - - logger.succ(`Downloaded to: ${path}`); - - const text = await util.promisify(fs.readFile)(path, 'utf8'); - - cleanup(); - - return text; + return text; + } finally { + cleanup(); + } } diff --git a/src/models/emoji.ts b/src/models/emoji.ts index 373d5f860..cbf939222 100644 --- a/src/models/emoji.ts +++ b/src/models/emoji.ts @@ -17,4 +17,5 @@ export type IEmoji = { updatedAt?: Date; /** AP object id */ uri?: string; + type?: string; }; diff --git a/src/remote/activitypub/renderer/emoji.ts b/src/remote/activitypub/renderer/emoji.ts index b18337d27..1a05b4e89 100644 --- a/src/remote/activitypub/renderer/emoji.ts +++ b/src/remote/activitypub/renderer/emoji.ts @@ -8,7 +8,7 @@ export default (emoji: IEmoji) => ({ updated: emoji.updatedAt != null ? emoji.updatedAt.toISOString() : new Date().toISOString, icon: { type: 'Image', - mediaType: 'image/png', //Mei-TODO + mediaType: emoji.type || 'image/png', url: emoji.url } }); diff --git a/src/server/api/endpoints/admin/emoji/add.ts b/src/server/api/endpoints/admin/emoji/add.ts index 99439f89a..c126c8380 100644 --- a/src/server/api/endpoints/admin/emoji/add.ts +++ b/src/server/api/endpoints/admin/emoji/add.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import Emoji from '../../../../../models/emoji'; import define from '../../../define'; +import { detectUrlMine } from '../../../../../misc/detect-url-mine'; export const meta = { desc: { @@ -29,12 +30,15 @@ export const meta = { }; export default define(meta, async (ps) => { + const type = await detectUrlMine(ps.url); + const emoji = await Emoji.insert({ updatedAt: new Date(), name: ps.name, host: null, aliases: ps.aliases, - url: ps.url + url: ps.url, + type, }); return { diff --git a/src/server/api/endpoints/admin/emoji/update.ts b/src/server/api/endpoints/admin/emoji/update.ts index 38d90c65a..8b1c07be9 100644 --- a/src/server/api/endpoints/admin/emoji/update.ts +++ b/src/server/api/endpoints/admin/emoji/update.ts @@ -2,6 +2,7 @@ import $ from 'cafy'; import Emoji from '../../../../../models/emoji'; import define from '../../../define'; import ID from '../../../../../misc/cafy-id'; +import { detectUrlMine } from '../../../../../misc/detect-url-mine'; export const meta = { desc: { @@ -39,12 +40,15 @@ export default define(meta, async (ps) => { if (emoji == null) throw new Error('emoji not found'); + const type = await detectUrlMine(ps.url); + await Emoji.update({ _id: emoji._id }, { $set: { updatedAt: new Date(), name: ps.name, aliases: ps.aliases, - url: ps.url + url: ps.url, + type, } }); diff --git a/src/server/proxy/proxy-media.ts b/src/server/proxy/proxy-media.ts index 2eec2012f..357715bb9 100644 --- a/src/server/proxy/proxy-media.ts +++ b/src/server/proxy/proxy-media.ts @@ -1,27 +1,19 @@ import * as fs from 'fs'; -import * as URL from 'url'; -import * as tmp from 'tmp'; import * as Koa from 'koa'; -import * as request from 'request'; -import fileType from 'file-type'; import { serverLogger } from '..'; -import config from '../../config'; import { IImage, ConvertToPng, ConvertToJpeg } from '../../services/drive/image-processor'; -import checkSvg from '../../misc/check-svg'; +import { createTemp } from '../../misc/create-temp'; +import { downloadUrl } from '../../misc/donwload-url'; +import { detectMine } from '../../misc/detect-mine'; export async function proxyMedia(ctx: Koa.BaseContext) { const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url; // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); try { - await fetch(url, path); + await downloadUrl(url, path); const [type, ext] = await detectMine(path); @@ -54,70 +46,3 @@ export async function proxyMedia(ctx: Koa.BaseContext) { cleanup(); } } - -async function fetch(url: string, path: string) { - await new Promise((res, rej) => { - const writable = fs.createWriteStream(path); - - writable.on('finish', () => { - res(); - }); - - writable.on('error', error => { - rej(error); - }); - - const requestUrl = URL.parse(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url; - - const req = request({ - url: requestUrl, - proxy: config.proxy, - timeout: 10 * 1000, - headers: { - 'User-Agent': config.userAgent - } - }); - - req.pipe(writable); - - req.on('response', response => { - if (response.statusCode !== 200) { - writable.close(); - rej(response.statusCode); - } - }); - - req.on('error', error => { - writable.close(); - rej(error); - }); - }); -} - -async function detectMine(path: string) { - return new Promise<[string, string]>((res, rej) => { - const readable = fs.createReadStream(path); - readable - .on('error', rej) - .once('data', (buffer: Buffer) => { - readable.destroy(); - const type = fileType(buffer); - if (type) { - if (type.mime == 'application/xml' && checkSvg(path)) { - res(['image/svg+xml', 'svg']); - } else { - res([type.mime, type.ext]); - } - } else if (checkSvg(path)) { - res(['image/svg+xml', 'svg']); - } else { - // 種類が同定できなかったら application/octet-stream にする - res(['application/octet-stream', null]); - } - }) - .on('end', () => { - // maybe 0 bytes - res(['application/octet-stream', null]); - }); - }); -} diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 5be71bc0a..cdbcb34de 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -6,7 +6,6 @@ import * as crypto from 'crypto'; import * as Minio from 'minio'; import * as uuid from 'uuid'; import * as sharp from 'sharp'; -import fileType from 'file-type'; import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file'; import DriveFolder from '../../models/drive-folder'; @@ -25,8 +24,8 @@ import { GenerateVideoThumbnail } from './generate-video-thumbnail'; import { driveLogger } from './logger'; import { IImage, ConvertToJpeg, ConvertToWebp, ConvertToPng } from './image-processor'; import Instance from '../../models/instance'; -import checkSvg from '../../misc/check-svg'; import { contentDisposition } from '../../misc/content-disposition'; +import { detectMine } from '../../misc/detect-mine'; const logger = driveLogger.createSubLogger('register', 'yellow'); @@ -306,33 +305,6 @@ export default async function( }); }); - // Detect content type - const detectMime = new Promise<[string, string]>((res, rej) => { - const readable = fs.createReadStream(path); - readable - .on('error', rej) - .once('data', (buffer: Buffer) => { - readable.destroy(); - const type = fileType(buffer); - if (type) { - if (type.mime == 'application/xml' && checkSvg(path)) { - res(['image/svg+xml', 'svg']); - } else { - res([type.mime, type.ext]); - } - } else if (checkSvg(path)) { - res(['image/svg+xml', 'svg']); - } else { - // 種類が同定できなかったら application/octet-stream にする - res(['application/octet-stream', null]); - } - }) - .on('end', () => { - // maybe 0 bytes - res(['application/octet-stream', null]); - }); - }); - // Get file size const getFileSize = new Promise((res, rej) => { fs.stat(path, (err, stats) => { @@ -341,7 +313,7 @@ export default async function( }); }); - const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMime, getFileSize]); + const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMine(path), getFileSize]); logger.info(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); diff --git a/src/services/drive/upload-from-url.ts b/src/services/drive/upload-from-url.ts index 89efe8a50..cdf6ba0ce 100644 --- a/src/services/drive/upload-from-url.ts +++ b/src/services/drive/upload-from-url.ts @@ -1,15 +1,12 @@ -import * as fs from 'fs'; import * as URL from 'url'; -import * as tmp from 'tmp'; -import * as request from 'request'; import { IDriveFile, validateFileName } from '../../models/drive-file'; import create from './add-file'; -import config from '../../config'; import { IUser } from '../../models/user'; import * as mongodb from 'mongodb'; import { driveLogger } from './logger'; -import chalk from 'chalk'; +import { createTemp } from '../../misc/create-temp'; +import { downloadUrl } from '../../misc/donwload-url'; const logger = driveLogger.createSubLogger('downloader'); @@ -28,62 +25,10 @@ export default async ( } // Create temp file - const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { - tmp.file((e, path, fd, cleanup) => { - if (e) return rej(e); - res([path, cleanup]); - }); - }); + const [path, cleanup] = await createTemp(); // write content at URL to temp file - await new Promise((res, rej) => { - logger.info(`Downloading ${chalk.cyan(url)} ...`); - - const writable = fs.createWriteStream(path); - - writable.on('finish', () => { - logger.succ(`Download finished: ${chalk.cyan(url)}`); - res(); - }); - - writable.on('error', error => { - logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, { - url: url, - e: error - }); - rej(error); - }); - - const requestUrl = URL.parse(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url; - - const req = request({ - url: requestUrl, - proxy: config.proxy, - timeout: 10 * 1000, - headers: { - 'User-Agent': config.userAgent - } - }); - - req.pipe(writable); - - req.on('response', response => { - if (response.statusCode !== 200) { - logger.error(`Got ${response.statusCode} (${url})`); - writable.close(); - rej(response.statusCode); - } - }); - - req.on('error', error => { - logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, { - url: url, - e: error - }); - writable.close(); - rej(error); - }); - }); + await downloadUrl(url, path); let driveFile: IDriveFile; let error;