diff --git a/package.json b/package.json index 051eb1cb8..6ea91a7f5 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "clean": "gulp clean", "cleanall": "gulp cleanall", "lint": "gulp lint", - "test": "gulp test" + "test": "gulp test" }, "devDependencies": { "@types/bcryptjs": "2.4.0", @@ -95,6 +95,7 @@ "webpack": "3.8.1" }, "dependencies": { + "@prezzemolo/rap": "0.1.2", "accesses": "2.5.0", "animejs": "2.2.0", "autwh": "0.0.1", diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 714eeb520..f9c22ccac 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -4,14 +4,27 @@ import * as gm from 'gm'; import * as debug from 'debug'; import fileType = require('file-type'); import prominence = require('prominence'); -import DriveFile from '../models/drive-file'; +import DriveFile, { getGridFSBucket } from '../models/drive-file'; import DriveFolder from '../models/drive-folder'; import serialize from '../serializers/drive-file'; import event from '../event'; import config from '../../conf'; +import { Duplex } from 'stream'; const log = debug('misskey:register-drive-file'); +const addToGridFS = (name, binary, metadata): Promise => new Promise(async (resolve, reject) => { + const dataStream = new Duplex(); + dataStream.push(binary); + dataStream.push(null); + + const bucket = await getGridFSBucket(); + const writeStream = bucket.openUploadStream(name, { metadata }); + writeStream.once('finish', (doc) => { resolve(doc); }); + writeStream.on('error', reject); + dataStream.pipe(writeStream); +}); + /** * Add file to drive * @@ -58,7 +71,7 @@ export default ( // Generate hash const hash = crypto - .createHash('sha256') + .createHash('md5') .update(data) .digest('hex') as string; @@ -67,8 +80,8 @@ export default ( if (!force) { // Check if there is a file with the same hash const much = await DriveFile.findOne({ - user_id: user._id, - hash: hash + md5: hash, + 'metadata.user_id': user._id }); if (much !== null) { @@ -82,13 +95,13 @@ export default ( // Calculate drive usage const usage = ((await DriveFile .aggregate([ - { $match: { user_id: user._id } }, + { $match: { 'metadata.user_id': user._id } }, { $project: { - datasize: true + length: true }}, { $group: { _id: null, - usage: { $sum: '$datasize' } + usage: { $sum: '$length' } }} ]))[0] || { usage: 0 @@ -131,21 +144,15 @@ export default ( } // Create DriveFile document - const file = await DriveFile.insert({ - created_at: new Date(), + const file = await addToGridFS(`${user._id}/${name}`, data, { user_id: user._id, folder_id: folder !== null ? folder._id : null, - data: data, - datasize: size, type: mime, name: name, comment: comment, - hash: hash, properties: properties }); - delete file.data; - log(`drive file has been created ${file._id}`); resolve(file); diff --git a/src/api/endpoints/drive.ts b/src/api/endpoints/drive.ts index 41ad6301d..d92473633 100644 --- a/src/api/endpoints/drive.ts +++ b/src/api/endpoints/drive.ts @@ -14,16 +14,16 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Calculate drive usage const usage = ((await DriveFile .aggregate([ - { $match: { user_id: user._id } }, + { $match: { 'metadata.user_id': user._id } }, { $project: { - datasize: true + length: true } }, { $group: { _id: null, - usage: { $sum: '$datasize' } + usage: { $sum: '$length' } } } ]))[0] || { diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts index a68ae3481..53b48a8be 100644 --- a/src/api/endpoints/drive/files.ts +++ b/src/api/endpoints/drive/files.ts @@ -13,35 +13,35 @@ import serialize from '../../serializers/drive-file'; * @param {any} app * @return {Promise} */ -module.exports = (params, user, app) => new Promise(async (res, rej) => { +module.exports = async (params, user, app) => { // Get 'limit' parameter const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$; - if (limitErr) return rej('invalid limit param'); + if (limitErr) throw 'invalid limit param'; // Get 'since_id' parameter const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$; - if (sinceIdErr) return rej('invalid since_id param'); + if (sinceIdErr) throw 'invalid since_id param'; // Get 'max_id' parameter const [maxId, maxIdErr] = $(params.max_id).optional.id().$; - if (maxIdErr) return rej('invalid max_id param'); + if (maxIdErr) throw 'invalid max_id param'; // Check if both of since_id and max_id is specified if (sinceId && maxId) { - return rej('cannot set since_id and max_id'); + throw 'cannot set since_id and max_id'; } // Get 'folder_id' parameter const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; - if (folderIdErr) return rej('invalid folder_id param'); + if (folderIdErr) throw 'invalid folder_id param'; // Construct query const sort = { _id: -1 }; const query = { - user_id: user._id, - folder_id: folderId + 'metadata.user_id': user._id, + 'metadata.folder_id': folderId } as any; if (sinceId) { sort._id = 1; @@ -57,14 +57,11 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { // Issue query const files = await DriveFile .find(query, { - fields: { - data: false - }, limit: limit, sort: sort }); // Serialize - res(await Promise.all(files.map(async file => - await serialize(file)))); -}); + const _files = await Promise.all(files.map(file => serialize(file))); + return _files; +}; diff --git a/src/api/endpoints/drive/files/find.ts b/src/api/endpoints/drive/files/find.ts index cd0b33f2c..1c818131d 100644 --- a/src/api/endpoints/drive/files/find.ts +++ b/src/api/endpoints/drive/files/find.ts @@ -24,13 +24,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Issue query const files = await DriveFile .find({ - name: name, - user_id: user._id, - folder_id: folderId - }, { - fields: { - data: false - } + 'metadata.name': name, + 'metadata.user_id': user._id, + 'metadata.folder_id': folderId }); // Serialize diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts index 8dbc297e4..3c7cf774f 100644 --- a/src/api/endpoints/drive/files/show.ts +++ b/src/api/endpoints/drive/files/show.ts @@ -12,28 +12,26 @@ import serialize from '../../../serializers/drive-file'; * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise(async (res, rej) => { +module.exports = async (params, user) => { // Get 'file_id' parameter const [fileId, fileIdErr] = $(params.file_id).id().$; - if (fileIdErr) return rej('invalid file_id param'); + if (fileIdErr) throw 'invalid file_id param'; // Fetch file const file = await DriveFile .findOne({ _id: fileId, - user_id: user._id - }, { - fields: { - data: false - } + 'metadata.user_id': user._id }); if (file === null) { - return rej('file-not-found'); + throw 'file-not-found'; } // Serialize - res(await serialize(file, { + const _file = await serialize(file, { detail: true - })); -}); + }); + + return _file; +}; diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts index 1cfbdd8f0..d7b858c2b 100644 --- a/src/api/endpoints/drive/files/update.ts +++ b/src/api/endpoints/drive/files/update.ts @@ -24,11 +24,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const file = await DriveFile .findOne({ _id: fileId, - user_id: user._id - }, { - fields: { - data: false - } + 'metadata.user_id': user._id }); if (file === null) { @@ -38,7 +34,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Get 'name' parameter const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$; if (nameErr) return rej('invalid name param'); - if (name) file.name = name; + if (name) file.metadata.name = name; // Get 'folder_id' parameter const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$; @@ -46,7 +42,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (folderId !== undefined) { if (folderId === null) { - file.folder_id = null; + file.metadata.folder_id = null; } else { // Fetch folder const folder = await DriveFolder @@ -59,14 +55,14 @@ module.exports = (params, user) => new Promise(async (res, rej) => { return rej('folder-not-found'); } - file.folder_id = folder._id; + file.metadata.folder_id = folder._id; } } - DriveFile.update(file._id, { + await DriveFile.update(file._id, { $set: { - name: file.name, - folder_id: file.folder_id + 'metadata.name': file.metadata.name, + 'metadata.folder_id': file.metadata.folder_id } }); diff --git a/src/api/endpoints/drive/folders/find.ts b/src/api/endpoints/drive/folders/find.ts index cdf055839..a5eb8e015 100644 --- a/src/api/endpoints/drive/folders/find.ts +++ b/src/api/endpoints/drive/folders/find.ts @@ -30,6 +30,5 @@ module.exports = (params, user) => new Promise(async (res, rej) => { }); // Serialize - res(await Promise.all(folders.map(async folder => - await serialize(folder)))); + res(await Promise.all(folders.map(folder => serialize(folder)))); }); diff --git a/src/api/endpoints/drive/folders/update.ts b/src/api/endpoints/drive/folders/update.ts index eec275787..4f2e3d2a7 100644 --- a/src/api/endpoints/drive/folders/update.ts +++ b/src/api/endpoints/drive/folders/update.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import DriveFolder from '../../../models/drive-folder'; import { isValidFolderName } from '../../../models/drive-folder'; -import serialize from '../../../serializers/drive-file'; +import serialize from '../../../serializers/drive-folder'; import event from '../../../event'; /** diff --git a/src/api/endpoints/messaging/messages/create.ts b/src/api/endpoints/messaging/messages/create.ts index 8af55d850..149852c09 100644 --- a/src/api/endpoints/messaging/messages/create.ts +++ b/src/api/endpoints/messaging/messages/create.ts @@ -54,9 +54,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (fileId !== undefined) { file = await DriveFile.findOne({ _id: fileId, - user_id: user._id - }, { - data: false + 'metadata.user_id': user._id }); if (file === null) { diff --git a/src/api/endpoints/posts/create.ts b/src/api/endpoints/posts/create.ts index f982b9ee9..4f4b7e2e8 100644 --- a/src/api/endpoints/posts/create.ts +++ b/src/api/endpoints/posts/create.ts @@ -44,9 +44,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => { // SELECT _id const entity = await DriveFile.findOne({ _id: mediaId, - user_id: user._id - }, { - _id: true + 'metadata.user_id': user._id }); if (entity === null) { diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts index aa5aff5ba..203413e23 100644 --- a/src/api/endpoints/posts/timeline.ts +++ b/src/api/endpoints/posts/timeline.ts @@ -2,6 +2,7 @@ * Module dependencies */ import $ from 'cafy'; +import rap from '@prezzemolo/rap'; import Post from '../../models/post'; import ChannelWatching from '../../models/channel-watching'; import getFriends from '../../common/get-friends'; @@ -15,32 +16,33 @@ import serialize from '../../serializers/post'; * @param {any} app * @return {Promise} */ -module.exports = (params, user, app) => new Promise(async (res, rej) => { +module.exports = async (params, user, app) => { // Get 'limit' parameter const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$; - if (limitErr) return rej('invalid limit param'); + if (limitErr) throw 'invalid limit param'; // Get 'since_id' parameter const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$; - if (sinceIdErr) return rej('invalid since_id param'); + if (sinceIdErr) throw 'invalid since_id param'; // Get 'max_id' parameter const [maxId, maxIdErr] = $(params.max_id).optional.id().$; - if (maxIdErr) return rej('invalid max_id param'); + if (maxIdErr) throw 'invalid max_id param'; // Check if both of since_id and max_id is specified if (sinceId && maxId) { - return rej('cannot set since_id and max_id'); + throw 'cannot set since_id and max_id'; } - // ID list of the user itself and other users who the user follows - const followingIds = await getFriends(user._id); - - // Watchしているチャンネルを取得 - const watches = await ChannelWatching.find({ - user_id: user._id, - // 削除されたドキュメントは除く - deleted_at: { $exists: false } + const { followingIds, watchChannelIds } = await rap({ + // ID list of the user itself and other users who the user follows + followingIds: getFriends(user._id), + // Watchしているチャンネルを取得 + watchChannelIds: ChannelWatching.find({ + user_id: user._id, + // 削除されたドキュメントは除く + deleted_at: { $exists: false } + }).then(watches => watches.map(w => w.channel_id)) }); //#region Construct query @@ -65,7 +67,7 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }, { // Watchしているチャンネルへの投稿 channel_id: { - $in: watches.map(w => w.channel_id) + $in: watchChannelIds } }] } as any; @@ -90,7 +92,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }); // Serialize - res(await Promise.all(timeline.map(async post => - await serialize(post, user) - ))); -}); + const _timeline = await Promise.all(timeline.map(post => serialize(post, user))); + return _timeline; +}; diff --git a/src/api/models/drive-file.ts b/src/api/models/drive-file.ts index 8d158cf56..8968d065c 100644 --- a/src/api/models/drive-file.ts +++ b/src/api/models/drive-file.ts @@ -1,11 +1,22 @@ -import db from '../../db/mongodb'; +import * as mongodb from 'mongodb'; +import monkDb, { nativeDbConn } from '../../db/mongodb'; -const collection = db.get('drive_files'); +const collection = monkDb.get('drive_files.files'); (collection as any).createIndex('hash'); // fuck type definition export default collection as any; // fuck type definition +const getGridFSBucket = async (): Promise => { + const db = await nativeDbConn(); + const bucket = new mongodb.GridFSBucket(db, { + bucketName: 'drive_files' + }); + return bucket; +}; + +export { getGridFSBucket }; + export function validateFileName(name: string): boolean { return ( (name.trim().length > 0) && diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts index b4e2ab064..2af7db572 100644 --- a/src/api/serializers/drive-file.ts +++ b/src/api/serializers/drive-file.ts @@ -31,44 +31,40 @@ export default ( if (mongo.ObjectID.prototype.isPrototypeOf(file)) { _file = await DriveFile.findOne({ _id: file - }, { - fields: { - data: false - } - }); + }); } else if (typeof file === 'string') { _file = await DriveFile.findOne({ _id: new mongo.ObjectID(file) - }, { - fields: { - data: false - } - }); + }); } else { _file = deepcopy(file); } - // Rename _id to id - _file.id = _file._id; - delete _file._id; + if (!_file) return reject('invalid file arg.'); - delete _file.data; + // rendered target + let _target: any = {}; - _file.url = `${config.drive_url}/${_file.id}/${encodeURIComponent(_file.name)}`; + _target.id = _file._id; + _target.created_at = _file.uploadDate; - if (opts.detail && _file.folder_id) { + _target = Object.assign(_target, _file.metadata); + + _target.url = `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`; + + if (opts.detail && _target.folder_id) { // Populate folder - _file.folder = await serializeDriveFolder(_file.folder_id, { + _target.folder = await serializeDriveFolder(_target.folder_id, { detail: true }); } - if (opts.detail && _file.tags) { + if (opts.detail && _target.tags) { // Populate tags - _file.tags = await _file.tags.map(async (tag: any) => + _target.tags = await _target.tags.map(async (tag: any) => await serializeDriveTag(tag) ); } - resolve(_file); + resolve(_target); }); diff --git a/src/api/serializers/drive-folder.ts b/src/api/serializers/drive-folder.ts index a42846410..6ebf454a2 100644 --- a/src/api/serializers/drive-folder.ts +++ b/src/api/serializers/drive-folder.ts @@ -44,7 +44,7 @@ const self = ( }); const childFilesCount = await DriveFile.count({ - folder_id: _folder.id + 'metadata.folder_id': _folder.id }); _folder.folders_count = childFoldersCount; diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index 7c3690ef7..03fd12077 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -12,6 +12,7 @@ import serializeChannel from './channel'; import serializeUser from './user'; import serializeDriveFile from './drive-file'; import parse from '../common/text'; +import rap from '@prezzemolo/rap'; /** * Serialize a post @@ -21,13 +22,13 @@ import parse from '../common/text'; * @param options? serialize options * @return response */ -const self = ( +const self = async ( post: string | mongo.ObjectID | IPost, me?: string | mongo.ObjectID | IUser, options?: { detail: boolean } -) => new Promise(async (resolve, reject) => { +) => { const opts = options || { detail: true, }; @@ -56,6 +57,8 @@ const self = ( _post = deepcopy(post); } + if (!_post) throw 'invalid post arg.'; + const id = _post._id; // Rename _id to id @@ -70,105 +73,120 @@ const self = ( } // Populate user - _post.user = await serializeUser(_post.user_id, meId); + _post.user = serializeUser(_post.user_id, meId); // Populate app if (_post.app_id) { - _post.app = await serializeApp(_post.app_id); + _post.app = serializeApp(_post.app_id); } // Populate channel if (_post.channel_id) { - _post.channel = await serializeChannel(_post.channel_id); + _post.channel = serializeChannel(_post.channel_id); } // Populate media if (_post.media_ids) { - _post.media = await Promise.all(_post.media_ids.map(async fileId => - await serializeDriveFile(fileId) + _post.media = Promise.all(_post.media_ids.map(fileId => + serializeDriveFile(fileId) )); } // When requested a detailed post data if (opts.detail) { // Get previous post info - const prev = await Post.findOne({ - user_id: _post.user_id, - _id: { - $lt: id - } - }, { - fields: { - _id: true - }, - sort: { - _id: -1 - } - }); - _post.prev = prev ? prev._id : null; + _post.prev = (async () => { + const prev = await Post.findOne({ + user_id: _post.user_id, + _id: { + $lt: id + } + }, { + fields: { + _id: true + }, + sort: { + _id: -1 + } + }); + return prev ? prev._id : null; + })(); // Get next post info - const next = await Post.findOne({ - user_id: _post.user_id, - _id: { - $gt: id - } - }, { - fields: { - _id: true - }, - sort: { - _id: 1 - } - }); - _post.next = next ? next._id : null; + _post.next = (async () => { + const next = await Post.findOne({ + user_id: _post.user_id, + _id: { + $gt: id + } + }, { + fields: { + _id: true + }, + sort: { + _id: 1 + } + }); + return next ? next._id : null; + })(); if (_post.reply_id) { // Populate reply to post - _post.reply = await self(_post.reply_id, meId, { + _post.reply = self(_post.reply_id, meId, { detail: false }); } if (_post.repost_id) { // Populate repost - _post.repost = await self(_post.repost_id, meId, { + _post.repost = self(_post.repost_id, meId, { detail: _post.text == null }); } // Poll if (meId && _post.poll) { - const vote = await Vote - .findOne({ - user_id: meId, - post_id: id - }); + _post.poll = (async (poll) => { + const vote = await Vote + .findOne({ + user_id: meId, + post_id: id + }); - if (vote != null) { - const myChoice = _post.poll.choices - .filter(c => c.id == vote.choice)[0]; + if (vote != null) { + const myChoice = poll.choices + .filter(c => c.id == vote.choice)[0]; - myChoice.is_voted = true; - } + myChoice.is_voted = true; + } + + return poll; + })(_post.poll); } // Fetch my reaction if (meId) { - const reaction = await Reaction - .findOne({ - user_id: meId, - post_id: id, - deleted_at: { $exists: false } - }); + _post.my_reaction = (async () => { + const reaction = await Reaction + .findOne({ + user_id: meId, + post_id: id, + deleted_at: { $exists: false } + }); - if (reaction) { - _post.my_reaction = reaction.reaction; - } + if (reaction) { + return reaction.reaction; + } + + return null; + })(); } } - resolve(_post); -}); + // resolve promises in _post object + _post = await rap(_post); + + return _post; +}; export default self; diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts index 3deff2d00..0d24d6cc0 100644 --- a/src/api/serializers/user.ts +++ b/src/api/serializers/user.ts @@ -8,6 +8,7 @@ import serializePost from './post'; import Following from '../models/following'; import getFriends from '../common/get-friends'; import config from '../../conf'; +import rap from '@prezzemolo/rap'; /** * Serialize a user @@ -55,6 +56,8 @@ export default ( _user = deepcopy(user); } + if (!_user) return reject('invalid user arg.'); + // Me const meId: mongo.ObjectID = me ? mongo.ObjectID.prototype.isPrototypeOf(me) @@ -104,26 +107,30 @@ export default ( if (meId && !meId.equals(_user.id)) { // If the user is following - const follow = await Following.findOne({ - follower_id: meId, - followee_id: _user.id, - deleted_at: { $exists: false } - }); - _user.is_following = follow !== null; + _user.is_following = (async () => { + const follow = await Following.findOne({ + follower_id: meId, + followee_id: _user.id, + deleted_at: { $exists: false } + }); + return follow !== null; + })(); // If the user is followed - const follow2 = await Following.findOne({ - follower_id: _user.id, - followee_id: meId, - deleted_at: { $exists: false } - }); - _user.is_followed = follow2 !== null; + _user.is_followed = (async () => { + const follow2 = await Following.findOne({ + follower_id: _user.id, + followee_id: meId, + deleted_at: { $exists: false } + }); + return follow2 !== null; + })(); } if (opts.detail) { if (_user.pinned_post_id) { // Populate pinned post - _user.pinned_post = await serializePost(_user.pinned_post_id, meId, { + _user.pinned_post = serializePost(_user.pinned_post_id, meId, { detail: true }); } @@ -132,23 +139,24 @@ export default ( const myFollowingIds = await getFriends(meId); // Get following you know count - const followingYouKnowCount = await Following.count({ + _user.following_you_know_count = Following.count({ followee_id: { $in: myFollowingIds }, follower_id: _user.id, deleted_at: { $exists: false } }); - _user.following_you_know_count = followingYouKnowCount; // Get followers you know count - const followersYouKnowCount = await Following.count({ + _user.followers_you_know_count = Following.count({ followee_id: _user.id, follower_id: { $in: myFollowingIds }, deleted_at: { $exists: false } }); - _user.followers_you_know_count = followersYouKnowCount; } } + // resolve promises in _user object + _user = await rap(_user); + resolve(_user); }); /* diff --git a/src/db/mongodb.ts b/src/db/mongodb.ts index 6ee7f4534..c978e6460 100644 --- a/src/db/mongodb.ts +++ b/src/db/mongodb.ts @@ -1,11 +1,38 @@ -import * as mongo from 'monk'; - import config from '../conf'; const uri = config.mongodb.user && config.mongodb.pass - ? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}` - : `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`; +? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}` +: `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`; + +/** + * monk + */ +import * as mongo from 'monk'; const db = mongo(uri); export default db; + +/** + * MongoDB native module (officialy) + */ +import * as mongodb from 'mongodb'; + +let mdb: mongodb.Db; + +const nativeDbConn = async (): Promise => { + if (mdb) return mdb; + + const db = await ((): Promise => new Promise((resolve, reject) => { + mongodb.MongoClient.connect(uri, (e, db) => { + if (e) return reject(e); + resolve(db); + }); + }))(); + + mdb = db; + + return db; +}; + +export { nativeDbConn }; diff --git a/src/file/server.ts b/src/file/server.ts index ee67cf786..375f29487 100644 --- a/src/file/server.ts +++ b/src/file/server.ts @@ -9,7 +9,7 @@ import * as cors from 'cors'; import * as mongodb from 'mongodb'; import * as gm from 'gm'; -import File from '../api/models/drive-file'; +import DriveFile, { getGridFSBucket } from '../api/models/drive-file'; /** * Init app @@ -97,17 +97,28 @@ app.get('/:id', async (req, res) => { return; } - const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) }); + const fileId = new mongodb.ObjectID(req.params.id); + const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { - res.status(404).sendFile(`${__dirname} / assets / dummy.png`); - return; - } else if (file.data == null) { - res.sendStatus(400); + res.status(404).sendFile(`${__dirname}/assets/dummy.png`); return; } - send(file.data.buffer, file.type, req, res); + const bucket = await getGridFSBucket(); + + const buffer = await ((id): Promise => new Promise((resolve, reject) => { + const chunks = []; + const readableStream = bucket.openDownloadStream(id); + readableStream.on('data', chunk => { + chunks.push(chunk); + }); + readableStream.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + }))(fileId); + + send(buffer, file.metadata.type, req, res); }); app.get('/:id/:name', async (req, res) => { @@ -117,17 +128,28 @@ app.get('/:id/:name', async (req, res) => { return; } - const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) }); + const fileId = new mongodb.ObjectID(req.params.id); + const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { res.status(404).sendFile(`${__dirname}/assets/dummy.png`); return; - } else if (file.data == null) { - res.sendStatus(400); - return; } - send(file.data.buffer, file.type, req, res); + const bucket = await getGridFSBucket(); + + const buffer = await ((id): Promise => new Promise((resolve, reject) => { + const chunks = []; + const readableStream = bucket.openDownloadStream(id); + readableStream.on('data', chunk => { + chunks.push(chunk); + }); + readableStream.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + }))(fileId); + + send(buffer, file.metadata.type, req, res); }); module.exports = app; diff --git a/test/api.js b/test/api.js index b43eb7ff6..c0da9d6c5 100644 --- a/test/api.js +++ b/test/api.js @@ -1152,9 +1152,12 @@ async function insertHimawari(opts) { } async function insertDriveFile(opts) { - return await db.get('drive_files').insert(Object.assign({ - name: 'strawberry-pasta.png' - }, opts)); + return await db.get('drive_files.files').insert({ + length: opts.datasize, + metadata: Object.assign({ + name: 'strawberry-pasta.png' + }, opts) + }); } async function insertDriveFolder(opts) { diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js new file mode 100644 index 000000000..d41514416 --- /dev/null +++ b/tools/migration/use-gridfs.js @@ -0,0 +1,49 @@ +// for Node.js interpret + +const { default: db } = require('../../built/db/mongodb') +const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file') +const { Duplex } = require('stream') + +const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => { + const writeStream = bucket.openUploadStreamWithId(...rest) + + const dataStream = new Duplex() + dataStream.push(buffer) + dataStream.push(null) + + writeStream.once('finish', resolve) + writeStream.on('error', reject) + + dataStream.pipe(writeStream) +}) + +const migrateToGridFS = async (doc) => { + const id = doc._id + const buffer = doc.data.buffer + const created_at = doc.created_at + + delete doc._id + delete doc.created_at + delete doc.datasize + delete doc.hash + delete doc.data + + const bucket = await getGridFSBucket() + const added = await writeToGridFS(bucket, buffer, id, `${id}/${doc.name}`, { metadata: doc }) + + const result = await DriveFile.update(id, { + $set: { + uploadDate: created_at + } + }) + + return added && result.ok === 1 +} + +const main = async () => { + const docs = await db.get('drive_files').find() + const all = await Promise.all(docs.map(migrateToGridFS)) + return all +} + +main().then(console.dir).catch(console.error)