From d833b7dce9a5f8219b4c4f4d974cef77f80a38f0 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 8 Sep 2017 22:10:25 +0900 Subject: [PATCH] Implement #771 --- CHANGELOG.md | 4 + locales/en.yml | 4 + locales/ja.yml | 4 + src/tools/analysis/extract-user-domains.ts | 120 +++++++++++++++++++++ src/web/app/mobile/tags/user.tag | 40 +++++++ 5 files changed, 172 insertions(+) create mode 100644 src/tools/analysis/extract-user-domains.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index fc7e2c7db..b071274d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ChangeLog (Release Notes) ========================= 主に notable な changes を書いていきます +unreleased +---------- +* New: ユーザーページによく使うドメインを表示 (#771) + 2566 (2017/09/07) ----------------- * New: 投稿することの多いキーワードをユーザーページに表示する (#768) diff --git a/locales/en.yml b/locales/en.yml index 05b4663c1..fd07be2ae 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -498,6 +498,7 @@ mobile: images: "Images" activity: "Activity" keywords: "Keywords" + domains: "Domains" followers-you-know: "Followers you know" last-used-at: "Latest used at" @@ -512,6 +513,9 @@ mobile: mk-user-overview-keywords: no-keywords: "No keywords" + mk-user-overview-domains: + no-domains: "No domains" + mk-user-overview-followers-you-know: loading: "Loading" no-users: "No users" diff --git a/locales/ja.yml b/locales/ja.yml index 86da08272..832390f5f 100644 --- a/locales/ja.yml +++ b/locales/ja.yml @@ -498,6 +498,7 @@ mobile: images: "画像" activity: "アクティビティ" keywords: "キーワード" + domains: "頻出ドメイン" followers-you-know: "知り合いのフォロワー" last-used-at: "最終ログイン" @@ -512,6 +513,9 @@ mobile: mk-user-overview-keywords: no-keywords: "キーワードはありません(十分な数の投稿をしていない可能性があります)" + mk-user-overview-domains: + no-domains: "よく表れるドメインは検出されませんでした" + mk-user-overview-followers-you-know: loading: "読み込み中" no-users: "知り合いのユーザーはいません" diff --git a/src/tools/analysis/extract-user-domains.ts b/src/tools/analysis/extract-user-domains.ts new file mode 100644 index 000000000..bc120f5c1 --- /dev/null +++ b/src/tools/analysis/extract-user-domains.ts @@ -0,0 +1,120 @@ +import * as URL from 'url'; + +import Post from '../../api/models/post'; +import User from '../../api/models/user'; +import parse from '../../api/common/text'; + +process.on('unhandledRejection', console.dir); + +function tokenize(text: string) { + if (text == null) return []; + + // パース + const ast = parse(text); + + const domains = ast + // URLを抽出 + .filter(t => t.type == 'url' || t.type == 'link') + .map(t => URL.parse(t.url).hostname); + + return domains; +} + +// Fetch all users +User.find({}, { + fields: { + _id: true + } +}).then(users => { + let i = -1; + + const x = cb => { + if (++i == users.length) return cb(); + extractDomainsOne(users[i]._id).then(() => x(cb), err => { + console.error(err); + setTimeout(() => { + i--; + x(cb); + }, 1000); + }); + }; + + x(() => { + console.log('complete'); + }); +}); + +function extractDomainsOne(id) { + return new Promise(async (resolve, reject) => { + process.stdout.write(`extracting domains of ${id} ...`); + + // Fetch recent posts + const recentPosts = await Post.find({ + user_id: id, + text: { + $exists: true + } + }, { + sort: { + _id: -1 + }, + limit: 10000, + fields: { + _id: false, + text: true + } + }); + + // 投稿が少なかったら中断 + if (recentPosts.length < 100) { + process.stdout.write(' >>> -\n'); + return resolve(); + } + + const domains = {}; + + // Extract domains from recent posts + recentPosts.forEach(post => { + const domainsOfPost = tokenize(post.text); + + domainsOfPost.forEach(domain => { + if (domains[domain]) { + domains[domain]++; + } else { + domains[domain] = 1; + } + }); + }); + + // Calc peak + let peak = 0; + Object.keys(domains).forEach(domain => { + if (domains[domain] > peak) peak = domains[domain]; + }); + + // Sort domains by frequency + const domainsSorted = Object.keys(domains).sort((a, b) => domains[b] - domains[a]); + + // Lookup top 10 domains + const topDomains = domainsSorted.slice(0, 10); + + process.stdout.write(' >>> ' + topDomains.join(', ') + '\n'); + + // Make domains object (includes weights) + const domainsObj = topDomains.map(domain => ({ + domain: domain, + weight: domains[domain] / peak + })); + + // Save + User.update({ _id: id }, { + $set: { + domains: domainsObj + } + }).then(() => { + resolve(); + }, err => { + reject(err); + }); + }); +} diff --git a/src/web/app/mobile/tags/user.tag b/src/web/app/mobile/tags/user.tag index ca777b8fd..a32355921 100644 --- a/src/web/app/mobile/tags/user.tag +++ b/src/web/app/mobile/tags/user.tag @@ -240,6 +240,12 @@ +
+

%i18n:mobile.tags.mk-user-overview.domains%

+
+ +
+

%i18n:mobile.tags.mk-user-overview.followers-you-know%

@@ -579,6 +585,40 @@ + +
1 }> + + { domain.domain } + +
+

%i18n:mobile.tags.mk-user-overview-domains.no-domains%

+ + + +

%i18n:mobile.tags.mk-user-overview-followers-you-know.loading%

0 }>