From 25dedfb9112da43b6e5601e716e1f384ec5565e0 Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 5 Nov 2019 22:14:42 +0900 Subject: [PATCH] Store nodeinfo per federated instances (#5578) * Store nodeinfo per federated instances * Update fetch-nodeinfo.ts * Update fetch-nodeinfo.ts * update --- migration/1572760203493-nodeinfo.ts | 29 ++++++ src/misc/app-lock.ts | 4 + src/models/entities/instance.ts | 50 ++++++++-- src/queue/processors/deliver.ts | 3 + src/queue/processors/inbox.ts | 3 + src/remote/activitypub/models/person.ts | 2 + src/services/fetch-nodeinfo.ts | 91 +++++++++++++++++++ .../register-or-fetch-instance-doc.ts | 1 - 8 files changed, 173 insertions(+), 10 deletions(-) create mode 100644 migration/1572760203493-nodeinfo.ts create mode 100644 src/services/fetch-nodeinfo.ts diff --git a/migration/1572760203493-nodeinfo.ts b/migration/1572760203493-nodeinfo.ts new file mode 100644 index 000000000..88d8df723 --- /dev/null +++ b/migration/1572760203493-nodeinfo.ts @@ -0,0 +1,29 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class nodeinfo1572760203493 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "system"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "softwareName" character varying(64) DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "softwareVersion" character varying(64) DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "openRegistrations" boolean DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "name" character varying(256) DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "description" character varying(4096) DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerName" character varying(128) DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerEmail" character varying(256) DEFAULT null`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "infoUpdatedAt" TIMESTAMP WITH TIME ZONE`, undefined); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "infoUpdatedAt"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerEmail"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerName"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "description"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "name"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "openRegistrations"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareVersion"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareName"`, undefined); + await queryRunner.query(`ALTER TABLE "instance" ADD "system" character varying(64)`, undefined); + } + +} diff --git a/src/misc/app-lock.ts b/src/misc/app-lock.ts index 30579ed93..3d5ff9188 100644 --- a/src/misc/app-lock.ts +++ b/src/misc/app-lock.ts @@ -20,3 +20,7 @@ const lock: (key: string, timeout?: number) => Promise<() => void> export function getApLock(uri: string, timeout = 30 * 1000) { return lock(`ap-object:${uri}`, timeout); } + +export function getNodeinfoLock(host: string, timeout = 30 * 1000) { + return lock(`nodeinfo:${host}`, timeout); +} diff --git a/src/models/entities/instance.ts b/src/models/entities/instance.ts index 52c5215f1..dd0de100d 100644 --- a/src/models/entities/instance.ts +++ b/src/models/entities/instance.ts @@ -25,15 +25,6 @@ export class Instance { }) public host: string; - /** - * インスタンスのシステム (MastodonとかMisskeyとかPleromaとか) - */ - @Column('varchar', { - length: 64, nullable: true, - comment: 'The system of the Instance.' - }) - public system: string | null; - /** * インスタンスのユーザー数 */ @@ -129,4 +120,45 @@ export class Instance { default: false }) public isMarkedAsClosed: boolean; + + @Column('varchar', { + length: 64, nullable: true, default: null, + comment: 'The software of the Instance.' + }) + public softwareName: string | null; + + @Column('varchar', { + length: 64, nullable: true, default: null, + }) + public softwareVersion: string | null; + + @Column('boolean', { + nullable: true, default: null, + }) + public openRegistrations: boolean | null; + + @Column('varchar', { + length: 256, nullable: true, default: null, + }) + public name: string | null; + + @Column('varchar', { + length: 4096, nullable: true, default: null, + }) + public description: string | null; + + @Column('varchar', { + length: 128, nullable: true, default: null, + }) + public maintainerName: string | null; + + @Column('varchar', { + length: 256, nullable: true, default: null, + }) + public maintainerEmail: string | null; + + @Column('timestamp with time zone', { + nullable: true, + }) + public infoUpdatedAt: Date | null; } diff --git a/src/queue/processors/deliver.ts b/src/queue/processors/deliver.ts index 8837c80d8..b252c2016 100644 --- a/src/queue/processors/deliver.ts +++ b/src/queue/processors/deliver.ts @@ -4,6 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins import Logger from '../../services/logger'; import { Instances } from '../../models'; import { instanceChart } from '../../services/chart'; +import { fetchNodeinfo } from '../../services/fetch-nodeinfo'; const logger = new Logger('deliver'); @@ -28,6 +29,8 @@ export default async (job: Bull.Job) => { isNotResponding: false }); + fetchNodeinfo(i); + instanceChart.requestSent(i.host, true); }); diff --git a/src/queue/processors/inbox.ts b/src/queue/processors/inbox.ts index e71181ee7..1a583ec86 100644 --- a/src/queue/processors/inbox.ts +++ b/src/queue/processors/inbox.ts @@ -13,6 +13,7 @@ import { fetchMeta } from '../../misc/fetch-meta'; import { toPuny } from '../../misc/convert-host'; import { validActor } from '../../remote/activitypub/type'; import { ensure } from '../../prelude/ensure'; +import { fetchNodeinfo } from '../../services/fetch-nodeinfo'; const logger = new Logger('inbox'); @@ -105,6 +106,8 @@ export default async (job: Bull.Job): Promise => { isNotResponding: false }); + fetchNodeinfo(i); + instanceChart.requestReceived(i.host); }); diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index 198fd78bd..c7a6d5663 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -27,6 +27,7 @@ import { validActor } from '../../../remote/activitypub/type'; import { getConnection } from 'typeorm'; import { ensure } from '../../../prelude/ensure'; import { toArray } from '../../../prelude/array'; +import { fetchNodeinfo } from '../../../services/fetch-nodeinfo'; const logger = apLogger; @@ -191,6 +192,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise { Instances.increment({ id: i.id }, 'usersCount', 1); instanceChart.newUser(i.host); + fetchNodeinfo(i); }); usersChart.update(user!, true); diff --git a/src/services/fetch-nodeinfo.ts b/src/services/fetch-nodeinfo.ts new file mode 100644 index 000000000..e5d652a6b --- /dev/null +++ b/src/services/fetch-nodeinfo.ts @@ -0,0 +1,91 @@ +import * as request from 'request-promise-native'; +import { Instance } from '../models/entities/instance'; +import { Instances } from '../models'; +import config from '../config'; +import { getNodeinfoLock } from '../misc/app-lock'; +import Logger from '../services/logger'; + +export const logger = new Logger('nodeinfo', 'cyan'); + +export async function fetchNodeinfo(instance: Instance) { + const unlock = await getNodeinfoLock(instance.host); + + const _instance = await Instances.findOne({ host: instance.host }); + const now = Date.now(); + if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { + unlock(); + return; + } + + logger.info(`Fetching nodeinfo of ${instance.host} ...`); + + try { + const wellknown = await request({ + url: 'https://' + instance.host + '/.well-known/nodeinfo', + proxy: config.proxy, + timeout: 1000 * 10, + forever: true, + headers: { + 'User-Agent': config.userAgent, + Accept: 'application/json, */*' + }, + json: true + }).catch(e => { + if (e.statusCode === 404) { + throw 'No nodeinfo provided'; + } else { + throw e.statusCode || e.message; + } + }); + + if (wellknown.links == null || !Array.isArray(wellknown.links)) { + throw 'No wellknown links'; + } + + const links = wellknown.links as any[]; + + const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0'); + const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0'); + const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1'); + const link = lnik2_1 || lnik2_0 || lnik1_0; + + if (link == null) { + throw 'No nodeinfo link provided'; + } + + const info = await request({ + url: link.href, + proxy: config.proxy, + timeout: 1000 * 10, + forever: true, + headers: { + 'User-Agent': config.userAgent, + Accept: 'application/json, */*' + }, + json: true + }).catch(e => { + throw e.statusCode || e.message; + }); + + await Instances.update(instance.id, { + infoUpdatedAt: new Date(), + softwareName: info.software.name.toLowerCase(), + softwareVersion: info.software.version, + openRegistrations: info.openRegistrations, + name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null, + description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null, + maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null, + maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null, + }); + + logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`); + } catch (e) { + logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`); + + await Instances.update(instance.id, { + infoUpdatedAt: new Date(), + }); + } finally { + unlock(); + } +} diff --git a/src/services/register-or-fetch-instance-doc.ts b/src/services/register-or-fetch-instance-doc.ts index 9957edd3d..3501e20de 100644 --- a/src/services/register-or-fetch-instance-doc.ts +++ b/src/services/register-or-fetch-instance-doc.ts @@ -15,7 +15,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise