diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 0207cf58a0..bcbd47736f 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -4,6 +4,7 @@ import type { UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import { fetch } from 'undici'; type CaptchaResponse = { success: boolean; @@ -35,7 +36,7 @@ export class CaptchaService { }, // TODO //timeout: 10 * 1000, - agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy), + dispatcher: this.httpRequestService.getAgentByUrl(new URL(url)), }).catch(err => { throw `${err.message ?? err}`; }); diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 62123246a7..13e0066d3d 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -34,57 +34,29 @@ export class DownloadService { @bindThis public async downloadUrl(url: string, path: string): Promise { this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); - + const timeout = 30 * 1000; const operationTimeout = 60 * 1000; const maxSize = this.config.maxFileSize ?? 262144000; - const req = got.stream(url, { + const response = await this.httpRequestService.fetch({ + method: 'GET', + url, headers: { 'User-Agent': this.config.userAgent, }, - timeout: { - lookup: timeout, - connect: timeout, - secureConnect: timeout, - socket: timeout, // read timeout - response: timeout, - send: timeout, - request: operationTimeout, // whole operation timeout - }, - agent: { - http: this.httpRequestService.httpAgent, - https: this.httpRequestService.httpsAgent, - }, - http2: false, // default - retry: { - limit: 0, - }, - }).on('response', (res: Got.Response) => { - if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) { - if (this.isPrivateIp(res.ip)) { - this.logger.warn(`Blocked address: ${res.ip}`); - req.destroy(); - } - } - - const contentLength = res.headers['content-length']; - if (contentLength != null) { - const size = Number(contentLength); - if (size > maxSize) { - this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); - req.destroy(); - } - } - }).on('downloadProgress', (progress: Got.Progress) => { - if (progress.transferred > maxSize) { - this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); - req.destroy(); - } + timeout, + size: maxSize, + ipCheckers: + (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && + !this.config.proxy ? + [{ type: 'black', fn: this.isPrivateIp }] : + undefined, + // http2: false, // default }); try { - await pipeline(req, fs.createWriteStream(path)); + await pipeline(stream.Readable.fromWeb(streaming), fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); @@ -124,6 +96,6 @@ export class DownloadService { } } - return PrivateIp(ip); + return PrivateIp(ip) ?? false; } } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 7eea45200e..bf42c6529a 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -1,7 +1,6 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import { JSDOM } from 'jsdom'; -import fetch from 'node-fetch'; import tinycolor from 'tinycolor2'; import type { Instance } from '@/models/entities/Instance.js'; import type { InstancesRepository } from '@/models/index.js'; @@ -191,10 +190,8 @@ export class FetchInstanceMetadataService { const faviconUrl = url + '/favicon.ico'; - const favicon = await fetch(faviconUrl, { - // TODO - //timeout: 10000, - agent: url => this.httpRequestService.getAgentByUrl(url), + const favicon = await this.httpRequestService.fetch({ + url: faviconUrl, }); if (favicon.ok) { diff --git a/packages/backend/src/core/HttpFetchService.ts b/packages/backend/src/core/HttpFetchService.ts deleted file mode 100644 index 3f43bbe071..0000000000 --- a/packages/backend/src/core/HttpFetchService.ts +++ /dev/null @@ -1,178 +0,0 @@ -import CacheableLookup from 'cacheable-lookup'; -import { Inject, Injectable } from '@nestjs/common'; -import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; -import { StatusError } from '@/misc/status-error.js'; -import { bindThis } from '@/decorators.js'; -import * as undici from 'undici'; -import { LookupFunction } from 'node:net'; -import { TransformStream } from 'node:stream/web'; - -@Injectable() -export class HttpRequestService { - /** - * Get http non-proxy agent - */ - private agent: undici.Agent; - - /** - * Get http proxy or non-proxy agent - */ - public proxiedAgent: undici.ProxyAgent | undici.Agent; - - public readonly clientDefaults: undici.Agent.Options; - - constructor( - @Inject(DI.config) - private config: Config, - ) { - const cache = new CacheableLookup({ - maxTtl: 3600, // 1hours - errorTtl: 30, // 30secs - lookup: false, // nativeのdns.lookupにfallbackしない - }); - - this.clientDefaults = { - keepAliveTimeout: 4 * 1000, - keepAliveMaxTimeout: 10 * 60 * 1000, - keepAliveTimeoutThreshold: 1 * 1000, - strictContentLength: true, - connect: { - maxCachedSessions: 100, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 - lookup: cache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 - }, - } - - this.agent = new undici.Agent({ - ...this.clientDefaults, - }); - - const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); - - this.proxiedAgent = config.proxy - ? new undici.ProxyAgent({ - ...this.clientDefaults, - connections: maxSockets, - - uri: config.proxy, - }) - : this.agent; - } - - /** - * Get agent by URL - * @param url URL - * @param bypassProxy Allways bypass proxy - */ - @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { - if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { - return this.agent; - } else { - return this.proxiedAgent; - } - } - - @bindThis - public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - size: 1024 * 256, - }); - - return await res.json(); - } - - @bindThis - public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - }); - - return await res.text(); - } - - @bindThis - public async getResponse(args: { - url: string, - method: string, - body?: string, - headers: Record, - timeout?: number, - size?: number, - redirect?: RequestRedirect | undefined, - dispatcher?: undici.Dispatcher, - }): Promise { - const timeout = args.timeout ?? 10 * 1000; - - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, timeout * 6); - - const res = await Promise.race([ - undici.fetch(args.url, { - method: args.method, - headers: args.headers, - body: args.body, - redirect: args.redirect, - dispatcher: args.dispatcher ?? this.getAgentByUrl(new URL(args.url)), - keepalive: true, - signal: controller.signal, - }), - new Promise((res) => setTimeout(() => res(null))) - ]); - - if (res == null) { - throw new StatusError(`Request Timeout`, 408, 'Request Timeout'); - } - - if (!res.ok) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - - return ({ - ...res, - body: this.fetchLimiter(res, args.size), - }); - } - - /** - * Fetch body limiter - * @param res undici.Response - * @param size number of Max size (Bytes) (default: 10MiB) - * @returns ReadableStream (provided by node:stream/web) - */ - @bindThis - private fetchLimiter(res: undici.Response, size: number = 10 * 1024 * 1024) { - if (res.body == null) return null; - - let total = 0; - return res.body.pipeThrough(new TransformStream({ - start() {}, - transform(chunk, controller) { - // TypeScirptグローバルの定義はUnit8ArrayだがundiciはReadableStreamを渡してくるので一応変換 - const uint8 = new Uint8Array(chunk); - total += uint8.length; - if (total > size) { - controller.error(new StatusError(`Payload Too Large`, 413, 'Payload Too Large')); - } else { - controller.enqueue(uint8); - } - }, - flush() {}, - })); - } -} diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 49b28ae523..90bad67723 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -1,18 +1,31 @@ import * as http from 'node:http'; import * as https from 'node:https'; import CacheableLookup from 'cacheable-lookup'; -import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; -import type { Response } from 'node-fetch'; -import type { URL } from 'node:url'; +import * as undici from 'undici'; +import { LookupFunction } from 'node:net'; +import { TransformStream } from 'node:stream/web'; +import * as dns from 'node:dns'; + +export type IpChecker = (ip: string) => boolean; @Injectable() export class HttpRequestService { + /** + * Get http non-proxy agent (undici) + */ + private agent: undici.Agent; + + /** + * Get http proxy or non-proxy agent (undici) + */ + public proxiedAgent: undici.ProxyAgent | undici.Agent; + /** * Get http non-proxy agent */ @@ -33,30 +46,57 @@ export class HttpRequestService { */ public httpsAgent: https.Agent; + public readonly dnsCache: CacheableLookup; + public readonly clientDefaults: undici.Agent.Options; + constructor( @Inject(DI.config) private config: Config, ) { - const cache = new CacheableLookup({ + this.dnsCache = new CacheableLookup({ maxTtl: 3600, // 1hours errorTtl: 30, // 30secs lookup: false, // nativeのdns.lookupにfallbackしない }); - + + this.clientDefaults = { + keepAliveTimeout: 4 * 1000, + keepAliveMaxTimeout: 10 * 60 * 1000, + keepAliveTimeoutThreshold: 1 * 1000, + strictContentLength: true, + connect: { + maxCachedSessions: 100, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 + lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 + }, + } + + this.agent = new undici.Agent({ + ...this.clientDefaults, + }); + this.http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, + lookup: this.dnsCache.lookup, } as http.AgentOptions); this.https = new https.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, + lookup: this.dnsCache.lookup, } as https.AgentOptions); - + const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); - + + this.proxiedAgent = config.proxy + ? new undici.ProxyAgent({ + ...this.clientDefaults, + connections: maxSockets, + + uri: config.proxy, + }) + : this.agent; + this.httpAgent = config.proxy ? new HttpProxyAgent({ keepAlive: true, @@ -86,19 +126,49 @@ export class HttpRequestService { * @param bypassProxy Allways bypass proxy */ @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { + public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { + if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { + return this.agent; + } else { + return this.proxiedAgent; + } + } + + /** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ + @bindThis + public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { return url.protocol === 'http:' ? this.http : this.https; } else { return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent; } } + /** + * check ip + */ + @bindThis + public checkIp(url: URL, fn: IpChecker): Promise { + const lookup = this.dnsCache.lookup as LookupFunction || dns.lookup; + + return new Promise((resolve, reject) => { + lookup(url.hostname, {}, (err, ip) => { + if (err) { + resolve(false); + } else { + resolve(fn(ip)); + } + }); + }); + } @bindThis - public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ + public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { + const res = await this.fetch({ url, - method: 'GET', headers: Object.assign({ 'User-Agent': this.config.userAgent, Accept: accept, @@ -112,9 +182,8 @@ export class HttpRequestService { @bindThis public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ + const res = await this.fetch({ url, - method: 'GET', headers: Object.assign({ 'User-Agent': this.config.userAgent, Accept: accept, @@ -126,14 +195,35 @@ export class HttpRequestService { } @bindThis - public async getResponse(args: { + public async fetch(args: { url: string, - method: string, + method?: string, body?: string, - headers: Record, + headers?: Record, timeout?: number, size?: number, - }): Promise { + redirect?: RequestRedirect | undefined, + dispatcher?: undici.Dispatcher, + ipCheckers?: { + type: 'black' | 'white', + fn: IpChecker, + }[], + noOkError?: boolean, + }): Promise { + const url = new URL(args.url); + + if (args.ipCheckers) { + for (const check of args.ipCheckers) { + const result = await this.checkIp(url, check.fn); + if ( + (check.type === 'black' && result === true) || + (check.type === 'white' && result === false) + ) { + throw new StatusError('IP is not allowed', 403, 'IP is not allowed'); + } + } + } + const timeout = args.timeout ?? 10 * 1000; const controller = new AbortController(); @@ -141,20 +231,57 @@ export class HttpRequestService { controller.abort(); }, timeout * 6); - const res = await fetch(args.url, { - method: args.method, - headers: args.headers, - body: args.body, - timeout, - size: args.size ?? 10 * 1024 * 1024, - agent: (url) => this.getAgentByUrl(url), - signal: controller.signal, - }); + const res = await Promise.race([ + undici.fetch(args.url, { + method: args.method ?? 'GET', + headers: args.headers, + body: args.body, + redirect: args.redirect, + dispatcher: args.dispatcher ?? this.getAgentByUrl(url), + keepalive: true, + signal: controller.signal, + }), + new Promise((res) => setTimeout(() => res(null))) + ]); - if (!res.ok) { + if (res == null) { + throw new StatusError(`Request Timeout`, 408, 'Request Timeout'); + } + + if (!res.ok && !args.noOkError) { throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); } - return res; + return ({ + ...res, + body: this.fetchLimiter(res, args.size), + }); + } + + /** + * Fetch body limiter + * @param res undici.Response + * @param size number of Max size (Bytes) (default: 10MiB) + * @returns ReadableStream (provided by node:stream/web) + */ + @bindThis + private fetchLimiter(res: undici.Response, size: number = 10 * 1024 * 1024) { + if (res.body == null) return null; + + let total = 0; + return res.body.pipeThrough(new TransformStream({ + start() {}, + transform(chunk, controller) { + // TypeScirptグローバルの定義はUnit8ArrayだがundiciはReadableStreamを渡してくるので一応変換 + const uint8 = new Uint8Array(chunk); + total += uint8.length; + if (total > size) { + controller.error(new StatusError(`Payload Too Large`, 413, 'Payload Too Large')); + } else { + controller.enqueue(uint8); + } + }, + flush() {}, + })); } } diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 0ce69aaa74..930188ce6e 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -33,7 +33,7 @@ export class S3Service { ? false : meta.objectStorageS3ForcePathStyle, httpOptions: { - agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), + agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy), }, }); } diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts index 4c91ab8438..69df2d0c1b 100644 --- a/packages/backend/src/core/WebfingerService.ts +++ b/packages/backend/src/core/WebfingerService.ts @@ -30,7 +30,7 @@ export class WebfingerService { public async webfinger(query: string): Promise { const url = this.genUrl(query); - return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger; + return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json'); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index d1edd579fa..b09bae7c26 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -152,7 +152,7 @@ export class ApRequestService { }, }); - await this.httpRequestService.getResponse({ + await this.httpRequestService.fetch({ url, method: req.request.method, headers: req.request.headers, @@ -180,7 +180,7 @@ export class ApRequestService { }, }); - const res = await this.httpRequestService.getResponse({ + const res = await this.httpRequestService.fetch({ url, method: req.request.method, headers: req.request.headers, diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index e96c84f148..21bff88b2e 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -96,8 +96,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject; + ? await this.apRequestService.signedGet(value, this.user) as IObject + : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')); if (object == null || ( Array.isArray(object['@context']) ? diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index b71320ed0b..5533fc70f2 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -116,13 +116,14 @@ class LdSignature { @bindThis private async fetchDocument(url: string) { - const json = await fetch(url, { + const json = await this.httpRequestService.fetch({ + url, headers: { Accept: 'application/ld+json, application/json', }, + noOkError: true, // TODO //timeout: this.loderTimeout, - agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent, }).then(res => { if (!res.ok) { throw `${res.status} ${res.statusText}`; diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 183ef07477..57ebd87522 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -33,7 +33,7 @@ export class WebhookDeliverProcessorService { try { this.logger.debug(`delivering ${job.data.webhookId}`); - const res = await this.httpRequestService.getResponse({ + const res = await this.httpRequestService.fetch({ url: job.data.to, method: 'POST', headers: { diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index 58fa01ac48..537f562231 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -33,7 +33,7 @@ export default class extends Endpoint { private httpRequestService: HttpRequestService, ) { super(meta, paramDef, async (ps, me) => { - const res = await this.httpRequestService.getResponse({ + const res = await this.httpRequestService.fetch({ url: ps.url, method: 'GET', headers: Object.assign({ diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index ec16965998..0ae1aa12b1 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -84,17 +84,17 @@ export default class extends Endpoint { const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; - const res = await fetch(endpoint, { + const res = await this.httpRequestService.fetch({ + url: endpoint, method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': config.userAgent, Accept: 'application/json, */*', }, - body: params, + body: params.toString(), // TODO //timeout: 10000, - agent: (url) => this.httpRequestService.getAgentByUrl(url), }); const json = (await res.json()) as { diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index baef8fa993..1545f0a9f9 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import summaly from 'summaly'; +import * as summaly from 'summaly'; import { DI } from '@/di-symbols.js'; import type { UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -65,7 +65,7 @@ export class UrlPreviewService { : `Getting preview of ${url}@${lang} ...`); try { - const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({ + const summary = meta.summalyProxy ? await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ url: url, lang: lang ?? 'ja-JP', })}`) : await summaly.default(url, {