forked from mirror/misskey
Merge commit from fork
* fix(backend): check target IP before sending HTTP request * fix(backend): allow accessing private IP when testing * Apply suggestions from code review Co-authored-by: anatawa12 <anatawa12@icloud.com> * fix(backend): lint and typecheck * fix(backend): add isLocalAddressAllowed option to getAgentByUrl and send (HttpRequestService) * fix(backend): allow fetchSummaryFromProxy, trueMail to access local addresses --------- Co-authored-by: anatawa12 <anatawa12@icloud.com> Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>
This commit is contained in:
parent
b9cb949eb1
commit
090e9392cd
@ -6,7 +6,6 @@
|
|||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
import * as stream from 'node:stream/promises';
|
import * as stream from 'node:stream/promises';
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import ipaddr from 'ipaddr.js';
|
|
||||||
import chalk from 'chalk';
|
import chalk from 'chalk';
|
||||||
import got, * as Got from 'got';
|
import got, * as Got from 'got';
|
||||||
import { parse } from 'content-disposition';
|
import { parse } from 'content-disposition';
|
||||||
@ -70,13 +69,6 @@ export class DownloadService {
|
|||||||
},
|
},
|
||||||
enableUnixSockets: false,
|
enableUnixSockets: false,
|
||||||
}).on('response', (res: Got.Response) => {
|
}).on('response', (res: Got.Response) => {
|
||||||
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
|
|
||||||
if (this.isPrivateIp(res.ip)) {
|
|
||||||
this.logger.warn(`Blocked address: ${res.ip}`);
|
|
||||||
req.destroy();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const contentLength = res.headers['content-length'];
|
const contentLength = res.headers['content-length'];
|
||||||
if (contentLength != null) {
|
if (contentLength != null) {
|
||||||
const size = Number(contentLength);
|
const size = Number(contentLength);
|
||||||
@ -139,18 +131,4 @@ export class DownloadService {
|
|||||||
cleanup();
|
cleanup();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
|
||||||
private isPrivateIp(ip: string): boolean {
|
|
||||||
const parsedIp = ipaddr.parse(ip);
|
|
||||||
|
|
||||||
for (const net of this.config.allowedPrivateNetworks ?? []) {
|
|
||||||
const cidr = ipaddr.parseCIDR(net);
|
|
||||||
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return parsedIp.range() !== 'unicast';
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -312,6 +312,7 @@ export class EmailService {
|
|||||||
Accept: 'application/json',
|
Accept: 'application/json',
|
||||||
Authorization: truemailAuthKey,
|
Authorization: truemailAuthKey,
|
||||||
},
|
},
|
||||||
|
isLocalAddressAllowed: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
const json = (await res.json()) as {
|
const json = (await res.json()) as {
|
||||||
|
@ -6,6 +6,7 @@
|
|||||||
import * as http from 'node:http';
|
import * as http from 'node:http';
|
||||||
import * as https from 'node:https';
|
import * as https from 'node:https';
|
||||||
import * as net from 'node:net';
|
import * as net from 'node:net';
|
||||||
|
import ipaddr from 'ipaddr.js';
|
||||||
import CacheableLookup from 'cacheable-lookup';
|
import CacheableLookup from 'cacheable-lookup';
|
||||||
import fetch from 'node-fetch';
|
import fetch from 'node-fetch';
|
||||||
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
|
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
|
||||||
@ -25,8 +26,102 @@ export type HttpRequestSendOptions = {
|
|||||||
validators?: ((res: Response) => void)[];
|
validators?: ((res: Response) => void)[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
declare module 'node:http' {
|
||||||
|
interface Agent {
|
||||||
|
createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class HttpRequestServiceAgent extends http.Agent {
|
||||||
|
constructor(
|
||||||
|
private config: Config,
|
||||||
|
options?: http.AgentOptions,
|
||||||
|
) {
|
||||||
|
super(options);
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
|
||||||
|
const socket = super.createConnection(options, callback)
|
||||||
|
.on('connect', () => {
|
||||||
|
const address = socket.remoteAddress;
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
if (address && ipaddr.isValid(address)) {
|
||||||
|
if (this.isPrivateIp(address)) {
|
||||||
|
socket.destroy(new Error(`Blocked address: ${address}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return socket;
|
||||||
|
};
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
private isPrivateIp(ip: string): boolean {
|
||||||
|
const parsedIp = ipaddr.parse(ip);
|
||||||
|
|
||||||
|
for (const net of this.config.allowedPrivateNetworks ?? []) {
|
||||||
|
const cidr = ipaddr.parseCIDR(net);
|
||||||
|
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsedIp.range() !== 'unicast';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class HttpsRequestServiceAgent extends https.Agent {
|
||||||
|
constructor(
|
||||||
|
private config: Config,
|
||||||
|
options?: https.AgentOptions,
|
||||||
|
) {
|
||||||
|
super(options);
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
|
||||||
|
const socket = super.createConnection(options, callback)
|
||||||
|
.on('connect', () => {
|
||||||
|
const address = socket.remoteAddress;
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
if (address && ipaddr.isValid(address)) {
|
||||||
|
if (this.isPrivateIp(address)) {
|
||||||
|
socket.destroy(new Error(`Blocked address: ${address}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return socket;
|
||||||
|
};
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
private isPrivateIp(ip: string): boolean {
|
||||||
|
const parsedIp = ipaddr.parse(ip);
|
||||||
|
|
||||||
|
for (const net of this.config.allowedPrivateNetworks ?? []) {
|
||||||
|
const cidr = ipaddr.parseCIDR(net);
|
||||||
|
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsedIp.range() !== 'unicast';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class HttpRequestService {
|
export class HttpRequestService {
|
||||||
|
/**
|
||||||
|
* Get http non-proxy agent (without local address filtering)
|
||||||
|
*/
|
||||||
|
private httpNative: http.Agent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get https non-proxy agent (without local address filtering)
|
||||||
|
*/
|
||||||
|
private httpsNative: https.Agent;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get http non-proxy agent
|
* Get http non-proxy agent
|
||||||
*/
|
*/
|
||||||
@ -57,19 +152,20 @@ export class HttpRequestService {
|
|||||||
lookup: false, // nativeのdns.lookupにfallbackしない
|
lookup: false, // nativeのdns.lookupにfallbackしない
|
||||||
});
|
});
|
||||||
|
|
||||||
this.http = new http.Agent({
|
const agentOption = {
|
||||||
keepAlive: true,
|
keepAlive: true,
|
||||||
keepAliveMsecs: 30 * 1000,
|
keepAliveMsecs: 30 * 1000,
|
||||||
lookup: cache.lookup as unknown as net.LookupFunction,
|
lookup: cache.lookup as unknown as net.LookupFunction,
|
||||||
localAddress: config.outgoingAddress,
|
localAddress: config.outgoingAddress,
|
||||||
});
|
};
|
||||||
|
|
||||||
this.https = new https.Agent({
|
this.httpNative = new http.Agent(agentOption);
|
||||||
keepAlive: true,
|
|
||||||
keepAliveMsecs: 30 * 1000,
|
this.httpsNative = new https.Agent(agentOption);
|
||||||
lookup: cache.lookup as unknown as net.LookupFunction,
|
|
||||||
localAddress: config.outgoingAddress,
|
this.http = new HttpRequestServiceAgent(config, agentOption);
|
||||||
});
|
|
||||||
|
this.https = new HttpsRequestServiceAgent(config, agentOption);
|
||||||
|
|
||||||
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
|
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
|
||||||
|
|
||||||
@ -104,16 +200,22 @@ export class HttpRequestService {
|
|||||||
* @param bypassProxy Allways bypass proxy
|
* @param bypassProxy Allways bypass proxy
|
||||||
*/
|
*/
|
||||||
@bindThis
|
@bindThis
|
||||||
public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
|
public getAgentByUrl(url: URL, bypassProxy = false, isLocalAddressAllowed = false): http.Agent | https.Agent {
|
||||||
if (bypassProxy || (this.config.proxyBypassHosts ?? []).includes(url.hostname)) {
|
if (bypassProxy || (this.config.proxyBypassHosts ?? []).includes(url.hostname)) {
|
||||||
|
if (isLocalAddressAllowed) {
|
||||||
|
return url.protocol === 'http:' ? this.httpNative : this.httpsNative;
|
||||||
|
}
|
||||||
return url.protocol === 'http:' ? this.http : this.https;
|
return url.protocol === 'http:' ? this.http : this.https;
|
||||||
} else {
|
} else {
|
||||||
|
if (isLocalAddressAllowed && (!this.config.proxy)) {
|
||||||
|
return url.protocol === 'http:' ? this.httpNative : this.httpsNative;
|
||||||
|
}
|
||||||
return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent;
|
return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public async getActivityJson(url: string): Promise<IObject> {
|
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObject> {
|
||||||
const res = await this.send(url, {
|
const res = await this.send(url, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: {
|
headers: {
|
||||||
@ -121,6 +223,7 @@ export class HttpRequestService {
|
|||||||
},
|
},
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
size: 1024 * 256,
|
size: 1024 * 256,
|
||||||
|
isLocalAddressAllowed: isLocalAddressAllowed,
|
||||||
}, {
|
}, {
|
||||||
throwErrorWhenResponseNotOk: true,
|
throwErrorWhenResponseNotOk: true,
|
||||||
validators: [validateContentTypeSetAsActivityPub],
|
validators: [validateContentTypeSetAsActivityPub],
|
||||||
@ -135,7 +238,7 @@ export class HttpRequestService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public async getJson<T = unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
|
public async getJson<T = unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>, isLocalAddressAllowed = false): Promise<T> {
|
||||||
const res = await this.send(url, {
|
const res = await this.send(url, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: Object.assign({
|
headers: Object.assign({
|
||||||
@ -143,19 +246,21 @@ export class HttpRequestService {
|
|||||||
}, headers ?? {}),
|
}, headers ?? {}),
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
size: 1024 * 256,
|
size: 1024 * 256,
|
||||||
|
isLocalAddressAllowed: isLocalAddressAllowed,
|
||||||
});
|
});
|
||||||
|
|
||||||
return await res.json() as T;
|
return await res.json() as T;
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
|
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>, isLocalAddressAllowed = false): Promise<string> {
|
||||||
const res = await this.send(url, {
|
const res = await this.send(url, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: Object.assign({
|
headers: Object.assign({
|
||||||
Accept: accept,
|
Accept: accept,
|
||||||
}, headers ?? {}),
|
}, headers ?? {}),
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
|
isLocalAddressAllowed: isLocalAddressAllowed,
|
||||||
});
|
});
|
||||||
|
|
||||||
return await res.text();
|
return await res.text();
|
||||||
@ -170,6 +275,7 @@ export class HttpRequestService {
|
|||||||
headers?: Record<string, string>,
|
headers?: Record<string, string>,
|
||||||
timeout?: number,
|
timeout?: number,
|
||||||
size?: number,
|
size?: number,
|
||||||
|
isLocalAddressAllowed?: boolean,
|
||||||
} = {},
|
} = {},
|
||||||
extra: HttpRequestSendOptions = {
|
extra: HttpRequestSendOptions = {
|
||||||
throwErrorWhenResponseNotOk: true,
|
throwErrorWhenResponseNotOk: true,
|
||||||
@ -183,6 +289,8 @@ export class HttpRequestService {
|
|||||||
controller.abort();
|
controller.abort();
|
||||||
}, timeout);
|
}, timeout);
|
||||||
|
|
||||||
|
const isLocalAddressAllowed = args.isLocalAddressAllowed ?? false;
|
||||||
|
|
||||||
const res = await fetch(url, {
|
const res = await fetch(url, {
|
||||||
method: args.method ?? 'GET',
|
method: args.method ?? 'GET',
|
||||||
headers: {
|
headers: {
|
||||||
@ -191,7 +299,7 @@ export class HttpRequestService {
|
|||||||
},
|
},
|
||||||
body: args.body,
|
body: args.body,
|
||||||
size: args.size ?? 10 * 1024 * 1024,
|
size: args.size ?? 10 * 1024 * 1024,
|
||||||
agent: (url) => this.getAgentByUrl(url),
|
agent: (url) => this.getAgentByUrl(url, false, isLocalAddressAllowed),
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -145,6 +145,6 @@ export class UrlPreviewService {
|
|||||||
contentLengthRequired: meta.urlPreviewRequireContentLength,
|
contentLengthRequired: meta.urlPreviewRequireContentLength,
|
||||||
});
|
});
|
||||||
|
|
||||||
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
|
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user