Merge branch 'develop' into fetch-outbox

CHANGELOG.md

@@ -8,7 +8,7 @@
 -
 
 ### Server
--
+- 最初照会したユーザーの最新ノートを受け取るように
 
 -->
 

chart/files/default.yml

@@ -187,6 +187,10 @@ id: "aidx"
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 
+# Limit of notes to fetch from outbox with remote user first fetched (default: 5)
+# https://github.com/misskey-dev/misskey/pull/11130
+outboxNotesFetchLimit: 5
+
 #allowedPrivateNetworks: [
 #  '127.0.0.1/32'
 #]

packages/backend/src/config.ts

@@ -85,6 +85,7 @@ type Source = {
 	videoThumbnailGenerator?: string;
 
 	signToActivityPubGet?: boolean;
+	outboxNotesFetchLimit?: number;
 
 	perChannelMaxNoteCacheCount?: number;
 	perUserNotificationsMaxCount?: number;

packages/backend/src/core/activitypub/ApInboxService.ts

@@ -27,7 +27,7 @@ import { QueueService } from '@/core/QueueService.js';
 import type { UsersRepository, NotesRepository, FollowingsRepository, AbuseUserReportsRepository, FollowRequestsRepository } from '@/models/_.js';
 import { bindThis } from '@/decorators.js';
 import type { MiRemoteUser } from '@/models/entities/User.js';
-import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
+import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isOrderedCollectionPage, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
 import { ApNoteService } from './models/ApNoteService.js';
 import { ApLoggerService } from './ApLoggerService.js';
 import { ApDbResolverService } from './ApDbResolverService.js';
@@ -87,11 +87,19 @@ export class ApInboxService {
 	}
 
 	@bindThis
-	public async performActivity(actor: MiRemoteUser, activity: IObject): Promise<void> {
-		if (isCollectionOrOrderedCollection(activity)) {
+	public async performActivity(actor: MiRemoteUser, activity: IObject, {
+		limit = Infinity,
+		allow = null as (string[] | null) } = {},
+	): Promise<void> {
+		if (isCollectionOrOrderedCollection(activity) || isOrderedCollectionPage(activity)) {
 			const resolver = this.apResolverService.createResolver();
-			for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems)) {
+			for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems).slice(0, limit)) {
 				const act = await resolver.resolve(item);
+				const type = getApType(act);
+				if (allow && !allow.includes(type)) {
+					this.logger.info(`skipping activity type: ${type}`);
+					continue;
+				}
 				try {
 					await this.performOneActivity(actor, act);
 				} catch (err) {
@@ -367,7 +375,7 @@ export class ApInboxService {
 		});
 
 		if (isPost(object)) {
-			this.createNote(resolver, actor, object, false, activity);
+			await this.createNote(resolver, actor, object, false, activity);
 		} else {
 			this.logger.warn(`Unknown type: ${getApType(object)}`);
 		}

packages/backend/src/core/activitypub/ApResolverService.ts

@@ -15,11 +15,11 @@ import { UtilityService } from '@/core/UtilityService.js';
 import { bindThis } from '@/decorators.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import type Logger from '@/logger.js';
-import { isCollectionOrOrderedCollection } from './type.js';
+import { isCollectionOrOrderedCollection, isOrderedCollectionPage } from './type.js';
 import { ApDbResolverService } from './ApDbResolverService.js';
 import { ApRendererService } from './ApRendererService.js';
 import { ApRequestService } from './ApRequestService.js';
-import type { IObject, ICollection, IOrderedCollection } from './type.js';
+import type { IObject, ICollection, IOrderedCollection, IOrderedCollectionPage } from './type.js';
 
 export class Resolver {
 	private history: Set<string>;
@@ -64,6 +64,18 @@ export class Resolver {
 		}
 	}
 
+	public async resolveOrderedCollectionPage(value: string | IObject): Promise<IOrderedCollectionPage> {
+		const collection = typeof value === 'string'
+			? await this.resolve(value)
+			: value;
+
+		if (isOrderedCollectionPage(collection)) {
+			return collection;
+		} else {
+			throw new Error(`unrecognized collection type: ${collection.type}`);
+		}
+	}
+
 	@bindThis
 	public async resolve(value: string | IObject): Promise<IObject> {
 		if (typeof value !== 'string') {

packages/backend/src/core/activitypub/models/ApPersonService.ts

@@ -38,7 +38,8 @@ import { MetaService } from '@/core/MetaService.js';
 import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
 import type { AccountMoveService } from '@/core/AccountMoveService.js';
 import { checkHttps } from '@/misc/check-https.js';
-import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
+import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage, isPropertyValue } from '../type.js';
+import { ApInboxService } from '../ApInboxService.js';
 import { extractApHashtags } from './tag.js';
 import type { OnModuleInit } from '@nestjs/common';
 import type { ApNoteService } from './ApNoteService.js';
@@ -68,6 +69,7 @@ export class ApPersonService implements OnModuleInit {
 	private apResolverService: ApResolverService;
 	private apNoteService: ApNoteService;
 	private apImageService: ApImageService;
+	private apInboxService: ApInboxService;
 	private apMfmService: ApMfmService;
 	private mfmService: MfmService;
 	private hashtagService: HashtagService;
@@ -116,6 +118,7 @@ export class ApPersonService implements OnModuleInit {
 		this.apResolverService = this.moduleRef.get('ApResolverService');
 		this.apNoteService = this.moduleRef.get('ApNoteService');
 		this.apImageService = this.moduleRef.get('ApImageService');
+		this.apInboxService = this.moduleRef.get('ApInboxService');
 		this.apMfmService = this.moduleRef.get('ApMfmService');
 		this.mfmService = this.moduleRef.get('MfmService');
 		this.hashtagService = this.moduleRef.get('HashtagService');
@@ -384,7 +387,10 @@ export class ApPersonService implements OnModuleInit {
 		}
 		//#endregion
 
-		await this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err));
+		await Promise.allSettled([
+			this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err)),
+			this.updateOutboxFirstPage(user, person.outbox, resolver).catch(err => this.logger.error(err)),
+		]);
 
 		return user;
 	}
@@ -589,7 +595,7 @@ export class ApPersonService implements OnModuleInit {
 		const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
 		const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x)));
 
-		// Resolve and regist Notes
+		// Resolve and register Notes
 		const limit = promiseLimit<MiNote | null>(2);
 		const featuredNotes = await Promise.all(items
 			.filter(item => getApType(item) === 'Note')	// TODO: Noteでなくてもいいかも
@@ -616,6 +622,35 @@ export class ApPersonService implements OnModuleInit {
 		});
 	}
 
+	/**
+	 * Retrieve outbox from an actor object.
+	 *
+	 * This only retrieves the first page for now.
+	 */
+	public async updateOutboxFirstPage(user: RemoteUser, outbox: IActor['outbox'], resolver: Resolver): Promise<void> {
+		if (!this.config.outboxNotesFetchLimit) return;
+
+		// https://www.w3.org/TR/activitypub/#actor-objects
+		// Outbox is a required property for all actors
+		if (!outbox) {
+			throw new Error('No outbox property');
+		}
+
+		this.logger.info(`Fetching the outbox for ${user.uri}: ${outbox}`);
+
+		const collection = await resolver.resolveCollection(outbox);
+		if (!isOrderedCollection(collection)) {
+			throw new Error('Outbox must be an ordered collection');
+		}
+
+		const firstPage = collection.first ?
+			await resolver.resolveOrderedCollectionPage(collection.first) :
+			collection;
+
+		// Perform activity but only the first outboxNotesFetchLimit ones with `type: Create`
+		await this.apInboxService.performActivity(user, firstPage, { limit: this.config.outboxNotesFetchLimit, allow: ['Create'] });
+	}
+
 	/**
 	 * リモート由来のアカウント移行処理を行います
 	 * @param src 移行元アカウント（リモートかつupdatePerson後である必要がある、というかこれ自体がupdatePersonで呼ばれる前提）

packages/backend/src/core/activitypub/type.ts

@@ -92,16 +92,37 @@ export interface IActivity extends IObject {
 	};
 }
 
+// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection
 export interface ICollection extends IObject {
 	type: 'Collection';
 	totalItems: number;
+	current?: ICollectionPage | string;
+	first?: ICollectionPage | string;
+	last?: ICollectionPage | string;
 	items: ApObject;
 }
 
-export interface IOrderedCollection extends IObject {
+// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection
+export interface IOrderedCollection extends Omit<ICollection, 'type' | 'items'> {
 	type: 'OrderedCollection';
-	totalItems: number;
-	orderedItems: ApObject;
+
+	// orderedItems is not defined well
+	// https://github.com/w3c/activitystreams/issues/494
+	orderedItems?: ApObject;
+}
+
+// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage
+export interface ICollectionPage extends Omit<ICollection, 'type'> {
+	type: 'CollectionPage';
+	partOf?: ICollection | string;
+	next?: ICollectionPage | string;
+	prev?: ICollectionPage | string;
+}
+
+// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollectionpage
+export interface IOrderedCollectionPage extends Omit<IOrderedCollection, 'type'>, Omit<ICollectionPage, 'type' | 'items'> {
+	type: 'OrderedCollectionPage';
+	startIndex?: number,
 }
 
 export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
@@ -188,6 +209,9 @@ export const isCollection = (object: IObject): object is ICollection =>
 export const isOrderedCollection = (object: IObject): object is IOrderedCollection =>
 	getApType(object) === 'OrderedCollection';
 
+export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
+	getApType(object) === 'OrderedCollectionPage';
+
 export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
 	isCollection(object) || isOrderedCollection(object);
 

packages/backend/test/misc/mock-resolver.ts

@@ -68,7 +68,7 @@ export class MockResolver extends Resolver {
 		const r = this.#responseMap.get(value);
 
 		if (!r) {
-			throw new Error('Not registed for mock');
+			throw new Error('Not registered for mock');
 		}
 
 		const object = JSON.parse(r.content);

packages/backend/test/unit/activitypub.ts

@@ -17,7 +17,7 @@ import { GlobalModule } from '@/GlobalModule.js';
 import { CoreModule } from '@/core/CoreModule.js';
 import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
 import { LoggerService } from '@/core/LoggerService.js';
-import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js';
+import type { IActivity, IApDocument, IActor, ICollection, IObject, IOrderedCollection, IOrderedCollectionPage, IPost } from '@/core/activitypub/type.js';
 import { MiMeta, MiNote } from '@/models/_.js';
 import { secureRndstr } from '@/misc/secure-rndstr.js';
 import { DownloadService } from '@/core/DownloadService.js';
@@ -29,6 +29,16 @@ const host = 'https://host1.test';
 
 type NonTransientIActor = IActor & { id: string };
 type NonTransientIPost = IPost & { id: string };
+type NonTransientICollection = ICollection & { id: string };
+type NonTransientIOrderedCollection = IOrderedCollection & { id: string };
+type NonTransientIOrderedCollectionPage = IOrderedCollectionPage & { id: string };
+
+/**
+ * Use when the order of the array is not definitive
+ */
+function deepSortedEqual<T extends unknown[]>(array1: unknown[], array2: T): asserts array1 is T {
+	return assert.deepStrictEqual(array1.sort(), array2.sort());
+}
 
 function createRandomActor({ actorHost = host } = {}): NonTransientIActor {
 	const preferredUsername = secureRndstr(8);
@@ -60,7 +70,7 @@ function createRandomNotes(actor: NonTransientIActor, length: number): NonTransi
 	return new Array(length).fill(null).map(() => createRandomNote(actor));
 }
 
-function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): ICollection {
+function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): NonTransientICollection {
 	const items = createRandomNotes(actor, length);
 
 	return {
@@ -72,6 +82,53 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe
 	};
 }
 
+function createRandomActivities(actor: NonTransientIActor, type: string, length: number): IActivity[] {
+	return new Array(length).fill(null).map((): IActivity => {
+		const note = createRandomNote(actor);
+
+		return {
+			type,
+			id: `${note.id}/activity`,
+			actor,
+			object: note,
+		};
+	});
+}
+
+function createRandomNonPagedOutbox(actor: NonTransientIActor, length: number): NonTransientIOrderedCollection {
+	const orderedItems = createRandomActivities(actor, 'Create', length);
+
+	return {
+		'@context': 'https://www.w3.org/ns/activitystreams',
+		type: 'OrderedCollection',
+		id: actor.outbox as string,
+		totalItems: orderedItems.length,
+		orderedItems,
+	};
+}
+
+function createRandomOutboxPage(actor: NonTransientIActor, id: string, length: number): NonTransientIOrderedCollectionPage {
+	const orderedItems = createRandomActivities(actor, 'Create', length);
+
+	return {
+		'@context': 'https://www.w3.org/ns/activitystreams',
+		type: 'OrderedCollectionPage',
+		id,
+		totalItems: orderedItems.length,
+		orderedItems,
+	};
+}
+
+function createRandomPagedOutbox(actor: NonTransientIActor): NonTransientIOrderedCollection {
+	return {
+		'@context': 'https://www.w3.org/ns/activitystreams',
+		type: 'OrderedCollection',
+		id: actor.outbox as string,
+		totalItems: 10,
+		first: `${actor.outbox}?first`,
+	};
+}
+
 async function createRandomRemoteUser(
 	resolver: MockResolver,
 	personService: ApPersonService,
@@ -196,7 +253,7 @@ describe('ActivityPub', () => {
 
 	describe('Renderer', () => {
 		test('Render an announce with visibility: followers', () => {
-			rendererService.renderAnnounce(null, {
+			rendererService.renderAnnounce('hoge', {
 				createdAt: new Date(0),
 				visibility: 'followers',
 			} as MiNote);
@@ -216,7 +273,7 @@ describe('ActivityPub', () => {
 			await personService.createPerson(actor.id, resolver);
 
 			// All notes in `featured` are same-origin, no need to fetch notes again
-			assert.deepStrictEqual(resolver.remoteGetTrials(), [actor.id, actor.featured]);
+			deepSortedEqual(resolver.remoteGetTrials(), [actor.id, actor.featured, actor.outbox]);
 
 			// Created notes without resolving anything
 			for (const item of featured.items as IPost[]) {
@@ -247,9 +304,9 @@ describe('ActivityPub', () => {
 			await personService.createPerson(actor1.id, resolver);
 
 			// actor2Note is from a different server and needs to be fetched again
-			assert.deepStrictEqual(
+			deepSortedEqual(
 				resolver.remoteGetTrials(),
-				[actor1.id, actor1.featured, actor2Note.id, actor2.id],
+				[actor1.id, actor1.featured, actor1.outbox, actor2Note.id, actor2.id, actor2.outbox],
 			);
 
 			const note = await noteService.fetchNote(actor2Note.id);
@@ -276,6 +333,95 @@ describe('ActivityPub', () => {
 		});
 	});
 
+	describe('Outbox', () => {
+		test('Fetch non-paged outbox from IActor', async () => {
+			const actor = createRandomActor();
+			const outbox = createRandomNonPagedOutbox(actor, 10);
+
+			resolver.register(actor.id, actor);
+			resolver.register(actor.outbox as string, outbox);
+
+			await personService.createPerson(actor.id, resolver);
+
+			deepSortedEqual(
+				resolver.remoteGetTrials(),
+				[actor.id, actor.outbox],
+			);
+
+			for (const item of outbox.orderedItems as IActivity[]) {
+				const note = await noteService.fetchNote(item.object);
+				assert.ok(note);
+				assert.strictEqual(note.text, 'test test foo');
+				assert.strictEqual(note.uri, (item.object as IObject).id);
+			}
+		});
+
+		test('Fetch paged outbox from IActor', async () => {
+			const actor = createRandomActor();
+			const outbox = createRandomPagedOutbox(actor);
+			const page = createRandomOutboxPage(actor, outbox.id, 10);
+
+			resolver.register(actor.id, actor);
+			resolver.register(actor.outbox as string, outbox);
+			resolver.register(outbox.first as string, page);
+
+			await personService.createPerson(actor.id, resolver);
+
+			deepSortedEqual(
+				resolver.remoteGetTrials(),
+				[actor.id, actor.outbox, outbox.first],
+			);
+
+			for (const item of page.orderedItems as IActivity[]) {
+				const note = await noteService.fetchNote(item.object);
+				assert.ok(note);
+				assert.strictEqual(note.text, 'test test foo');
+				assert.strictEqual(note.uri, (item.object as IObject).id);
+			}
+		});
+
+		test('Fetch only the first 20 items', async () => {
+			const actor = createRandomActor();
+			const outbox = createRandomNonPagedOutbox(actor, 200);
+
+			resolver.register(actor.id, actor);
+			resolver.register(actor.outbox as string, outbox);
+
+			await personService.createPerson(actor.id, resolver);
+
+			const items = outbox.orderedItems as IActivity[];
+
+			deepSortedEqual(
+				resolver.remoteGetTrials(),
+				[actor.id, actor.outbox],
+			);
+
+			assert.ok(await noteService.fetchNote(items[19].object));
+			assert.ok(!await noteService.fetchNote(items[20].object));
+		});
+
+		test('Perform only Create activities', async () => {
+			const actor = createRandomActor();
+			const outbox = createRandomNonPagedOutbox(actor, 0);
+			outbox.orderedItems = createRandomActivities(actor, 'Announce', 10);
+
+			resolver.register(actor.id, actor);
+			resolver.register(actor.outbox as string, outbox);
+
+			await personService.createPerson(actor.id, resolver);
+
+			deepSortedEqual(
+				resolver.remoteGetTrials(),
+				[actor.id, actor.outbox],
+			);
+
+			for (const item of outbox.orderedItems as IActivity[]) {
+				const note = await noteService.fetchNote(item.object);
+				assert.ok(!note);
+			}
+		});
+	});
+
 	describe('Images', () => {
 		test('Create images', async () => {
 			const imageObject: IApDocument = {