1
0
forked from mirror/misskey

feat(backend): fetch the first page of outbox when resolving Person

This commit is contained in:
Kagami Sascha Rosylight 2023-07-05 03:38:59 +02:00
parent 92d9946f59
commit 2e1de4fca9
5 changed files with 217 additions and 25 deletions

View File

@ -24,7 +24,7 @@ import { QueueService } from '@/core/QueueService.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, AbuseUserReportsRepository, FollowRequestsRepository, } from '@/models/index.js';
import { bindThis } from '@/decorators.js';
import type { RemoteUser } from '@/models/entities/User.js';
import { getApHrefNullable, getApId, getApIds, getApType, getOneApHrefNullable, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
import { getApHrefNullable, getApId, getApIds, getApType, getOneApHrefNullable, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isOrderedCollectionPage, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -86,10 +86,10 @@ export class ApInboxService {
}
@bindThis
public async performActivity(actor: RemoteUser, activity: IObject) {
if (isCollectionOrOrderedCollection(activity)) {
public async performActivity(actor: RemoteUser, activity: IObject, limit = Infinity) {
if (isCollectionOrOrderedCollection(activity) || isOrderedCollectionPage(activity)) {
const resolver = this.apResolverService.createResolver();
for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems)) {
for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems).slice(0, limit)) {
const act = await resolver.resolve(item);
try {
await this.performOneActivity(actor, act);
@ -366,7 +366,7 @@ export class ApInboxService {
});
if (isPost(object)) {
this.createNote(resolver, actor, object, false, activity);
await this.createNote(resolver, actor, object, false, activity);
} else {
this.logger.warn(`Unknown type: ${getApType(object)}`);
}

View File

@ -10,11 +10,11 @@ import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { isCollectionOrOrderedCollection } from './type.js';
import { isCollectionOrOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js';
import type { IObject, ICollection, IOrderedCollection, IOrderedCollectionPage } from './type.js';
export class Resolver {
private history: Set<string>;
@ -59,6 +59,18 @@ export class Resolver {
}
}
public async resolveOrderedCollectionPage(value: string | IObject): Promise<IOrderedCollectionPage> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
if (isOrderedCollectionPage(collection)) {
return collection;
} else {
throw new Error(`unrecognized collection type: ${collection.type}`);
}
}
@bindThis
public async resolve(value: string | IObject): Promise<IObject> {
if (value == null) {

View File

@ -34,7 +34,8 @@ import { MetaService } from '@/core/MetaService.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import type { AccountMoveService } from '@/core/AccountMoveService.js';
import { checkHttps } from '@/misc/check-https.js';
import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage, isPropertyValue } from '../type.js';
import { ApInboxService } from '../ApInboxService.js';
import { extractApHashtags } from './tag.js';
import type { OnModuleInit } from '@nestjs/common';
import type { ApNoteService } from './ApNoteService.js';
@ -62,6 +63,7 @@ export class ApPersonService implements OnModuleInit {
private apResolverService: ApResolverService;
private apNoteService: ApNoteService;
private apImageService: ApImageService;
private apInboxService: ApInboxService;
private apMfmService: ApMfmService;
private mfmService: MfmService;
private hashtagService: HashtagService;
@ -128,6 +130,7 @@ export class ApPersonService implements OnModuleInit {
this.apResolverService = this.moduleRef.get('ApResolverService');
this.apNoteService = this.moduleRef.get('ApNoteService');
this.apImageService = this.moduleRef.get('ApImageService');
this.apInboxService = this.moduleRef.get('ApInboxService');
this.apMfmService = this.moduleRef.get('ApMfmService');
this.mfmService = this.moduleRef.get('MfmService');
this.hashtagService = this.moduleRef.get('HashtagService');
@ -281,7 +284,7 @@ export class ApPersonService implements OnModuleInit {
// Create user
let user: RemoteUser;
try {
// Start transaction
// Start transaction
await this.db.transaction(async transactionalEntityManager => {
user = await transactionalEntityManager.save(new User({
id: this.idService.genId(),
@ -327,9 +330,9 @@ export class ApPersonService implements OnModuleInit {
}
});
} catch (e) {
// duplicate key error
// duplicate key error
if (isDuplicateKeyValueError(e)) {
// /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応
// /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応
const u = await this.usersRepository.findOneBy({
uri: person.id,
});
@ -406,7 +409,10 @@ export class ApPersonService implements OnModuleInit {
});
//#endregion
await this.updateFeatured(user!.id, resolver).catch(err => this.logger.error(err));
await Promise.all([
this.updateFeatured(user!.id, resolver),
this.updateOutboxFirstPage(user!, person.outbox, resolver),
]).catch(err => this.logger.error(err));
return user!;
}
@ -415,7 +421,7 @@ export class ApPersonService implements OnModuleInit {
* Personの情報を更新します
* Misskeyに対象のPersonが登録されていなければ無視します
*
*
*
* @param uri URI of Person
* @param resolver Resolver
* @param hint Hint of Person object (Personの場合Remote resolveをせずに更新に利用します)
@ -498,7 +504,7 @@ export class ApPersonService implements OnModuleInit {
(!exist.movedToUri && updates.movedToUri) ||
// 移行先がある→別のもの
(exist.movedToUri !== updates.movedToUri && exist.movedToUri && updates.movedToUri);
// 移行先がある→ない、ない→ないは無視
// 移行先がある→ない、ない→ないは無視
if (moving) updates.movedAt = new Date();
@ -598,9 +604,9 @@ export class ApPersonService implements OnModuleInit {
@bindThis
public analyzeAttachments(attachments: IObject | IObject[] | undefined) {
const fields: {
name: string,
value: string
}[] = [];
name: string,
value: string
}[] = [];
if (Array.isArray(attachments)) {
for (const attachment of attachments.filter(isPropertyValue)) {
fields.push({
@ -613,8 +619,35 @@ export class ApPersonService implements OnModuleInit {
return { fields };
}
/**
* Retrieve outbox from an actor object.
*
* This only retrieves the first page for now.
*/
public async updateOutboxFirstPage(user: RemoteUser, outbox: IActor['outbox'], resolver: Resolver): Promise<void> {
// https://www.w3.org/TR/activitypub/#actor-objects
// Outbox is a required property for all actors
if (!outbox) {
throw new Error('No outbox property');
}
this.logger.info(`Fetching the outbox for ${user.uri}: ${outbox}`);
const collection = await resolver.resolveCollection(outbox);
if (!isOrderedCollection(collection)) {
throw new Error('Outbox must be an ordered collection');
}
const firstPage = collection.first ?
await resolver.resolveOrderedCollectionPage(collection.first) :
collection;
// Perform activity but only the first 100 ones
await this.apInboxService.performActivity(user, firstPage, 100);
}
@bindThis
public async updateFeatured(userId: User['id'], resolver?: Resolver) {
public async updateFeatured(userId: User['id'], resolver?: Resolver): Promise<void> {
const user = await this.usersRepository.findOneByOrFail({ id: userId });
if (!this.userEntityService.isRemoteUser(user)) return;
if (!user.featured) return;
@ -631,7 +664,7 @@ export class ApPersonService implements OnModuleInit {
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x)));
// Resolve and regist Notes
// Resolve and register Notes
const limit = promiseLimit<Note | null>(2);
const featuredNotes = await Promise.all(items
.filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも
@ -688,7 +721,7 @@ export class ApPersonService implements OnModuleInit {
// (uriが存在しなかったり応答がなかったりする場合resolvePersonはthrow Errorする)
dst = await this.resolvePerson(src.movedToUri);
}
if (dst.movedToUri === dst.uri) return 'skip: movedTo itself (dst)'; //
if (src.movedToUri !== dst.uri) return 'skip: missmatch uri'; //
if (dst.movedToUri === src.uri) return 'skip: dst.movedToUri === src.uri';

View File

@ -87,16 +87,37 @@ export interface IActivity extends IObject {
};
}
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection
export interface ICollection extends IObject {
type: 'Collection';
totalItems: number;
current?: ICollectionPage | string;
first?: ICollectionPage | string;
last?: ICollectionPage | string;
items: ApObject;
}
export interface IOrderedCollection extends IObject {
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection
export interface IOrderedCollection extends Omit<ICollection, 'type' | 'items'> {
type: 'OrderedCollection';
totalItems: number;
orderedItems: ApObject;
// orderedItems is not defined well
// https://github.com/w3c/activitystreams/issues/494
orderedItems?: ApObject;
}
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage
export interface ICollectionPage extends Omit<ICollection, 'type'> {
type: 'CollectionPage';
partOf?: ICollection | string;
next?: ICollectionPage | string;
prev?: ICollectionPage | string;
}
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollectionpage
export interface IOrderedCollectionPage extends Omit<IOrderedCollection, 'type'>, Omit<ICollectionPage, 'type' | 'items'> {
type: 'OrderedCollectionPage';
startIndex?: number,
}
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
@ -183,6 +204,9 @@ export const isCollection = (object: IObject): object is ICollection =>
export const isOrderedCollection = (object: IObject): object is IOrderedCollection =>
getApType(object) === 'OrderedCollection';
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
isCollection(object) || isOrderedCollection(object);

View File

@ -11,7 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js';
import { CoreModule } from '@/core/CoreModule.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IActor } from '@/core/activitypub/type.js';
import type { IActor, ICreate, IObject, IOrderedCollection, IOrderedCollectionPage, IPost } from '@/core/activitypub/type.js';
import { Note } from '@/models/index.js';
import { secureRndstr } from '@/misc/secure-rndstr.js';
import { MockResolver } from '../misc/mock-resolver.js';
@ -32,6 +32,59 @@ function createRandomActor(): IActor & { id: string } {
};
}
function createRandomCreateActivity(actor: IActor, length: number): ICreate[] {
return new Array(length).fill(null).map((): ICreate => {
const id = secureRndstr(8);
const noteId = `${host}/notes/${id}`;
return {
type: 'Create',
id: `${noteId}/activity`,
actor,
object: {
id: noteId,
type: 'Note',
attributedTo: actor.id,
content: 'test test foo',
} satisfies IPost,
};
});
}
function createRandomNonPagedOutbox(actor: IActor, length: number): IOrderedCollection {
const orderedItems = createRandomCreateActivity(actor, length);
return {
'@context': 'https://www.w3.org/ns/activitystreams',
type: 'OrderedCollection',
id: actor.outbox as string,
totalItems: orderedItems.length,
orderedItems,
};
}
function createRandomOutboxPage(actor: IActor, id: string, length: number): IOrderedCollectionPage {
const orderedItems = createRandomCreateActivity(actor, length);
return {
'@context': 'https://www.w3.org/ns/activitystreams',
type: 'OrderedCollectionPage',
id,
totalItems: orderedItems.length,
orderedItems,
};
}
function createRandomPagedOutbox(actor: IActor): IOrderedCollection {
return {
'@context': 'https://www.w3.org/ns/activitystreams',
type: 'OrderedCollection',
id: actor.outbox as string,
totalItems: 10,
first: `${actor.outbox}?first`,
};
}
describe('ActivityPub', () => {
let noteService: ApNoteService;
let personService: ApPersonService;
@ -53,7 +106,7 @@ describe('ActivityPub', () => {
// Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error
const federatedInstanceService = app.get<FederatedInstanceService>(FederatedInstanceService);
jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => {}));
jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => { }));
});
describe('Parse minimum object', () => {
@ -126,4 +179,74 @@ describe('ActivityPub', () => {
} as Note);
});
});
describe('Outbox', () => {
test('Fetch non-paged outbox from IActor', async () => {
const actor = createRandomActor();
const outbox = createRandomNonPagedOutbox(actor, 10);
resolver._register(actor.id, actor);
resolver._register(actor.outbox as string, outbox);
// XXX: This shouldn't be needed as the collection already has the full information
// But somehow the resolver currently doesn't use it at all and always fetches again
for (const item of outbox.orderedItems as IObject[]) {
resolver._register(item.id!, item);
}
await personService.createPerson(actor.id, resolver);
for (const item of outbox.orderedItems as ICreate[]) {
const note = await noteService.fetchNote(item.object);
assert.ok(note);
assert.strictEqual(note.text, 'test test foo');
assert.strictEqual(note.uri, (item.object as IObject).id);
}
});
test('Fetch paged outbox from IActor', async () => {
const actor = createRandomActor();
const outbox = createRandomPagedOutbox(actor);
const page = createRandomOutboxPage(actor, outbox.id!, 10);
resolver._register(actor.id, actor);
resolver._register(actor.outbox as string, outbox);
resolver._register(outbox.first as string, page);
// XXX: This shouldn't be needed as the collection already has the full information
// But somehow the resolver currently doesn't use it at all and always fetches again
for (const item of page.orderedItems as IObject[]) {
resolver._register(item.id!, item);
}
await personService.createPerson(actor.id, resolver);
for (const item of page.orderedItems as ICreate[]) {
const note = await noteService.fetchNote(item.object);
assert.ok(note);
assert.strictEqual(note.text, 'test test foo');
assert.strictEqual(note.uri, (item.object as IObject).id);
}
});
test('Fetch only the first 100 items', async () => {
const actor = createRandomActor();
const outbox = createRandomNonPagedOutbox(actor, 200);
resolver._register(actor.id, actor);
resolver._register(actor.outbox as string, outbox);
// XXX: This shouldn't be needed as the collection already has the full information
// But somehow the resolver currently doesn't use it at all and always fetches again
for (const item of outbox.orderedItems as IObject[]) {
resolver._register(item.id!, item);
}
await personService.createPerson(actor.id, resolver);
const items = outbox.orderedItems as ICreate[];
assert.ok(await noteService.fetchNote(items[99].object));
assert.ok(!await noteService.fetchNote(items[100].object));
});
});
});