diff --git a/CHANGELOG.md b/CHANGELOG.md index ee2f11b156..28515c74ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,13 @@ --> +## 2023.10.0 +### NOTE +- muted_noteテーブルは使われなくなったため手動で削除を行ってください。 + +### Server +- タイムライン取得時のパフォーマンスを改善 + ## 2023.9.3 ### General - Enhance: ノートの翻訳機能の利用可否をロールで設定可能に diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index d9f31079ca..5bb423ae35 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -478,16 +478,12 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); - if (data.reply) { + if (data.visibility === 'public' || data.visibility === 'home') { + this.pushToTl(note, user); + } else if (data.visibility === 'followers') { + this.pushToTl(note, user); + } else if (data.visibility === 'specified') { // TODO - } else { - if (data.visibility === 'public' || data.visibility === 'home') { - this.pushToTl(note, user); - } else if (data.visibility === 'followers') { - this.pushToTl(note, user); - } else if (data.visibility === 'specified') { - // TODO - } } this.antennaService.addNoteToAntennas(note, user); @@ -802,78 +798,104 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { - // TODO: 休眠ユーザーを弾く - // TODO: チャンネルフォロー - // TODO: キャッシュ？ - const followings = await this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - }, - select: ['followerId'], - }); - - let userLists = await this.userListJoiningsRepository.find({ - where: { - userId: user.id, - }, - select: ['userListId'], - }); - const redisPipeline = this.redisClient.pipeline(); - // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする - for (const following of followings) { + if (note.replyId) { + if (note.visibility === 'public' || note.visibility === 'home') { + redisPipeline.xadd( + `userTimelineWithReplies:${user.id}`, + 'MAXLEN', '~', '300', + '*', + 'note', note.id); + } + } else { + // TODO: 休眠ユーザーを弾く + // TODO: チャンネルフォロー + // TODO: キャッシュ？ + const followings = await this.followingsRepository.find({ + where: { + followeeId: user.id, + followerHost: IsNull(), + }, + select: ['followerId'], + }); + + let userLists = await this.userListJoiningsRepository.find({ + where: { + userId: user.id, + }, + select: ['userListId'], + }); + + // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする + for (const following of followings) { + redisPipeline.xadd( + `homeTimeline:${following.followerId}`, + 'MAXLEN', '~', '300', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `homeTimelineWithFiles:${following.followerId}`, + 'MAXLEN', '~', '300', + '*', + 'note', note.id); + } + } + + if (note.visibility === 'followers') { + // TODO: 重そうだから何とかしたい Set 使う？ + userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListId)); + } + + for (const userList of userLists) { + redisPipeline.xadd( + `userListTimeline:${userList.userListId}`, + 'MAXLEN', '~', '300', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `userListTimelineWithFiles:${userList.userListId}`, + 'MAXLEN', '~', '300', + '*', + 'note', note.id); + } + } + redisPipeline.xadd( - `homeTimeline:${following.followerId}`, + `homeTimeline:${user.id}`, 'MAXLEN', '~', '300', '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( - `homeTimelineWithFiles:${following.followerId}`, + `homeTimelineWithFiles:${user.id}`, 'MAXLEN', '~', '300', '*', 'note', note.id); } - } - if (note.visibility === 'followers') { - // TODO: 重そうだから何とかしたい Set 使う？ - userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListId)); - } - - for (const userList of userLists) { - redisPipeline.xadd( - `userListTimeline:${userList.userListId}`, - 'MAXLEN', '~', '300', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { + if (note.visibility === 'public' || note.visibility === 'home') { redisPipeline.xadd( - `userListTimelineWithFiles:${userList.userListId}`, + `userTimeline:${user.id}`, 'MAXLEN', '~', '300', '*', 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `userTimelineWithFiles:${user.id}`, + 'MAXLEN', '~', '300', + '*', + 'note', note.id); + } } } - redisPipeline.xadd( - `homeTimeline:${user.id}`, - 'MAXLEN', '~', '300', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', '300', - '*', - 'note', note.id); - } - redisPipeline.exec(); } diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index c883c96ba2..822b4a9448 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -205,7 +205,6 @@ import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js'; import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js'; -import * as ep___i_getWordMutedNotesCount from './endpoints/i/get-word-muted-notes-count.js'; import * as ep___i_importBlocking from './endpoints/i/import-blocking.js'; import * as ep___i_importFollowing from './endpoints/i/import-following.js'; import * as ep___i_importMuting from './endpoints/i/import-muting.js'; @@ -554,7 +553,6 @@ const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass: const $i_favorites: Provider = { provide: 'ep:i/favorites', useClass: ep___i_favorites.default }; const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep___i_gallery_likes.default }; const $i_gallery_posts: Provider = { provide: 'ep:i/gallery/posts', useClass: ep___i_gallery_posts.default }; -const $i_getWordMutedNotesCount: Provider = { provide: 'ep:i/get-word-muted-notes-count', useClass: ep___i_getWordMutedNotesCount.default }; const $i_importBlocking: Provider = { provide: 'ep:i/import-blocking', useClass: ep___i_importBlocking.default }; const $i_importFollowing: Provider = { provide: 'ep:i/import-following', useClass: ep___i_importFollowing.default }; const $i_importMuting: Provider = { provide: 'ep:i/import-muting', useClass: ep___i_importMuting.default }; @@ -907,7 +905,6 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_favorites, $i_gallery_likes, $i_gallery_posts, - $i_getWordMutedNotesCount, $i_importBlocking, $i_importFollowing, $i_importMuting, @@ -1254,7 +1251,6 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_favorites, $i_gallery_likes, $i_gallery_posts, - $i_getWordMutedNotesCount, $i_importBlocking, $i_importFollowing, $i_importMuting, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index b40d654f9c..a46f136fb2 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -205,7 +205,6 @@ import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js'; import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js'; -import * as ep___i_getWordMutedNotesCount from './endpoints/i/get-word-muted-notes-count.js'; import * as ep___i_importBlocking from './endpoints/i/import-blocking.js'; import * as ep___i_importFollowing from './endpoints/i/import-following.js'; import * as ep___i_importMuting from './endpoints/i/import-muting.js'; @@ -552,7 +551,6 @@ const eps = [ ['i/favorites', ep___i_favorites], ['i/gallery/likes', ep___i_gallery_likes], ['i/gallery/posts', ep___i_gallery_posts], - ['i/get-word-muted-notes-count', ep___i_getWordMutedNotesCount], ['i/import-blocking', ep___i_importBlocking], ['i/import-following', ep___i_importFollowing], ['i/import-muting', ep___i_importMuting], diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index c76cdf28ff..da6fbae8fd 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -68,9 +68,11 @@ export default class extends Endpoint { // eslint- const [ userIdsWhoMeMuting, userIdsWhoMeMutingRenotes, + userIdsWhoBlockingMe, ] = await Promise.all([ this.cacheService.userMutingsCache.fetch(me.id), this.cacheService.renoteMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), ]); let timeline: MiNote[] = []; @@ -103,15 +105,11 @@ export default class extends Endpoint { // eslint- timeline = await query.getMany(); - // ミュート等考慮 timeline = timeline.filter(note => { - // TODO: インスタンスミュートの考慮 - // TODO: ブロックの考慮 - if (note.userId === me.id) { return true; } - + if (isUserRelated(note, userIdsWhoBlockingMe)) return false; if (isUserRelated(note, userIdsWhoMeMuting)) return false; if (note.renoteId) { if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 8075c0d8ce..08d765144c 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -94,9 +94,11 @@ export default class extends Endpoint { // eslint- const [ userIdsWhoMeMuting, userIdsWhoMeMutingRenotes, + userIdsWhoBlockingMe, ] = await Promise.all([ this.cacheService.userMutingsCache.fetch(me.id), this.cacheService.renoteMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), ]); let timeline: MiNote[] = []; @@ -129,11 +131,11 @@ export default class extends Endpoint { // eslint- timeline = await query.getMany(); - // ミュート等考慮 timeline = timeline.filter(note => { - // TODO: インスタンスミュートの考慮 - // TODO: ブロックの考慮 - + if (note.userId === me.id) { + return true; + } + if (isUserRelated(note, userIdsWhoBlockingMe)) return false; if (isUserRelated(note, userIdsWhoMeMuting)) return false; if (note.renoteId) { if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index e660a0bb25..6d8a2386d7 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -5,12 +5,14 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { NotesRepository } from '@/models/_.js'; +import * as Redis from 'ioredis'; +import type { MiNote, NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { IdService } from '@/core/IdService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -50,9 +52,6 @@ export const paramDef = { untilDate: { type: 'integer' }, includeMyRenotes: { type: 'boolean', default: true }, withFiles: { type: 'boolean', default: false }, - fileType: { type: 'array', items: { - type: 'string', - } }, excludeNsfw: { type: 'boolean', default: false }, }, required: ['userId'], @@ -61,87 +60,63 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + @Inject(DI.notesRepository) private notesRepository: NotesRepository, private noteEntityService: NoteEntityService, - private queryService: QueryService, private getterService: GetterService, + private cacheService: CacheService, + private idService: IdService, ) { super(meta, paramDef, async (ps, me) => { - // Lookup user - const user = await this.getterService.getUser(ps.userId).catch(err => { - if (err.id === '15348ddd-432d-49c2-8a5a-8069753becff') throw new ApiError(meta.errors.noSuchUser); - throw err; - }); + let timeline: MiNote[] = []; - //#region Construct query - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.userId = :userId', { userId: user.id }) + const limit = ps.limit + (ps.untilId ? 1 : 0); // untilIdに指定したものも含まれるため+1 + let noteIdsRes: [string, string[]][] = []; + + if (!ps.sinceId && !ps.sinceDate) { + noteIdsRes = await this.redisClient.xrevrange( + ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : ps.withReplies ? `userTimelineWithReplies:${ps.userId}` : `userTimeline:${ps.userId}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + '-', + 'COUNT', limit); + } + + const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId); + + if (noteIds.length === 0) { + return []; + } + + const isFollowing = me ? (await this.cacheService.userFollowingsCache.fetch(me.id)).has(ps.userId) : false; + + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('note.channel', 'channel') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); - query.andWhere(new Brackets(qb => { - qb.orWhere('note.channelId IS NULL'); - qb.orWhere('channel.isSensitive = false'); - })); + timeline = await query.getMany(); - this.queryService.generateVisibilityQuery(query, me); - if (me) { - this.queryService.generateMutedUserQuery(query, me, user); - this.queryService.generateBlockedUserQuery(query, me); - } - - if (ps.withFiles) { - query.andWhere('note.fileIds != \'{}\''); - } - - if (ps.fileType != null) { - query.andWhere('note.fileIds != \'{}\''); - query.andWhere(new Brackets(qb => { - for (const type of ps.fileType!) { - const i = ps.fileType!.indexOf(type); - qb.orWhere(`:type${i} = ANY(note.attachedFileTypes)`, { [`type${i}`]: type }); + timeline = timeline.filter(note => { + if (note.renoteId) { + if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { + if (ps.withRenotes === false) return false; } - })); - - if (ps.excludeNsfw) { - query.andWhere('note.cw IS NULL'); - query.andWhere('0 = (SELECT COUNT(*) FROM drive_file df WHERE df.id = ANY(note."fileIds") AND df."isSensitive" = TRUE)'); } - } - if (!ps.withReplies) { - query.andWhere('note.replyId IS NULL'); - } + if (note.visibility === 'followers' && !isFollowing) return false; - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere(new Brackets(qb => { - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - })); - })); - } + return true; + }); - if (ps.includeMyRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :userId', { userId: user.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - //#endregion - - const timeline = await query.limit(ps.limit).getMany(); + timeline.sort((a, b) => a.id > b.id ? -1 : 1); return await this.noteEntityService.packMany(timeline, me); }); diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index f0fc47c207..3a0857a8fd 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -1381,10 +1381,6 @@ export type Endpoints = { req: TODO; res: TODO; }; - 'i/get-word-muted-notes-count': { - req: TODO; - res: TODO; - }; 'i/import-following': { req: TODO; res: TODO; diff --git a/packages/misskey-js/src/api.types.ts b/packages/misskey-js/src/api.types.ts index e69d8324a1..a7a2ea1b36 100644 --- a/packages/misskey-js/src/api.types.ts +++ b/packages/misskey-js/src/api.types.ts @@ -371,7 +371,6 @@ export type Endpoints = { 'i/favorites': { req: { limit?: number; sinceId?: NoteFavorite['id']; untilId?: NoteFavorite['id']; }; res: NoteFavorite[]; }; 'i/gallery/likes': { req: TODO; res: TODO; }; 'i/gallery/posts': { req: TODO; res: TODO; }; - 'i/get-word-muted-notes-count': { req: TODO; res: TODO; }; 'i/import-following': { req: TODO; res: TODO; }; 'i/import-user-lists': { req: TODO; res: TODO; }; 'i/move': { req: TODO; res: TODO; };