From f0c4565787eb3dc3ff16981b2a5f9235e5316e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sat, 30 Nov 2024 16:52:08 +0900 Subject: [PATCH 1/5] =?UTF-8?q?returning=E3=82=92=E5=90=AB=E3=82=80?= =?UTF-8?q?=E3=82=AF=E3=82=A8=E3=83=AA=E3=82=92master=E3=81=A7=E5=8B=95?= =?UTF-8?q?=E3=81=8B=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/models/_.ts | 39 ++++++++++++++++++-------------- packages/backend/src/postgres.ts | 19 +++++++++++----- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index c72bdaa727..458c9b9e20 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -3,13 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { FindOneOptions, InsertQueryBuilder, ObjectLiteral, Repository, SelectQueryBuilder, TypeORMError } from 'typeorm'; -import { DriverUtils } from 'typeorm/driver/DriverUtils.js'; +import { FindOneOptions, InsertQueryBuilder, ObjectLiteral, Repository, SelectQueryBuilder } from 'typeorm'; import { RelationCountLoader } from 'typeorm/query-builder/relation-count/RelationCountLoader.js'; import { RelationIdLoader } from 'typeorm/query-builder/relation-id/RelationIdLoader.js'; -import { RawSqlResultsToEntityTransformer } from 'typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer.js'; -import { ObjectUtils } from 'typeorm/util/ObjectUtils.js'; -import { OrmUtils } from 'typeorm/util/OrmUtils.js'; +import { + RawSqlResultsToEntityTransformer, +} from 'typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer.js'; import { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; import { MiAbuseReportNotificationRecipient } from '@/models/AbuseReportNotificationRecipient.js'; import { MiAccessToken } from '@/models/AccessToken.js'; @@ -99,19 +98,25 @@ export const miRepository = { mainAlias.name = 't'; const columnNames = this.createTableColumnNames(); queryBuilder.returning(columnNames.reduce((a, c) => `${a}, ${queryBuilder.escape(c)}`, '').slice(2)); - const builder = this.createQueryBuilder().addCommonTableExpression(queryBuilder, 'cte', { columnNames }); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - builder.expressionMap.mainAlias!.tablePath = 'cte'; - this.selectAliasColumnNames(queryBuilder, builder); - if (findOptions) { - builder.setFindOptions(findOptions); + + const queryRunner = this.manager.connection.createQueryRunner('master'); + try { + const builder = this.createQueryBuilder(undefined, queryRunner).addCommonTableExpression(queryBuilder, 'cte', { columnNames }); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + builder.expressionMap.mainAlias!.tablePath = 'cte'; + this.selectAliasColumnNames(queryBuilder, builder); + if (findOptions) { + builder.setFindOptions(findOptions); + } + const raw = await builder.execute(); + mainAlias.name = name; + const relationId = await new RelationIdLoader(builder.connection, this.queryRunner, builder.expressionMap.relationIdAttributes).load(raw); + const relationCount = await new RelationCountLoader(builder.connection, this.queryRunner, builder.expressionMap.relationCountAttributes).load(raw); + const result = new RawSqlResultsToEntityTransformer(builder.expressionMap, builder.connection.driver, relationId, relationCount, this.queryRunner).transform(raw, mainAlias); + return result[0]; + } finally { + await queryRunner.release(); } - const raw = await builder.execute(); - mainAlias.name = name; - const relationId = await new RelationIdLoader(builder.connection, this.queryRunner, builder.expressionMap.relationIdAttributes).load(raw); - const relationCount = await new RelationCountLoader(builder.connection, this.queryRunner, builder.expressionMap.relationCountAttributes).load(raw); - const result = new RawSqlResultsToEntityTransformer(builder.expressionMap, builder.connection.driver, relationId, relationCount, this.queryRunner).transform(raw, mainAlias); - return result[0]; }, selectAliasColumnNames(queryBuilder, builder) { let selectOrAddSelect = (selection: string, selectionAliasName?: string) => { diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 251a03c303..05f2340adf 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -7,6 +7,7 @@ import pg from 'pg'; import { DataSource, Logger } from 'typeorm'; import * as highlight from 'cli-highlight'; +import { type QueryRunner } from 'typeorm'; import { entities as charts } from '@/core/chart/entities.js'; import { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; @@ -98,18 +99,24 @@ class MyCustomLogger implements Logger { } @bindThis - public logQuery(query: string, parameters?: any[]) { - sqlLogger.info(this.highlight(query).substring(0, 100)); + private replicationMode(runner?: QueryRunner) { + const mode = runner?.getReplicationMode(); + return mode ? `[${mode}]` : '[default]'; } @bindThis - public logQueryError(error: string, query: string, parameters?: any[]) { - sqlLogger.error(this.highlight(query)); + public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { + sqlLogger.info(this.replicationMode(queryRunner) + ' ' + this.highlight(query).substring(0, 100)); } @bindThis - public logQuerySlow(time: number, query: string, parameters?: any[]) { - sqlLogger.warn(this.highlight(query)); + public logQueryError(error: string, query: string, parameters?: any[], queryRunner?: QueryRunner) { + sqlLogger.error(this.replicationMode(queryRunner) + ' ' + this.highlight(query)); + } + + @bindThis + public logQuerySlow(time: number, query: string, parameters?: any[], queryRunner?: QueryRunner) { + sqlLogger.warn(this.replicationMode(queryRunner) + ' ' + this.highlight(query)); } @bindThis From e4e388475886a6eddcdbcf1c4f49e9d4465c418d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:14:10 +0900 Subject: [PATCH 2/5] wip --- packages/backend/src/models/_.ts | 4 ++++ packages/backend/src/postgres.ts | 30 ++++++++++++++++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index 458c9b9e20..04cd78daa4 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -9,6 +9,7 @@ import { RelationIdLoader } from 'typeorm/query-builder/relation-id/RelationIdLo import { RawSqlResultsToEntityTransformer, } from 'typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer.js'; +import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js'; import { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; import { MiAbuseReportNotificationRecipient } from '@/models/AbuseReportNotificationRecipient.js'; import { MiAccessToken } from '@/models/AccessToken.js'; @@ -91,6 +92,9 @@ export const miRepository = { return this.metadata.columns.filter(column => column.isSelect && !column.isVirtual).map(column => column.databaseName); }, async insertOne(entity, findOptions?) { + const opt = this.manager.connection.options as PostgresConnectionOptions; + console.log(opt.replication); + const queryBuilder = this.createQueryBuilder().insert().values(entity); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const mainAlias = queryBuilder.expressionMap.mainAlias!; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 05f2340adf..a680ddd28f 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -91,32 +91,38 @@ export const dbLogger = new MisskeyLogger('db'); const sqlLogger = dbLogger.createSubLogger('sql', 'gray'); class MyCustomLogger implements Logger { - @bindThis - private highlight(sql: string) { - return highlight.highlight(sql, { - language: 'sql', ignoreIllegals: true, - }); + constructor( + private printReplicationMode?: boolean, + ) { } @bindThis - private replicationMode(runner?: QueryRunner) { - const mode = runner?.getReplicationMode(); - return mode ? `[${mode}]` : '[default]'; + private highlight(sql: string, queryRunner?: QueryRunner) { + const result = highlight.highlight(sql, { + language: 'sql', ignoreIllegals: true, + }); + + if (this.printReplicationMode && queryRunner) { + const mode = queryRunner.getReplicationMode(); + return `[${mode}] ${result}`; + } else { + return result; + } } @bindThis public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { - sqlLogger.info(this.replicationMode(queryRunner) + ' ' + this.highlight(query).substring(0, 100)); + sqlLogger.info(this.highlight(query, queryRunner).substring(0, 100)); } @bindThis public logQueryError(error: string, query: string, parameters?: any[], queryRunner?: QueryRunner) { - sqlLogger.error(this.replicationMode(queryRunner) + ' ' + this.highlight(query)); + sqlLogger.error(this.highlight(query, queryRunner)); } @bindThis public logQuerySlow(time: number, query: string, parameters?: any[], queryRunner?: QueryRunner) { - sqlLogger.warn(this.replicationMode(queryRunner) + ' ' + this.highlight(query)); + sqlLogger.warn(this.highlight(query, queryRunner)); } @bindThis @@ -254,7 +260,7 @@ export function createPostgresDataSource(config: Config) { }, } : false, logging: log, - logger: log ? new MyCustomLogger() : undefined, + logger: log ? new MyCustomLogger(config.dbReplications) : undefined, maxQueryExecutionTime: 300, entities: entities, migrations: ['../../migration/*.js'], From f2c6485a748338e7d07b4c78cb4e52b883c265d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:51:58 +0900 Subject: [PATCH 3/5] wip --- packages/backend/src/models/_.ts | 61 +++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index 04cd78daa4..5973abe381 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -3,7 +3,14 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { FindOneOptions, InsertQueryBuilder, ObjectLiteral, Repository, SelectQueryBuilder } from 'typeorm'; +import { + FindOneOptions, + InsertQueryBuilder, + ObjectLiteral, + QueryRunner, + Repository, + SelectQueryBuilder, +} from 'typeorm'; import { RelationCountLoader } from 'typeorm/query-builder/relation-count/RelationCountLoader.js'; import { RelationIdLoader } from 'typeorm/query-builder/relation-id/RelationIdLoader.js'; import { @@ -83,7 +90,11 @@ import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialE export interface MiRepository { createTableColumnNames(this: Repository & MiRepository): string[]; + insertOne(this: Repository & MiRepository, entity: QueryDeepPartialEntity, findOptions?: Pick, 'relations'>): Promise; + + insertOneImpl(this: Repository & MiRepository, entity: QueryDeepPartialEntity, findOptions?: Pick, 'relations'>, queryRunner?: QueryRunner): Promise; + selectAliasColumnNames(this: Repository & MiRepository, queryBuilder: InsertQueryBuilder, builder: SelectQueryBuilder): void; } @@ -93,7 +104,19 @@ export const miRepository = { }, async insertOne(entity, findOptions?) { const opt = this.manager.connection.options as PostgresConnectionOptions; - console.log(opt.replication); + if (opt.replication) { + const queryRunner = this.manager.connection.createQueryRunner('master'); + try { + return this.insertOneImpl(entity, findOptions, queryRunner); + } finally { + await queryRunner.release(); + } + } else { + return this.insertOneImpl(entity, findOptions); + } + }, + async insertOneImpl(entity, findOptions?, queryRunner?) { + // ---- insert + returningの結果を共通テーブル式(CTE)に保持するクエリを生成 ---- const queryBuilder = this.createQueryBuilder().insert().values(entity); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -103,24 +126,20 @@ export const miRepository = { const columnNames = this.createTableColumnNames(); queryBuilder.returning(columnNames.reduce((a, c) => `${a}, ${queryBuilder.escape(c)}`, '').slice(2)); - const queryRunner = this.manager.connection.createQueryRunner('master'); - try { - const builder = this.createQueryBuilder(undefined, queryRunner).addCommonTableExpression(queryBuilder, 'cte', { columnNames }); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - builder.expressionMap.mainAlias!.tablePath = 'cte'; - this.selectAliasColumnNames(queryBuilder, builder); - if (findOptions) { - builder.setFindOptions(findOptions); - } - const raw = await builder.execute(); - mainAlias.name = name; - const relationId = await new RelationIdLoader(builder.connection, this.queryRunner, builder.expressionMap.relationIdAttributes).load(raw); - const relationCount = await new RelationCountLoader(builder.connection, this.queryRunner, builder.expressionMap.relationCountAttributes).load(raw); - const result = new RawSqlResultsToEntityTransformer(builder.expressionMap, builder.connection.driver, relationId, relationCount, this.queryRunner).transform(raw, mainAlias); - return result[0]; - } finally { - await queryRunner.release(); + // ---- 共通テーブル式(CTE)から結果を取得 ---- + const builder = this.createQueryBuilder(undefined, queryRunner).addCommonTableExpression(queryBuilder, 'cte', { columnNames }); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + builder.expressionMap.mainAlias!.tablePath = 'cte'; + this.selectAliasColumnNames(queryBuilder, builder); + if (findOptions) { + builder.setFindOptions(findOptions); } + const raw = await builder.execute(); + mainAlias.name = name; + const relationId = await new RelationIdLoader(builder.connection, this.queryRunner, builder.expressionMap.relationIdAttributes).load(raw); + const relationCount = await new RelationCountLoader(builder.connection, this.queryRunner, builder.expressionMap.relationCountAttributes).load(raw); + const result = new RawSqlResultsToEntityTransformer(builder.expressionMap, builder.connection.driver, relationId, relationCount, this.queryRunner).transform(raw, mainAlias); + return result[0]; }, selectAliasColumnNames(queryBuilder, builder) { let selectOrAddSelect = (selection: string, selectionAliasName?: string) => { @@ -206,7 +225,9 @@ export { }; export type AbuseUserReportsRepository = Repository & MiRepository; -export type AbuseReportNotificationRecipientRepository = Repository & MiRepository; +export type AbuseReportNotificationRecipientRepository = + Repository + & MiRepository; export type AccessTokensRepository = Repository & MiRepository; export type AdsRepository = Repository & MiRepository; export type AnnouncementsRepository = Repository & MiRepository; From 302b227f5157b22f42d2190d0ae3c3c1d3b97f46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:45:18 +0900 Subject: [PATCH 4/5] fix CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7896a42883..c84c1271d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Server - Fix: ユーザーのプロフィール画面をアドレス入力などで直接表示した際に概要タブの描画に失敗する問題の修正( #15032 ) - +- Fix: リードレプリカ設定時にレコードの追加・更新・削除を伴うクエリを発行した際はmasterノードで実行されるように調整( #10897 ) ## 2024.11.0 From ca03491301c155da2180b29cab144c8a5f84485b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Tue, 10 Dec 2024 19:00:15 +0900 Subject: [PATCH 5/5] =?UTF-8?q?=E8=AA=BF=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/postgres.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index a680ddd28f..df865b2a01 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -5,9 +5,8 @@ // https://github.com/typeorm/typeorm/issues/2400 import pg from 'pg'; -import { DataSource, Logger } from 'typeorm'; +import { DataSource, Logger, type QueryRunner } from 'typeorm'; import * as highlight from 'cli-highlight'; -import { type QueryRunner } from 'typeorm'; import { entities as charts } from '@/core/chart/entities.js'; import { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; @@ -97,32 +96,36 @@ class MyCustomLogger implements Logger { } @bindThis - private highlight(sql: string, queryRunner?: QueryRunner) { - const result = highlight.highlight(sql, { + private highlight(sql: string) { + return highlight.highlight(sql, { language: 'sql', ignoreIllegals: true, }); + } - if (this.printReplicationMode && queryRunner) { - const mode = queryRunner.getReplicationMode(); - return `[${mode}] ${result}`; + @bindThis + private appendPrefixIfNeeded(message: string, opts?: { + queryRunner?: QueryRunner; + }): string { + if (this.printReplicationMode && opts?.queryRunner) { + return `[${opts.queryRunner.getReplicationMode()}] ${message}`; } else { - return result; + return message; } } @bindThis public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { - sqlLogger.info(this.highlight(query, queryRunner).substring(0, 100)); + sqlLogger.info(this.appendPrefixIfNeeded(this.highlight(query).substring(0, 100), { queryRunner })); } @bindThis public logQueryError(error: string, query: string, parameters?: any[], queryRunner?: QueryRunner) { - sqlLogger.error(this.highlight(query, queryRunner)); + sqlLogger.error(this.appendPrefixIfNeeded(this.highlight(query), { queryRunner })); } @bindThis public logQuerySlow(time: number, query: string, parameters?: any[], queryRunner?: QueryRunner) { - sqlLogger.warn(this.highlight(query, queryRunner)); + sqlLogger.warn(this.appendPrefixIfNeeded(this.highlight(query), { queryRunner })); } @bindThis