From 8a0a54700cdb946c333788189ac30716812195de Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 09:57:37 +0000 Subject: [PATCH 1/7] =?UTF-8?q?enhance(backend):=20=E8=A8=AD=E5=AE=9A?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=A7ioredis=E3=81=AE?= =?UTF-8?q?=E5=85=A8=E3=81=A6=E3=81=AE=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=82=92=E6=8C=87=E5=AE=9A=E5=8F=AF=E8=83=BD=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .config/example.yml | 9 +++++++++ CHANGELOG.md | 1 + packages/backend/check_connect.js | 1 + packages/backend/src/GlobalModule.ts | 3 +++ packages/backend/src/config.ts | 4 ++++ packages/backend/src/queue/const.ts | 1 + 6 files changed, 19 insertions(+) diff --git a/.config/example.yml b/.config/example.yml index fcf22debce70..cbb9c0bd514a 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -82,6 +82,9 @@ redis: #pass: example-pass #prefix: example-prefix #db: 1 + # Extra ioredis options + #extra: + # username: example-username #redisForPubsub: # host: localhost @@ -90,6 +93,9 @@ redis: # #pass: example-pass # #prefix: example-prefix # #db: 1 +# # Extra ioredis options +# #extra: +# # username: example-username #redisForJobQueue: # host: localhost @@ -98,6 +104,9 @@ redis: # #pass: example-pass # #prefix: example-prefix # #db: 1 +# # Extra ioredis options +# #extra: +# # username: example-username # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4b2ad31a8f..57935f0641f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,7 @@ ### Server - bullをbull-mqにアップグレードし、ジョブキューのパフォーマンスを改善 - ストリーミングのパフォーマンスを改善 +- 設定ファイルでioredisの全てのオプションを指定可能に (redis.extra) - Fix: 無効化されたアンテナにアクセスがあった際に再度有効化するように - Fix: お知らせの画像URLを空にできない問題を修正 - Fix: i/notificationsのsinceIdが機能しない問題を修正 diff --git a/packages/backend/check_connect.js b/packages/backend/check_connect.js index ef0a350fbfb8..bea8abb82cad 100644 --- a/packages/backend/check_connect.js +++ b/packages/backend/check_connect.js @@ -9,6 +9,7 @@ const redis = new Redis({ password: config.redis.pass, keyPrefix: `${config.redis.prefix}:`, db: config.redis.db ?? 0, + ...(config.redis.extra ?? {}), }); redis.on('connect', () => redis.disconnect()); diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 406e3192bb62..4b1de080d9ea 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -48,6 +48,7 @@ const $redis: Provider = { password: config.redis.pass, keyPrefix: `${config.redis.prefix}:`, db: config.redis.db ?? 0, + ...(config.redis.extra ?? {}), }); }, inject: [DI.config], @@ -63,6 +64,7 @@ const $redisForPub: Provider = { password: config.redisForPubsub.pass, keyPrefix: `${config.redisForPubsub.prefix}:`, db: config.redisForPubsub.db ?? 0, + ...(config.redisForPubsub.extra ?? {}), }); return redis; }, @@ -79,6 +81,7 @@ const $redisForSub: Provider = { password: config.redisForPubsub.pass, keyPrefix: `${config.redisForPubsub.prefix}:`, db: config.redisForPubsub.db ?? 0, + ...(config.redisForPubsub.extra ?? {}), }); redis.subscribe(config.host); return redis; diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 8c312256da87..d8a8ffbb2ada 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -6,6 +6,7 @@ import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, resolve } from 'node:path'; import * as yaml from 'js-yaml'; +import type { RedisOptions } from 'ioredis'; /** * ユーザーが設定する必要のある情報 @@ -42,6 +43,7 @@ export type Source = { pass: string; db?: number; prefix?: string; + extra?: RedisOptions; }; redisForPubsub?: { host: string; @@ -50,6 +52,7 @@ export type Source = { pass: string; db?: number; prefix?: string; + extra?: RedisOptions; }; redisForJobQueue?: { host: string; @@ -58,6 +61,7 @@ export type Source = { pass: string; db?: number; prefix?: string; + extra?: RedisOptions; }; meilisearch?: { host: string; diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index d240fe70e0c5..4b34daadf331 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -20,6 +20,7 @@ export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof t family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, password: config.redisForJobQueue.pass, db: config.redisForJobQueue.db ?? 0, + ...(config.redisForJobQueue.extra ?? {}), }, prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, }; From 11197ccd0b456f763e39400cc2418ffa29ef8ef0 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 10:11:58 +0000 Subject: [PATCH 2/7] yappa kousuru --- .config/example.yml | 15 +++---- packages/backend/check_connect.js | 10 +---- packages/backend/src/GlobalModule.ts | 30 ++------------ packages/backend/src/config.ts | 59 +++++++++++++--------------- packages/backend/src/queue/const.ts | 9 +---- 5 files changed, 38 insertions(+), 85 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index cbb9c0bd514a..b996a83fb50a 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -82,9 +82,8 @@ redis: #pass: example-pass #prefix: example-prefix #db: 1 - # Extra ioredis options - #extra: - # username: example-username + # You can specify more ioredis options... + #username: example-username #redisForPubsub: # host: localhost @@ -93,9 +92,8 @@ redis: # #pass: example-pass # #prefix: example-prefix # #db: 1 -# # Extra ioredis options -# #extra: -# # username: example-username +# # You can specify more ioredis options... +# #username: example-username #redisForJobQueue: # host: localhost @@ -104,9 +102,8 @@ redis: # #pass: example-pass # #prefix: example-prefix # #db: 1 -# # Extra ioredis options -# #extra: -# # username: example-username +# # You can specify more ioredis options... +# #username: example-username # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/packages/backend/check_connect.js b/packages/backend/check_connect.js index bea8abb82cad..da78c8ec7c30 100644 --- a/packages/backend/check_connect.js +++ b/packages/backend/check_connect.js @@ -2,15 +2,7 @@ import Redis from 'ioredis'; import { loadConfig } from './built/config.js'; const config = loadConfig(); -const redis = new Redis({ - port: config.redis.port, - host: config.redis.host, - family: config.redis.family == null ? 0 : config.redis.family, - password: config.redis.pass, - keyPrefix: `${config.redis.prefix}:`, - db: config.redis.db ?? 0, - ...(config.redis.extra ?? {}), -}); +const redis = new Redis(config.redis); redis.on('connect', () => redis.disconnect()); redis.on('error', (e) => { diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 4b1de080d9ea..4caf4c3e965a 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -41,15 +41,7 @@ const $meilisearch: Provider = { const $redis: Provider = { provide: DI.redis, useFactory: (config: Config) => { - return new Redis.Redis({ - port: config.redis.port, - host: config.redis.host, - family: config.redis.family == null ? 0 : config.redis.family, - password: config.redis.pass, - keyPrefix: `${config.redis.prefix}:`, - db: config.redis.db ?? 0, - ...(config.redis.extra ?? {}), - }); + return new Redis.Redis(config.redis); }, inject: [DI.config], }; @@ -57,15 +49,7 @@ const $redis: Provider = { const $redisForPub: Provider = { provide: DI.redisForPub, useFactory: (config: Config) => { - const redis = new Redis.Redis({ - port: config.redisForPubsub.port, - host: config.redisForPubsub.host, - family: config.redisForPubsub.family == null ? 0 : config.redisForPubsub.family, - password: config.redisForPubsub.pass, - keyPrefix: `${config.redisForPubsub.prefix}:`, - db: config.redisForPubsub.db ?? 0, - ...(config.redisForPubsub.extra ?? {}), - }); + const redis = new Redis.Redis(config.redisForPubsub); return redis; }, inject: [DI.config], @@ -74,15 +58,7 @@ const $redisForPub: Provider = { const $redisForSub: Provider = { provide: DI.redisForSub, useFactory: (config: Config) => { - const redis = new Redis.Redis({ - port: config.redisForPubsub.port, - host: config.redisForPubsub.host, - family: config.redisForPubsub.family == null ? 0 : config.redisForPubsub.family, - password: config.redisForPubsub.pass, - keyPrefix: `${config.redisForPubsub.prefix}:`, - db: config.redisForPubsub.db ?? 0, - ...(config.redisForPubsub.extra ?? {}), - }); + const redis = new Redis.Redis(config.redisForPubsub); redis.subscribe(config.host); return redis; }, diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index d8a8ffbb2ada..3d7475fd83bf 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -8,6 +8,15 @@ import { dirname, resolve } from 'node:path'; import * as yaml from 'js-yaml'; import type { RedisOptions } from 'ioredis'; +type RedisOptionsSource = Partial & { + host: string; + port: number; + family?: number; + pass: string; + db?: number; + prefix?: string; +}; + /** * ユーザーが設定する必要のある情報 */ @@ -36,33 +45,9 @@ export type Source = { user: string; pass: string; }[]; - redis: { - host: string; - port: number; - family?: number; - pass: string; - db?: number; - prefix?: string; - extra?: RedisOptions; - }; - redisForPubsub?: { - host: string; - port: number; - family?: number; - pass: string; - db?: number; - prefix?: string; - extra?: RedisOptions; - }; - redisForJobQueue?: { - host: string; - port: number; - family?: number; - pass: string; - db?: number; - prefix?: string; - extra?: RedisOptions; - }; + redis: RedisOptionsSource; + redisForPubsub?: RedisOptionsSource; + redisForJobQueue?: RedisOptionsSource; meilisearch?: { host: string; port: string; @@ -123,8 +108,9 @@ export type Mixin = { mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; - redisForPubsub: NonNullable; - redisForJobQueue: NonNullable; + redis: RedisOptions; + redisForPubsub: RedisOptions; + redisForJobQueue: RedisOptions; }; export type Config = Source & Mixin; @@ -186,9 +172,9 @@ export function loadConfig() { config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator : null; - if (!config.redis.prefix) config.redis.prefix = mixin.host; - if (config.redisForPubsub == null) config.redisForPubsub = config.redis; - if (config.redisForJobQueue == null) config.redisForJobQueue = config.redis; + mixin.redis = convertRedisOptions(config.redis, mixin.host); + mixin.redisForPubsub = config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, mixin.host) : mixin.redis; + mixin.redisForJobQueue = config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, mixin.host) : mixin.redis; return Object.assign(config, mixin); } @@ -200,3 +186,12 @@ function tryCreateUrl(url: string) { throw new Error(`url="${url}" is not a valid URL.`); } } + +function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOptions { + return { + ...options, + family: options.family == null ? 0 : options.family, + keyPrefix: `${options.prefix ?? host}:`, + db: options.db ?? 0, + }; +} diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index 4b34daadf331..18ad542c87dd 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -14,14 +14,7 @@ export const QUEUE = { export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { return { - connection: { - port: config.redisForJobQueue.port, - host: config.redisForJobQueue.host, - family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, - password: config.redisForJobQueue.pass, - db: config.redisForJobQueue.db ?? 0, - ...(config.redisForJobQueue.extra ?? {}), - }, + connection: config.redisForJobQueue, prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, }; } From 3d213f189b2d733010b4b7ea5bde52bdaa363506 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 10:13:00 +0000 Subject: [PATCH 3/7] fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57935f0641f7..9ef1533abc12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,7 +124,7 @@ ### Server - bullをbull-mqにアップグレードし、ジョブキューのパフォーマンスを改善 - ストリーミングのパフォーマンスを改善 -- 設定ファイルでioredisの全てのオプションを指定可能に (redis.extra) +- 設定ファイルでioredisの全てのオプションを指定可能に - Fix: 無効化されたアンテナにアクセスがあった際に再度有効化するように - Fix: お知らせの画像URLを空にできない問題を修正 - Fix: i/notificationsのsinceIdが機能しない問題を修正 From dfd6b4f21369ae18224a930a6e5d136b42bb44db Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 10:18:22 +0000 Subject: [PATCH 4/7] fix? --- packages/backend/src/config.ts | 6 +++--- packages/backend/src/queue/const.ts | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 3d7475fd83bf..2add8f7ac99c 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -108,9 +108,9 @@ export type Mixin = { mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; - redis: RedisOptions; - redisForPubsub: RedisOptions; - redisForJobQueue: RedisOptions; + redis: RedisOptions & RedisOptionsSource; + redisForPubsub: RedisOptions & RedisOptionsSource; + redisForJobQueue: RedisOptions & RedisOptionsSource; }; export type Config = Source & Mixin; diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index 18ad542c87dd..d49951a1c3d3 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -14,7 +14,10 @@ export const QUEUE = { export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { return { - connection: config.redisForJobQueue, + connection: { + ...config.redisForJobQueue, + keyPrefix: undefined + }, prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, }; } From 227f19ff3afcbbd560b831493975206263a1a5a3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 10:26:02 +0000 Subject: [PATCH 5/7] fix --- packages/backend/src/config.ts | 10 ++++++---- packages/backend/src/queue/const.ts | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 2add8f7ac99c..30c21ba68c2f 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -108,9 +108,9 @@ export type Mixin = { mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; - redis: RedisOptions & RedisOptionsSource; - redisForPubsub: RedisOptions & RedisOptionsSource; - redisForJobQueue: RedisOptions & RedisOptionsSource; + redis: RedisOptions; + redisForPubsub: RedisOptions; + redisForJobQueue: RedisOptions; }; export type Config = Source & Mixin; @@ -188,10 +188,12 @@ function tryCreateUrl(url: string) { } function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOptions { - return { + const result = { ...options, family: options.family == null ? 0 : options.family, keyPrefix: `${options.prefix ?? host}:`, db: options.db ?? 0, }; + delete result.prefix; + return result; } diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index d49951a1c3d3..4cf0aa605800 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -18,6 +18,6 @@ export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof t ...config.redisForJobQueue, keyPrefix: undefined }, - prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, + prefix: config.redisForJobQueue.keyPrefix ? `${config.redisForJobQueue.keyPrefix}queue:${queueName}` : `queue:${queueName}`, }; } From 365d9d5e19eae94e5a2bce74882889832327aab8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 10:26:29 +0000 Subject: [PATCH 6/7] Revert "fix" This reverts commit 227f19ff3afcbbd560b831493975206263a1a5a3. --- packages/backend/src/config.ts | 10 ++++------ packages/backend/src/queue/const.ts | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 30c21ba68c2f..2add8f7ac99c 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -108,9 +108,9 @@ export type Mixin = { mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; - redis: RedisOptions; - redisForPubsub: RedisOptions; - redisForJobQueue: RedisOptions; + redis: RedisOptions & RedisOptionsSource; + redisForPubsub: RedisOptions & RedisOptionsSource; + redisForJobQueue: RedisOptions & RedisOptionsSource; }; export type Config = Source & Mixin; @@ -188,12 +188,10 @@ function tryCreateUrl(url: string) { } function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOptions { - const result = { + return { ...options, family: options.family == null ? 0 : options.family, keyPrefix: `${options.prefix ?? host}:`, db: options.db ?? 0, }; - delete result.prefix; - return result; } diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index 4cf0aa605800..d49951a1c3d3 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -18,6 +18,6 @@ export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof t ...config.redisForJobQueue, keyPrefix: undefined }, - prefix: config.redisForJobQueue.keyPrefix ? `${config.redisForJobQueue.keyPrefix}queue:${queueName}` : `queue:${queueName}`, + prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, }; } From a2e05f7efd7415b3d3e8d8c08d7efbc3fd52730a Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 20 Jul 2023 10:27:02 +0000 Subject: [PATCH 7/7] fix --- packages/backend/src/config.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 2add8f7ac99c..bbacfe31a747 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -187,9 +187,10 @@ function tryCreateUrl(url: string) { } } -function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOptions { +function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOptions & RedisOptionsSource { return { ...options, + prefix: options.prefix ?? host, family: options.family == null ? 0 : options.family, keyPrefix: `${options.prefix ?? host}:`, db: options.db ?? 0,