From 5b4ee9a4ee15833e87e56aca23c7bdf014f80fb4 Mon Sep 17 00:00:00 2001 From: mattyatea Date: Sat, 13 Jul 2024 04:52:40 +0900 Subject: [PATCH 1/2] =?UTF-8?q?feat(backend):=20elasticsearch=E3=81=A7?= =?UTF-8?q?=E6=A4=9C=E7=B4=A2=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 皐月ăȘご (Nafu Satsuki) --- packages/backend/package.json | 1 + packages/backend/src/GlobalModule.ts | 29 ++++- packages/backend/src/config.ts | 20 ++++ packages/backend/src/core/SearchService.ts | 132 ++++++++++++++++++++- packages/backend/src/di-symbols.ts | 1 + pnpm-lock.yaml | 49 +++++++- 6 files changed, 226 insertions(+), 6 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 2ba124347506..bb14eb21a847 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -72,6 +72,7 @@ "@bull-board/fastify": "5.18.1", "@bull-board/ui": "5.18.1", "@discordapp/twemoji": "15.0.3", + "@elastic/elasticsearch": "^8.14.0", "@fastify/accepts": "4.3.0", "@fastify/cookie": "9.3.1", "@fastify/cors": "9.0.1", diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 5669c8f07102..d58baede2d6c 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -8,6 +8,7 @@ import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; import { MeiliSearch } from 'meilisearch'; +import { Client as ElasticSearch } from '@elastic/elasticsearch'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; @@ -44,6 +45,30 @@ const $meilisearch: Provider = { inject: [DI.config], }; +const $elasticsearch: Provider = { + provide: DI.elasticsearch, + useFactory: (config: Config) => { + if (config.elasticsearch) { + return new ElasticSearch({ + nodes: { + url: new URL(`${config.elasticsearch.ssl ? 'https' : 'http'}://${config.elasticsearch.host}:${config.elasticsearch.port}`), + ssl: { + rejectUnauthorized: config.elasticsearch.rejectUnauthorized, + }, + }, + auth: (config.elasticsearch.user && config.elasticsearch.pass) ? { + username: config.elasticsearch.user, + password: config.elasticsearch.pass, + } : undefined, + pingTimeout: 30000, + }); + } else { + return null; + } + }, + inject: [DI.config], +}; + const $redis: Provider = { provide: DI.redis, useFactory: (config: Config) => { @@ -160,8 +185,8 @@ const $redisForTimelines: Provider = { @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], - exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], + providers: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], + exports: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index caf14e5aa05f..918fa60216f6 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -66,6 +66,16 @@ type Source = { scope?: 'local' | 'global' | string[]; }; + elasticsearch?: { + host: string; + port: string; + user: string; + pass: string; + ssl?: boolean; + rejectUnauthorized?: boolean; + index: string; + }; + skebStatus?: { method: string; endpoint: string; @@ -149,6 +159,15 @@ export type Config = { index: string; scope?: 'local' | 'global' | string[]; } | undefined; + elasticsearch: { + host: string; + port: string; + user: string; + pass: string; + ssl?: boolean; + rejectUnauthorized?: boolean; + index: string; + } | undefined; skebStatus: { method: string; endpoint: string; @@ -272,6 +291,7 @@ export function loadConfig(): Config { dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, meilisearch: config.meilisearch, + elasticsearch: config.elasticsearch, redis, redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, redisForSystemQueue: config.redisForSystemQueue ? convertRedisOptions(config.redisForSystemQueue, host) : redisForJobQueue, diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 2e432977582e..f637c5ac95dc 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -17,7 +17,7 @@ import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; import type { Index, MeiliSearch } from 'meilisearch'; - +import type { Client as ElasticSearch } from '@elastic/elasticsearch'; type K = string; type V = string | number | boolean; type Q = @@ -65,7 +65,7 @@ function compileQuery(q: Q): string { export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; private meilisearchNoteIndex: Index | null = null; - + private elasticsearchNoteIndex: string | null = null; constructor( @Inject(DI.config) private config: Config, @@ -73,6 +73,9 @@ export class SearchService { @Inject(DI.meilisearch) private meilisearch: MeiliSearch | null, + @Inject(DI.elasticsearch) + private elasticsearch: ElasticSearch | null, + @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -104,8 +107,53 @@ export class SearchService { maxTotalHits: 10000, }, });*/ + } else if (this.elasticsearch) { + this.elasticsearchNoteIndex = `${config.elasticsearch!.index}---notes`; + this.elasticsearch.indices.exists({ + index: this.elasticsearchNoteIndex, + }).then((indexExists) => { + if (!indexExists) { + this.elasticsearch?.indices.create( + { + index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}`, + mappings: { + properties: { + text: { type: 'text' }, + cw: { type: 'text' }, + createdAt: { type: 'long' }, + userId: { type: 'keyword' }, + userHost: { type: 'keyword' }, + channelId: { type: 'keyword' }, + tags: { type: 'keyword' }, + }, + }, + settings: { + index: { + analysis: { + tokenizer: { + kuromoji: { + type: 'kuromoji_tokenizer', + mode: 'search', + }, + }, + analyzer: { + kuromoji_analyzer: { + type: 'custom', + tokenizer: 'kuromoji', + }, + }, + }, + }, + }, + }, + ).catch((error) => { + console.error(error); + }); + } + }).catch((error) => { + console.error(error); + }); } - if (config.meilisearch?.scope) { this.meilisearchIndexScope = config.meilisearch.scope; } @@ -144,6 +192,23 @@ export class SearchService { }], { primaryKey: 'id', }); + } else if (this.elasticsearch) { + const body = { + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + }; + await this.elasticsearch.index({ + index: this.elasticsearchNoteIndex + `-${new Date().toISOString().slice(0, 7).replace(/-/g, '')}` as string, + id: note.id, + body: body, + }).catch((error) => { + console.error(error); + }); } } @@ -204,6 +269,67 @@ export class SearchService { if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; return true; }); + return notes.sort((a, b) => a.id > b.id ? -1 : 1); + } else if (this.elasticsearch) { + const esFilter: any = { + bool: { + must: [], + }, + }; + + if (pagination.untilId) esFilter.bool.must.push({ range: { createdAt: { lt: this.idService.parse(pagination.untilId).date.getTime() } } }); + if (pagination.sinceId) esFilter.bool.must.push({ range: { createdAt: { gt: this.idService.parse(pagination.sinceId).date.getTime() } } }); + if (opts.userId) esFilter.bool.must.push({ term: { userId: opts.userId } }); + if (opts.channelId) esFilter.bool.must.push({ term: { channelId: opts.channelId } }); + if (opts.host) { + if (opts.host === '.') { + esFilter.bool.must.push({ bool: { must_not: [{ exists: { field: 'userHost' } }] } }); + } else { + esFilter.bool.must.push({ term: { userHost: opts.host } }); + } + } + + if (q !== '') { + esFilter.bool.must.push({ + bool: { + should: [ + { wildcard: { 'text': { value: q } } }, + { simple_query_string: { fields: ['text'], 'query': q, default_operator: 'and' } }, + { wildcard: { 'cw': { value: q } } }, + { simple_query_string: { fields: ['cw'], 'query': q, default_operator: 'and' } }, + ], + minimum_should_match: 1, + }, + }); + } + + const res = await (this.elasticsearch.search)({ + index: this.elasticsearchNoteIndex + '*' as string, + body: { + query: esFilter, + sort: [{ createdAt: { order: 'desc' } }], + }, + _source: ['id', 'createdAt'], + size: pagination.limit, + }); + + const noteIds = res.hits.hits.map((hit: any) => hit._id); + if (noteIds.length === 0) return []; + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) : [new Set(), new Set()]; + const notes = (await this.notesRepository.findBy({ + id: In(noteIds), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } else { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 0ba0b86c9d2b..fb570d0b4424 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -7,6 +7,7 @@ export const DI = { config: Symbol('config'), db: Symbol('db'), meilisearch: Symbol('meilisearch'), + elasticsearch: Symbol('elasticsearch'), redis: Symbol('redis'), redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4eccb599afa7..e3445d236813 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: '@discordapp/twemoji': specifier: 15.0.3 version: 15.0.3 + '@elastic/elasticsearch': + specifier: ^8.14.0 + version: 8.14.0 '@fastify/accepts': specifier: 4.3.0 version: 4.3.0 @@ -2199,6 +2202,14 @@ packages: resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} engines: {node: '>=10.0.0'} + '@elastic/elasticsearch@8.14.0': + resolution: {integrity: sha512-MGrgCI4y+Ozssf5Q2IkVJlqt5bUMnKIICG2qxeOfrJNrVugMCBCAQypyesmSSocAtNm8IX3LxfJ3jQlFHmKe2w==} + engines: {node: '>=18'} + + '@elastic/transport@8.7.0': + resolution: {integrity: sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A==} + engines: {node: '>=18'} + '@emnapi/runtime@1.2.0': resolution: {integrity: sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==} @@ -2989,6 +3000,10 @@ packages: '@open-draft/until@2.1.0': resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@peculiar/asn1-android@2.3.10': resolution: {integrity: sha512-z9Rx9cFJv7UUablZISe7uksNbFJCq13hO0yEAOoIpAymALTLlvUOSLnGiQS7okPaM5dP42oTLhezH6XDXRXjGw==} @@ -10410,6 +10425,10 @@ packages: resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==} engines: {node: '>=14.0'} + undici@6.19.2: + resolution: {integrity: sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA==} + engines: {node: '>=18.17'} + unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} engines: {node: '>=4'} @@ -10689,6 +10708,9 @@ packages: vue-component-type-helpers@2.0.19: resolution: {integrity: sha512-cN3f1aTxxKo4lzNeQAkVopswuImUrb5Iurll9Gaw5cqpnbTAxtEMM1mgi6ou4X79OCyqYv1U1mzBHJkzmiK82w==} + vue-component-type-helpers@2.0.26: + resolution: {integrity: sha512-sO9qQ8oC520SW6kqlls0iqDak53gsTVSrYylajgjmkt1c0vcgjsGSy1KzlDrbEx8pm02IEYhlUkU5hCYf8rwtg==} + vue-demi@0.14.7: resolution: {integrity: sha512-EOG8KXDQNwkJILkx/gPcoL/7vH+hORoBaKgGe+6W7VFMvCYJfmF2dGbvgDroVnI8LU7/kTu8mbjRZGBU1z9NTA==} engines: {node: '>=12'} @@ -12469,6 +12491,25 @@ snapshots: '@discoveryjs/json-ext@0.5.7': {} + '@elastic/elasticsearch@8.14.0': + dependencies: + '@elastic/transport': 8.7.0 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + + '@elastic/transport@8.7.0': + dependencies: + '@opentelemetry/api': 1.9.0 + debug: 4.3.4(supports-color@8.1.1) + hpagent: 1.2.0 + ms: 2.1.3 + secure-json-parse: 2.7.0 + tslib: 2.6.2 + undici: 6.19.2 + transitivePeerDependencies: + - supports-color + '@emnapi/runtime@1.2.0': dependencies: tslib: 2.6.2 @@ -13346,6 +13387,8 @@ snapshots: '@open-draft/until@2.1.0': {} + '@opentelemetry/api@1.9.0': {} + '@peculiar/asn1-android@2.3.10': dependencies: '@peculiar/asn1-schema': 2.3.8 @@ -14738,7 +14781,7 @@ snapshots: ts-dedent: 2.2.0 type-fest: 2.19.0 vue: 3.4.15(typescript@5.4.5) - vue-component-type-helpers: 2.0.19 + vue-component-type-helpers: 2.0.26 transitivePeerDependencies: - encoding - prettier @@ -22388,6 +22431,8 @@ snapshots: dependencies: '@fastify/busboy': 2.1.1 + undici@6.19.2: {} + unicode-canonical-property-names-ecmascript@2.0.0: {} unicode-match-property-ecmascript@2.0.0: @@ -22667,6 +22712,8 @@ snapshots: vue-component-type-helpers@2.0.19: {} + vue-component-type-helpers@2.0.26: {} + vue-demi@0.14.7(vue@3.4.15(typescript@5.4.5)): dependencies: vue: 3.4.15(typescript@5.4.5) From adc01f0ffd6ac5760d57568e1cc1ba1c9c4c6534 Mon Sep 17 00:00:00 2001 From: mattyatea Date: Sat, 13 Jul 2024 07:44:58 +0900 Subject: [PATCH 2/2] =?UTF-8?q?SearchService.ts=E3=81=AE=E6=94=B9=E8=A1=8C?= =?UTF-8?q?=E3=81=AA=E3=81=A9=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/SearchService.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index f637c5ac95dc..e20ed06a04c1 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -8,6 +8,7 @@ import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { MiNote } from '@/models/Note.js'; import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; @@ -16,8 +17,10 @@ import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import type Logger from '@/logger.js'; import type { Index, MeiliSearch } from 'meilisearch'; import type { Client as ElasticSearch } from '@elastic/elasticsearch'; + type K = string; type V = string | number | boolean; type Q = @@ -66,6 +69,8 @@ export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; private meilisearchNoteIndex: Index | null = null; private elasticsearchNoteIndex: string | null = null; + private logger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -82,9 +87,15 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { + this.logger = this.loggerService.getLogger('note:search'); + if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`); + if (config.meilisearch?.scope) { + this.meilisearchIndexScope = config.meilisearch.scope; + } /*this.meilisearchNoteIndex.updateSettings({ searchableAttributes: [ 'text', @@ -147,16 +158,13 @@ export class SearchService { }, }, ).catch((error) => { - console.error(error); + this.logger.error(error); }); } }).catch((error) => { - console.error(error); + this.logger.error('Error while checking if index exists', error); }); } - if (config.meilisearch?.scope) { - this.meilisearchIndexScope = config.meilisearch.scope; - } } @bindThis