Skip to content

Commit

Permalink
feat: upgrade cache-manager to v6
Browse files Browse the repository at this point in the history
  • Loading branch information
murongg committed Jan 14, 2025
1 parent e7bed58 commit 41308f1
Show file tree
Hide file tree
Showing 10 changed files with 4,434 additions and 5,078 deletions.
2 changes: 1 addition & 1 deletion packages/reku/src/event/crosschecker/cache/manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ export class CrossCheckerCacheManager extends SimpleStoreManager {
await this.set(`${this.storeKeyPrefix}checkpoint`, checkpoint, this.ttl)
}

async getCheckpoint(): Promise<number | undefined> {
async getCheckpoint(): Promise<number | undefined | null> {
return await this.get(`${this.storeKeyPrefix}checkpoint`)
}
}
5 changes: 4 additions & 1 deletion packages/utils/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,14 @@
"ethers": ">=6.13.0"
},
"dependencies": {
"@keyv/redis": "^4.2.0",
"@murongg/utils": "^0.1.28",
"cache-manager": "^5.7.6",
"cache-manager": "^6.3.2",
"cache-manager-ioredis-yet": "^2.1.1",
"cacheable": "^1.8.7",
"debug": "^4.3.7",
"ioredis": "^5.4.1",
"keyv": "^5.2.3",
"lodash.clonedeep": "^4.5.0",
"lru-cache": "^11.0.1",
"picocolors": "^1.0.1",
Expand Down
5 changes: 3 additions & 2 deletions packages/utils/src/store/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export type { Store, StoreConfig, Stores } from 'cache-manager'
export type { CacheManagerStore } from 'cache-manager'
export type { Store } from './types'
export { redisStore } from './redis'
export * from './manager'
export { memoryStore } from './memory'
export type { MemoryStore, MemoryCache, MemoryConfig } from './memory'
export type { MemoryConfig } from './memory'
53 changes: 37 additions & 16 deletions packages/utils/src/store/manager.ts
Original file line number Diff line number Diff line change
@@ -1,33 +1,42 @@
import type { Cache, Milliseconds, Store } from 'cache-manager'
import { createCache, memoryStore } from 'cache-manager'
import type { Cache, CreateCacheOptions } from 'cache-manager'
import { createCache } from 'cache-manager'
import type { RedisStore } from 'cache-manager-ioredis-yet'
import { Keyv } from 'keyv'
import { KeyvCacheableMemory } from 'cacheable'

export class SimpleStoreManager {
private cache: Cache
private store: Keyv

constructor(
store?: Store,
options?: (Omit<CreateCacheOptions, 'stores'>) & { store: Keyv },
) {
if (store) {
this.cache = createCache(store)
if (options) {
this.cache = createCache({
...options,
stores: [
options.store,
],
})
this.store = options.store
}
else {
this.cache = createCache(memoryStore({
max: 100,
ttl: 10 * 1000 /* milliseconds */,
}))
const store = new KeyvCacheableMemory({ ttl: 10 * 1000, lruSize: 100 })
const keyv = new Keyv({ store })
this.store = keyv
this.cache = createCache({ stores: [keyv] })
}
}

get redisStore(): RedisStore {
return <RedisStore> this.cache.store
return <RedisStore> this.store.store
}

async get<T>(key: string) {
return await this.cache.get<T>(key)
}

async set(key: string, value: unknown, ttl?: Milliseconds) {
async set(key: string, value: unknown, ttl?: number) {
await this.cache.set(key, value, ttl)
}

Expand All @@ -36,16 +45,28 @@ export class SimpleStoreManager {
}

async keys(pattern?: string): Promise<string[]> {
return await this.cache.store.keys(pattern)
const keys = []
for await (const [key] of this.store.store.entries()) {
if (pattern) {
const regex = new RegExp(`(?<!.)${pattern}`)
if (regex.test(key))
keys.push(key)
}
else {
keys.push(key)
}
}
return keys
}

async has(key: string): Promise<boolean> {
return !!await this.cache.store.get(key)
return !!await this.cache.get(key)
}

async getAll<T>(): Promise<T[]> {
const keys = await this.keys()
const values = await Promise.all(keys.map(async key => await this.get<T>(key)))
return values.filter(value => value !== undefined)
const values = []
for await (const [, value] of this.store.store.entries())
values.push(value)
return values
}
}
118 changes: 9 additions & 109 deletions packages/utils/src/store/memory.ts
Original file line number Diff line number Diff line change
@@ -1,113 +1,13 @@
import { LRUCache } from 'lru-cache'
import cloneDeep from 'lodash.clonedeep'
import type { Cache, Config, Store } from 'cache-manager'
import { Keyv } from 'keyv'
import type { CacheableMemoryOptions } from 'cacheable'
import { KeyvCacheableMemory } from 'cacheable'

function clone<T>(object: T): T {
if (typeof object === 'object' && object !== null)
return cloneDeep(object)

return object
export type MemoryConfig = CacheableMemoryOptions & {
namespace?: string
}

type LRU = LRUCache<string, any>

type Pre = LRUCache.OptionsTTLLimit<string, any, unknown>
type Options = Omit<Pre, 'ttlAutopurge'> & Partial<Pick<Pre, 'ttlAutopurge'>>
export type MemoryConfig = {
max?: number
sizeCalculation?: (value: unknown, key: string) => number
shouldCloneBeforeSet?: boolean
} & Options &
Config

export type MemoryStore = Store & {
dump: LRU['dump']
load: LRU['load']
calculatedSize: LRU['calculatedSize']
get size(): number
}
export type MemoryCache = Cache<MemoryStore>

/**
* Wrapper for lru-cache.
*/
export function memoryStore(arguments_?: MemoryConfig): MemoryStore {
const shouldCloneBeforeSet = arguments_?.shouldCloneBeforeSet !== false // Clone by default
const isCacheable = arguments_?.isCacheable ?? (value => value !== undefined)

const lruOptions = {
ttlAutopurge: true,
...arguments_,
max: arguments_?.max ?? 500,
ttl: arguments_?.ttl === undefined ? 0 : arguments_.ttl,
}

const lruCache = new LRUCache(lruOptions)

return {
async del(key) {
lruCache.delete(key)
},
get: async <T>(key: string) => lruCache.get(key) as T,
keys: async (pattern?: string) => {
const keys = [...lruCache.keys()]
if (!pattern)
return keys

const regex = new RegExp(`(?<!.)${pattern}`)
return keys.filter(key => regex.test(key))
},

mget: async (...arguments_) => arguments_.map(x => lruCache.get(x)),
async mset(arguments_, ttl?) {
const opt = { ttl: ttl ?? lruOptions.ttl } as const
for (const [key, value] of arguments_) {
if (!isCacheable(value))
throw new Error(`no cacheable value ${JSON.stringify(value)}`)

if (shouldCloneBeforeSet)
lruCache.set(key, clone(value), opt)
else
lruCache.set(key, value, opt)
}
},
async mdel(...arguments_) {
for (const key of arguments_)
lruCache.delete(key)
},
async reset() {
lruCache.clear()
},
ttl: async key => lruCache.getRemainingTTL(key),
async set(key, value, opt) {
if (!isCacheable(value))
throw new Error(`no cacheable value ${JSON.stringify(value)}`)

if (shouldCloneBeforeSet)
value = clone(value)

const ttl = opt ?? lruOptions.ttl

lruCache.set(key, value, { ttl })
},
get calculatedSize() {
return lruCache.calculatedSize
},
/**
* This method is not available in the caching modules.
*/
get size() {
return lruCache.size
},
/**
* This method is not available in the caching modules.
*/
dump: () => lruCache.dump(),
/**
* This method is not available in the caching modules.
*/
load(...arguments_: Parameters<LRU['load']>) {
lruCache.load(...arguments_)
},
}
export function memoryStore(options?: MemoryConfig) {
const store = new KeyvCacheableMemory(options)
const keyv = new Keyv({ store }, { namespace: '' })
return keyv
}
47 changes: 28 additions & 19 deletions packages/utils/src/store/redis.ts
Original file line number Diff line number Diff line change
@@ -1,24 +1,33 @@
import type { RedisOptions } from 'ioredis'
import { Redis } from 'ioredis'
import type { Config } from 'cache-manager'
import { RedisClusterConfig, redisInsStore } from 'cache-manager-ioredis-yet'
import type { KeyvRedisOptions, RedisClientConnectionType, RedisClientOptions, RedisClusterOptions } from '@keyv/redis'
import KeyvRedis, { Keyv } from '@keyv/redis'

export interface RedisConnect {
host?: string
port?: number
username?: string
password?: string
db?: number
}

export function redisStore(
options?: (RedisOptions | { clusterConfig: RedisClusterConfig }) & Config,
connect?: string | RedisClientOptions | RedisClusterOptions | RedisClientConnectionType | RedisConnect,
options?: KeyvRedisOptions,
) {
options ||= {}
const redisCache
= 'clusterConfig' in options
? new Redis.Cluster(
options.clusterConfig.nodes,
options.clusterConfig.options,
)
: new Redis(options)

return redisInsStore(redisCache, options)
}
if (typeof connect === 'object' && connect) {
if (Reflect.get(connect, 'host')) {
const c = connect as RedisConnect
let connectStr = 'redis://'
if (c.username)
connectStr += `${c.username}:${c.password}@`
connectStr += `${c.host}:${c.port}`
if (c.db)
connectStr += `/${c.db}`
connect = connectStr
}
}

export {
RedisClusterConfig,
redisInsStore,
const keyv = new Keyv(new KeyvRedis(connect, options), {
namespace: '',
})
return keyv
}
Loading

0 comments on commit 41308f1

Please sign in to comment.