[performance] add caching of status fave, boost of, in reply to ID lists

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	codeberg.org/gruf/go-bytesize v1.0.2
 	codeberg.org/gruf/go-byteutil v1.1.2
-	codeberg.org/gruf/go-cache/v3 v3.5.3
+	codeberg.org/gruf/go-cache/v3 v3.5.5
 	codeberg.org/gruf/go-debug v1.3.0
 	codeberg.org/gruf/go-errors/v2 v2.2.0
 	codeberg.org/gruf/go-fastcopy v1.1.2

go.sum

@@ -48,8 +48,8 @@ codeberg.org/gruf/go-bytesize v1.0.2/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacp
 codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
 codeberg.org/gruf/go-byteutil v1.1.2 h1:TQLZtTxTNca9xEfDIndmo7nBYxeS94nrv/9DS3Nk5Tw=
 codeberg.org/gruf/go-byteutil v1.1.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
-codeberg.org/gruf/go-cache/v3 v3.5.3 h1:CRO2syVQxT/JbqDnUxzjeJkLInihEmTlJOkrOgkTmqI=
-codeberg.org/gruf/go-cache/v3 v3.5.3/go.mod h1:NbsGQUgEdNFd631WSasvCHIVAaY9ovuiSeoBwtsIeDc=
+codeberg.org/gruf/go-cache/v3 v3.5.5 h1:Ce7odyvr8oF6h49LSjPL7AZs2QGyKMN9BPkgKcfR0BA=
+codeberg.org/gruf/go-cache/v3 v3.5.5/go.mod h1:NbsGQUgEdNFd631WSasvCHIVAaY9ovuiSeoBwtsIeDc=
 codeberg.org/gruf/go-debug v1.3.0 h1:PIRxQiWUFKtGOGZFdZ3Y0pqyfI0Xr87j224IYe2snZs=
 codeberg.org/gruf/go-debug v1.3.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg=
 codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4=

internal/cache/cache.go

@@ -196,6 +196,21 @@ func (c *Caches) setuphooks() {
 			// c.GTS.Media().Invalidate("StatusID") will not work.
 			c.GTS.Media().Invalidate("ID", id)
 		}
+
+		if status.BoostOfID != "" {
+			// Invalidate boost ID list of the original status.
+			c.GTS.BoostOfIDs().Invalidate(status.BoostOfID)
+		}
+
+		if status.InReplyToID != "" {
+			// Invalidate in reply to ID list of original status.
+			c.GTS.InReplyToIDs().Invalidate(status.InReplyToID)
+		}
+	})
+
+	c.GTS.StatusFave().SetInvalidateCallback(func(fave *gtsmodel.StatusFave) {
+		// Invalidate status fave ID list for this status.
+		c.GTS.StatusFaveIDs().Invalidate(fave.StatusID)
 	})
 
 	c.GTS.User().SetInvalidateCallback(func(user *gtsmodel.User) {

internal/cache/gts.go

@@ -34,6 +34,7 @@ type GTSCaches struct {
 	accountNote      *result.Cache[*gtsmodel.AccountNote]
 	block            *result.Cache[*gtsmodel.Block]
 	blockIDs         *SliceCache[string]
+	boostOfIDs       *SliceCache[string]
 	domainBlock      *domain.BlockCache
 	emoji            *result.Cache[*gtsmodel.Emoji]
 	emojiCategory    *result.Cache[*gtsmodel.EmojiCategory]
@@ -42,6 +43,7 @@ type GTSCaches struct {
 	followRequest    *result.Cache[*gtsmodel.FollowRequest]
 	followRequestIDs *SliceCache[string]
 	instance         *result.Cache[*gtsmodel.Instance]
+	inReplyToIDs     *SliceCache[string]
 	list             *result.Cache[*gtsmodel.List]
 	listEntry        *result.Cache[*gtsmodel.ListEntry]
 	marker           *result.Cache[*gtsmodel.Marker]
@@ -51,6 +53,7 @@ type GTSCaches struct {
 	report           *result.Cache[*gtsmodel.Report]
 	status           *result.Cache[*gtsmodel.Status]
 	statusFave       *result.Cache[*gtsmodel.StatusFave]
+	statusFaveIDs    *SliceCache[string]
 	tag              *result.Cache[*gtsmodel.Tag]
 	tombstone        *result.Cache[*gtsmodel.Tombstone]
 	user             *result.Cache[*gtsmodel.User]
@@ -66,13 +69,15 @@ func (c *GTSCaches) Init() {
 	c.initAccountNote()
 	c.initBlock()
 	c.initBlockIDs()
+	c.initBoostOfIDs()
 	c.initDomainBlock()
 	c.initEmoji()
 	c.initEmojiCategory()
 	c.initFollow()
 	c.initFollowIDs()
 	c.initFollowRequest()
 	c.initFollowRequestIDs()
+	c.initInReplyToIDs()
 	c.initInstance()
 	c.initList()
 	c.initListEntry()
@@ -84,6 +89,7 @@ func (c *GTSCaches) Init() {
 	c.initStatus()
 	c.initStatusFave()
 	c.initTag()
+	c.initStatusFaveIDs()
 	c.initTombstone()
 	c.initUser()
 	c.initWebfinger()
@@ -121,6 +127,11 @@ func (c *GTSCaches) BlockIDs() *SliceCache[string] {
 	return c.blockIDs
 }
 
+// BoostOfIDs provides access to the boost of IDs list database cache.
+func (c *GTSCaches) BoostOfIDs() *SliceCache[string] {
+	return c.boostOfIDs
+}
+
 // DomainBlock provides access to the domain block database cache.
 func (c *GTSCaches) DomainBlock() *domain.BlockCache {
 	return c.domainBlock
@@ -169,6 +180,11 @@ func (c *GTSCaches) Instance() *result.Cache[*gtsmodel.Instance] {
 	return c.instance
 }
 
+// InReplyToIDs provides access to the status in reply to IDs list database cache.
+func (c *GTSCaches) InReplyToIDs() *SliceCache[string] {
+	return c.inReplyToIDs
+}
+
 // List provides access to the gtsmodel List database cache.
 func (c *GTSCaches) List() *result.Cache[*gtsmodel.List] {
 	return c.list
@@ -219,6 +235,11 @@ func (c *GTSCaches) Tag() *result.Cache[*gtsmodel.Tag] {
 	return c.tag
 }
 
+// StatusFaveIDs provides access to the status fave IDs list database cache.
+func (c *GTSCaches) StatusFaveIDs() *SliceCache[string] {
+	return c.statusFaveIDs
+}
+
 // Tombstone provides access to the gtsmodel Tombstone database cache.
 func (c *GTSCaches) Tombstone() *result.Cache[*gtsmodel.Tombstone] {
 	return c.tombstone
@@ -247,7 +268,7 @@ func (c *GTSCaches) initAccount() {
 		{Name: "ID"},
 		{Name: "URI"},
 		{Name: "URL"},
-		{Name: "Username.Domain"},
+		{Name: "Username.Domain", AllowZero: true /* domain can be zero i.e. "" */},
 		{Name: "PublicKeyURI"},
 		{Name: "InboxURI"},
 		{Name: "OutboxURI"},
@@ -320,6 +341,20 @@ func (c *GTSCaches) initBlockIDs() {
 	)}
 }
 
+func (c *GTSCaches) initBoostOfIDs() {
+	// Calculate maximum cache size.
+	cap := calculateSliceCacheMax(
+		config.GetCacheBoostOfIDsMemRatio(),
+	)
+
+	log.Infof(nil, "BoostofIDs cache size = %d", cap)
+
+	c.boostOfIDs = &SliceCache[string]{Cache: simple.New[string, []string](
+		0,
+		cap,
+	)}
+}
+
 func (c *GTSCaches) initDomainBlock() {
 	c.domainBlock = new(domain.BlockCache)
 }
@@ -336,7 +371,7 @@ func (c *GTSCaches) initEmoji() {
 	c.emoji = result.New([]result.Lookup{
 		{Name: "ID"},
 		{Name: "URI"},
-		{Name: "Shortcode.Domain"},
+		{Name: "Shortcode.Domain", AllowZero: true /* domain can be zero i.e. "" */},
 		{Name: "ImageStaticURL"},
 		{Name: "CategoryID", Multi: true},
 	}, func(e1 *gtsmodel.Emoji) *gtsmodel.Emoji {
@@ -445,6 +480,20 @@ func (c *GTSCaches) initFollowRequestIDs() {
 	)}
 }
 
+func (c *GTSCaches) initInReplyToIDs() {
+	// Calculate maximum cache size.
+	cap := calculateSliceCacheMax(
+		config.GetCacheInReplyToIDsMemRatio(),
+	)
+
+	log.Infof(nil, "InReplyTo IDs cache size = %d", cap)
+
+	c.inReplyToIDs = &SliceCache[string]{Cache: simple.New[string, []string](
+		0,
+		cap,
+	)}
+}
+
 func (c *GTSCaches) initInstance() {
 	// Calculate maximum cache size.
 	cap := calculateResultCacheMax(
@@ -622,6 +671,7 @@ func (c *GTSCaches) initStatus() {
 		{Name: "ID"},
 		{Name: "URI"},
 		{Name: "URL"},
+		{Name: "BoostOfID.AccountID"},
 	}, func(s1 *gtsmodel.Status) *gtsmodel.Status {
 		s2 := new(gtsmodel.Status)
 		*s2 = *s1
@@ -643,6 +693,7 @@ func (c *GTSCaches) initStatusFave() {
 	c.statusFave = result.New([]result.Lookup{
 		{Name: "ID"},
 		{Name: "AccountID.StatusID"},
+		{Name: "StatusID", Multi: true},
 	}, func(f1 *gtsmodel.StatusFave) *gtsmodel.StatusFave {
 		f2 := new(gtsmodel.StatusFave)
 		*f2 = *f1
@@ -652,6 +703,20 @@ func (c *GTSCaches) initStatusFave() {
 	c.statusFave.IgnoreErrors(ignoreErrors)
 }
 
+func (c *GTSCaches) initStatusFaveIDs() {
+	// Calculate maximum cache size.
+	cap := calculateSliceCacheMax(
+		config.GetCacheStatusFaveIDsMemRatio(),
+	)
+
+	log.Infof(nil, "StatusFave IDs cache size = %d", cap)
+
+	c.statusFaveIDs = &SliceCache[string]{Cache: simple.New[string, []string](
+		0,
+		cap,
+	)}
+}
+
 func (c *GTSCaches) initTag() {
 	// Calculate maximum cache size.
 	cap := calculateResultCacheMax(

internal/config/config.go

@@ -180,12 +180,14 @@ type CacheConfiguration struct {
 	AccountNoteMemRatio      float64       `name:"account-note-mem-ratio"`
 	BlockMemRatio            float64       `name:"block-mem-ratio"`
 	BlockIDsMemRatio         float64       `name:"block-mem-ratio"`
+	BoostOfIDsMemRatio       float64       `name:"boost-of-ids-mem-ratio"`
 	EmojiMemRatio            float64       `name:"emoji-mem-ratio"`
 	EmojiCategoryMemRatio    float64       `name:"emoji-category-mem-ratio"`
 	FollowMemRatio           float64       `name:"follow-mem-ratio"`
 	FollowIDsMemRatio        float64       `name:"follow-ids-mem-ratio"`
 	FollowRequestMemRatio    float64       `name:"follow-request-mem-ratio"`
 	FollowRequestIDsMemRatio float64       `name:"follow-request-ids-mem-ratio"`
+	InReplyToIDsMemRatio     float64       `name:"in-reply-to-ids-mem-ratio"`
 	InstanceMemRatio         float64       `name:"instance-mem-ratio"`
 	ListMemRatio             float64       `name:"list-mem-ratio"`
 	ListEntryMemRatio        float64       `name:"list-entry-mem-ratio"`
@@ -196,6 +198,7 @@ type CacheConfiguration struct {
 	ReportMemRatio           float64       `name:"report-mem-ratio"`
 	StatusMemRatio           float64       `name:"status-mem-ratio"`
 	StatusFaveMemRatio       float64       `name:"status-fave-mem-ratio"`
+	StatusFaveIDsMemRatio    float64       `name:"status-fave-ids-mem-ratio"`
 	TagMemRatio              float64       `name:"tag-mem-ratio"`
 	TombstoneMemRatio        float64       `name:"tombstone-mem-ratio"`
 	UserMemRatio             float64       `name:"user-mem-ratio"`

internal/config/defaults.go

@@ -149,12 +149,14 @@ var Defaults = Configuration{
 		AccountNoteMemRatio:      0.1,
 		BlockMemRatio:            3,
 		BlockIDsMemRatio:         3,
+		BoostOfIDsMemRatio:       3,
 		EmojiMemRatio:            3,
 		EmojiCategoryMemRatio:    0.1,
 		FollowMemRatio:           4,
 		FollowIDsMemRatio:        4,
 		FollowRequestMemRatio:    2,
 		FollowRequestIDsMemRatio: 2,
+		InReplyToIDsMemRatio:     3,
 		InstanceMemRatio:         1,
 		ListMemRatio:             3,
 		ListEntryMemRatio:        3,
@@ -165,6 +167,7 @@ var Defaults = Configuration{
 		ReportMemRatio:           1,
 		StatusMemRatio:           18,
 		StatusFaveMemRatio:       5,
+		StatusFaveIDsMemRatio:    3,
 		TagMemRatio:              3,
 		TombstoneMemRatio:        2,
 		UserMemRatio:             0.1,

internal/config/helpers.gen.go

@@ -2549,6 +2549,31 @@ func GetCacheBlockIDsMemRatio() float64 { return global.GetCacheBlockIDsMemRatio
 // SetCacheBlockIDsMemRatio safely sets the value for global configuration 'Cache.BlockIDsMemRatio' field
 func SetCacheBlockIDsMemRatio(v float64) { global.SetCacheBlockIDsMemRatio(v) }
 
+// GetCacheBoostOfIDsMemRatio safely fetches the Configuration value for state's 'Cache.BoostOfIDsMemRatio' field
+func (st *ConfigState) GetCacheBoostOfIDsMemRatio() (v float64) {
+	st.mutex.RLock()
+	v = st.config.Cache.BoostOfIDsMemRatio
+	st.mutex.RUnlock()
+	return
+}
+
+// SetCacheBoostOfIDsMemRatio safely sets the Configuration value for state's 'Cache.BoostOfIDsMemRatio' field
+func (st *ConfigState) SetCacheBoostOfIDsMemRatio(v float64) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.BoostOfIDsMemRatio = v
+	st.reloadToViper()
+}
+
+// CacheBoostOfIDsMemRatioFlag returns the flag name for the 'Cache.BoostOfIDsMemRatio' field
+func CacheBoostOfIDsMemRatioFlag() string { return "cache-boost-of-ids-mem-ratio" }
+
+// GetCacheBoostOfIDsMemRatio safely fetches the value for global configuration 'Cache.BoostOfIDsMemRatio' field
+func GetCacheBoostOfIDsMemRatio() float64 { return global.GetCacheBoostOfIDsMemRatio() }
+
+// SetCacheBoostOfIDsMemRatio safely sets the value for global configuration 'Cache.BoostOfIDsMemRatio' field
+func SetCacheBoostOfIDsMemRatio(v float64) { global.SetCacheBoostOfIDsMemRatio(v) }
+
 // GetCacheEmojiMemRatio safely fetches the Configuration value for state's 'Cache.EmojiMemRatio' field
 func (st *ConfigState) GetCacheEmojiMemRatio() (v float64) {
 	st.mutex.RLock()
@@ -2699,6 +2724,31 @@ func GetCacheFollowRequestIDsMemRatio() float64 { return global.GetCacheFollowRe
 // SetCacheFollowRequestIDsMemRatio safely sets the value for global configuration 'Cache.FollowRequestIDsMemRatio' field
 func SetCacheFollowRequestIDsMemRatio(v float64) { global.SetCacheFollowRequestIDsMemRatio(v) }
 
+// GetCacheInReplyToIDsMemRatio safely fetches the Configuration value for state's 'Cache.InReplyToIDsMemRatio' field
+func (st *ConfigState) GetCacheInReplyToIDsMemRatio() (v float64) {
+	st.mutex.RLock()
+	v = st.config.Cache.InReplyToIDsMemRatio
+	st.mutex.RUnlock()
+	return
+}
+
+// SetCacheInReplyToIDsMemRatio safely sets the Configuration value for state's 'Cache.InReplyToIDsMemRatio' field
+func (st *ConfigState) SetCacheInReplyToIDsMemRatio(v float64) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.InReplyToIDsMemRatio = v
+	st.reloadToViper()
+}
+
+// CacheInReplyToIDsMemRatioFlag returns the flag name for the 'Cache.InReplyToIDsMemRatio' field
+func CacheInReplyToIDsMemRatioFlag() string { return "cache-in-reply-to-ids-mem-ratio" }
+
+// GetCacheInReplyToIDsMemRatio safely fetches the value for global configuration 'Cache.InReplyToIDsMemRatio' field
+func GetCacheInReplyToIDsMemRatio() float64 { return global.GetCacheInReplyToIDsMemRatio() }
+
+// SetCacheInReplyToIDsMemRatio safely sets the value for global configuration 'Cache.InReplyToIDsMemRatio' field
+func SetCacheInReplyToIDsMemRatio(v float64) { global.SetCacheInReplyToIDsMemRatio(v) }
+
 // GetCacheInstanceMemRatio safely fetches the Configuration value for state's 'Cache.InstanceMemRatio' field
 func (st *ConfigState) GetCacheInstanceMemRatio() (v float64) {
 	st.mutex.RLock()
@@ -2949,6 +2999,31 @@ func GetCacheStatusFaveMemRatio() float64 { return global.GetCacheStatusFaveMemR
 // SetCacheStatusFaveMemRatio safely sets the value for global configuration 'Cache.StatusFaveMemRatio' field
 func SetCacheStatusFaveMemRatio(v float64) { global.SetCacheStatusFaveMemRatio(v) }
 
+// GetCacheStatusFaveIDsMemRatio safely fetches the Configuration value for state's 'Cache.StatusFaveIDsMemRatio' field
+func (st *ConfigState) GetCacheStatusFaveIDsMemRatio() (v float64) {
+	st.mutex.RLock()
+	v = st.config.Cache.StatusFaveIDsMemRatio
+	st.mutex.RUnlock()
+	return
+}
+
+// SetCacheStatusFaveIDsMemRatio safely sets the Configuration value for state's 'Cache.StatusFaveIDsMemRatio' field
+func (st *ConfigState) SetCacheStatusFaveIDsMemRatio(v float64) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.StatusFaveIDsMemRatio = v
+	st.reloadToViper()
+}
+
+// CacheStatusFaveIDsMemRatioFlag returns the flag name for the 'Cache.StatusFaveIDsMemRatio' field
+func CacheStatusFaveIDsMemRatioFlag() string { return "cache-status-fave-ids-mem-ratio" }
+
+// GetCacheStatusFaveIDsMemRatio safely fetches the value for global configuration 'Cache.StatusFaveIDsMemRatio' field
+func GetCacheStatusFaveIDsMemRatio() float64 { return global.GetCacheStatusFaveIDsMemRatio() }
+
+// SetCacheStatusFaveIDsMemRatio safely sets the value for global configuration 'Cache.StatusFaveIDsMemRatio' field
+func SetCacheStatusFaveIDsMemRatio(v float64) { global.SetCacheStatusFaveIDsMemRatio(v) }
+
 // GetCacheTagMemRatio safely fetches the Configuration value for state's 'Cache.TagMemRatio' field
 func (st *ConfigState) GetCacheTagMemRatio() (v float64) {
 	st.mutex.RLock()