From dd998688265215e0440bcc2b6057a98b5502b7bc Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 11 Jun 2024 14:14:22 -0700 Subject: [PATCH 01/35] Implement conversations API --- .../action/debug/conversations/migrate.go | 86 +++ cmd/gotosocial/debug.go | 15 + .../conversations/conversationdelete.go | 90 ++++ .../client/conversations/conversationread.go | 95 ++++ .../api/client/conversations/conversations.go | 10 +- .../client/conversations/conversationsget.go | 32 +- internal/cache/cache.go | 2 + internal/cache/db.go | 51 ++ internal/cache/invalidate.go | 5 + internal/cache/size.go | 15 + internal/config/config.go | 2 + internal/config/defaults.go | 2 + internal/config/helpers.gen.go | 50 ++ internal/db/bundb/bundb.go | 6 + internal/db/bundb/conversation.go | 505 ++++++++++++++++++ internal/db/bundb/conversation_test.go | 192 +++++++ .../20240611190733_add_conversations.go | 76 +++ internal/db/conversation.go | 56 ++ internal/db/db.go | 1 + internal/gtsmodel/conversation.go | 58 ++ internal/gtsmodel/emoji.go | 5 + internal/media/refetch.go | 7 +- internal/processing/account/delete.go | 7 + internal/processing/admin/emoji.go | 4 +- .../processing/conversations/conversations.go | 35 ++ .../conversations/conversations_test.go | 184 +++++++ internal/processing/conversations/delete.go | 53 ++ .../conversations/delete_test.go} | 10 +- internal/processing/conversations/get.go | 104 ++++ internal/processing/conversations/get_test.go | 33 ++ internal/processing/conversations/read.go | 84 +++ .../processing/conversations/read_test.go | 32 ++ internal/processing/processor.go | 39 +- internal/processing/stream/conversation.go | 45 ++ internal/processing/workers/fromclientapi.go | 4 + .../processing/workers/fromclientapi_test.go | 162 ++++++ internal/processing/workers/fromfediapi.go | 4 + .../workers/surfaceconversations.go | 203 +++++++ internal/processing/workers/util.go | 5 + internal/processing/workers/workers_test.go | 1 + internal/stream/stream.go | 4 + internal/typeutils/internaltofrontend.go | 50 ++ test/envparsing.sh | 2 + 43 files changed, 2393 insertions(+), 33 deletions(-) create mode 100644 cmd/gotosocial/action/debug/conversations/migrate.go create mode 100644 internal/api/client/conversations/conversationdelete.go create mode 100644 internal/api/client/conversations/conversationread.go create mode 100644 internal/db/bundb/conversation.go create mode 100644 internal/db/bundb/conversation_test.go create mode 100644 internal/db/bundb/migrations/20240611190733_add_conversations.go create mode 100644 internal/db/conversation.go create mode 100644 internal/gtsmodel/conversation.go create mode 100644 internal/processing/conversations/conversations.go create mode 100644 internal/processing/conversations/conversations_test.go create mode 100644 internal/processing/conversations/delete.go rename internal/{util/emoji.go => processing/conversations/delete_test.go} (75%) create mode 100644 internal/processing/conversations/get.go create mode 100644 internal/processing/conversations/get_test.go create mode 100644 internal/processing/conversations/read.go create mode 100644 internal/processing/conversations/read_test.go create mode 100644 internal/processing/stream/conversation.go create mode 100644 internal/processing/workers/surfaceconversations.go diff --git a/cmd/gotosocial/action/debug/conversations/migrate.go b/cmd/gotosocial/action/debug/conversations/migrate.go new file mode 100644 index 0000000000..7e08ef9c37 --- /dev/null +++ b/cmd/gotosocial/action/debug/conversations/migrate.go @@ -0,0 +1,86 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "context" + "fmt" + + "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/email" + "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/processing/stream" + "github.com/superseriousbusiness/gotosocial/internal/processing/workers" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +func initState(ctx context.Context) (*state.State, error) { + var state state.State + state.Caches.Init() + state.Caches.Start() + + // Set the state DB connection + dbConn, err := bundb.NewBunDBService(ctx, &state) + if err != nil { + return nil, fmt.Errorf("error creating dbConn: %w", err) + } + state.DB = dbConn + + return &state, nil +} + +func stopState(state *state.State) error { + err := state.DB.Close() + state.Caches.Stop() + return err +} + +// Migrate processes every DM to create conversations. +var Migrate action.GTSAction = func(ctx context.Context) (err error) { + state, err := initState(ctx) + if err != nil { + return err + } + + defer func() { + // Ensure state gets stopped on return. + if err := stopState(state); err != nil { + log.Error(ctx, err) + } + }() + + streamProcessor := stream.New(state, oauth.New(ctx, state.DB)) + surface := workers.Surface{ + State: state, + Converter: typeutils.NewConverter(state), + Stream: &streamProcessor, + Filter: visibility.NewFilter(state), + } + if surface.EmailSender, err = email.NewNoopSender(func(toAddress string, message string) {}); err != nil { + return nil + } + + return state.DB.MigrateConversations(ctx, func(ctx context.Context, status *gtsmodel.Status) error { + return surface.UpdateConversationsForStatus(ctx, status, false) + }) +} diff --git a/cmd/gotosocial/debug.go b/cmd/gotosocial/debug.go index c01baeb8b1..f248167fdb 100644 --- a/cmd/gotosocial/debug.go +++ b/cmd/gotosocial/debug.go @@ -20,6 +20,7 @@ package main import ( "github.com/spf13/cobra" configaction "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/debug/config" + "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/debug/conversations" "github.com/superseriousbusiness/gotosocial/internal/config" ) @@ -41,5 +42,19 @@ func debugCommands() *cobra.Command { } config.AddServerFlags(debugConfigCmd) debugCmd.AddCommand(debugConfigCmd) + + debugMigrateConversationsCmd := &cobra.Command{ + Use: "migrate-conversations", + Short: "process EVERY DM to create conversations", + PreRunE: func(cmd *cobra.Command, args []string) error { + return preRun(preRunArgs{cmd: cmd}) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cmd.Context(), conversations.Migrate) + }, + } + config.AddServerFlags(debugMigrateConversationsCmd) + debugCmd.AddCommand(debugMigrateConversationsCmd) + return debugCmd } diff --git a/internal/api/client/conversations/conversationdelete.go b/internal/api/client/conversations/conversationdelete.go new file mode 100644 index 0000000000..2737e6cab7 --- /dev/null +++ b/internal/api/client/conversations/conversationdelete.go @@ -0,0 +1,90 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "net/http" + + "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// ConversationDELETEHandler swagger:operation DELETE /api/v1/conversations/{id} conversationDelete +// +// Delete a single conversation with the given ID. +// +// --- +// tags: +// - conversations +// +// produces: +// - application/json +// +// parameters: +// - +// name: id +// type: string +// description: ID of the conversation +// in: path +// required: true +// +// security: +// - OAuth2 Bearer: +// - write:conversations +// +// responses: +// '200': +// description: conversation deleted +// '400': +// description: bad request +// '401': +// description: unauthorized +// '404': +// description: not found +// '406': +// description: not acceptable +// '500': +// description: internal server error +func (m *Module) ConversationDELETEHandler(c *gin.Context) { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1) + return + } + + id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey)) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + errWithCode = m.processor.Conversations().Delete(c.Request.Context(), authed.Account, id) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + c.JSON(http.StatusOK, apiutil.EmptyJSONObject) +} diff --git a/internal/api/client/conversations/conversationread.go b/internal/api/client/conversations/conversationread.go new file mode 100644 index 0000000000..7f68a2a33f --- /dev/null +++ b/internal/api/client/conversations/conversationread.go @@ -0,0 +1,95 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "net/http" + + "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// ConversationReadPOSTHandler swagger:operation POST /api/v1/conversation/{id}/read conversationRead +// +// Mark a conversation with the given ID as read. +// +// --- +// tags: +// - conversations +// +// produces: +// - application/json +// +// parameters: +// - +// name: id +// in: path +// type: string +// required: true +// description: ID of the conversation. +// +// security: +// - OAuth2 Bearer: +// - write:conversations +// +// responses: +// '200': +// name: conversation +// description: Updated conversation. +// schema: +// "$ref": "#/definitions/conversation" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '404': +// description: not found +// '406': +// description: not acceptable +// '422': +// description: unprocessable content +// '500': +// description: internal server error +func (m *Module) ConversationReadPOSTHandler(c *gin.Context) { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1) + return + } + + id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey)) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + apiConversation, errWithCode := m.processor.Conversations().Read(c.Request.Context(), authed.Account, id) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + apiutil.JSON(c, http.StatusOK, apiConversation) +} diff --git a/internal/api/client/conversations/conversations.go b/internal/api/client/conversations/conversations.go index be19a9cdce..e742c8d3d9 100644 --- a/internal/api/client/conversations/conversations.go +++ b/internal/api/client/conversations/conversations.go @@ -21,13 +21,17 @@ import ( "net/http" "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/processing" ) const ( - // BasePath is the base URI path for serving - // conversations, minus the api prefix. + // BasePath is the base path for serving the conversations API, minus the 'api' prefix. BasePath = "/v1/conversations" + // BasePathWithID is the base path with the ID key in it, for operations on an existing conversation. + BasePathWithID = BasePath + "/:" + apiutil.IDKey + // ReadPathWithID is the path for marking an existing conversation as read. + ReadPathWithID = BasePathWithID + "/read" ) type Module struct { @@ -42,4 +46,6 @@ func New(processor *processing.Processor) *Module { func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) { attachHandler(http.MethodGet, BasePath, m.ConversationsGETHandler) + attachHandler(http.MethodDelete, BasePathWithID, m.ConversationDELETEHandler) + attachHandler(http.MethodPost, ReadPathWithID, m.ConversationReadPOSTHandler) } diff --git a/internal/api/client/conversations/conversationsget.go b/internal/api/client/conversations/conversationsget.go index 11bddb1cec..28925ae367 100644 --- a/internal/api/client/conversations/conversationsget.go +++ b/internal/api/client/conversations/conversationsget.go @@ -24,14 +24,13 @@ import ( apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/paging" ) // ConversationsGETHandler swagger:operation GET /api/v1/conversations conversationsGet // // Get an array of (direct message) conversations that requesting account is involved in. // -// NOT IMPLEMENTED YET: Will currently always return an array of length 0. -// // The next and previous queries can be parsed from the returned Link header. // Example: // @@ -108,7 +107,8 @@ import ( // '500': // description: internal server error func (m *Module) ConversationsGETHandler(c *gin.Context) { - if _, err := oauth.Authed(c, true, true, true, true); err != nil { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) return } @@ -118,5 +118,29 @@ func (m *Module) ConversationsGETHandler(c *gin.Context) { return } - apiutil.Data(c, http.StatusOK, apiutil.AppJSON, apiutil.EmptyJSONArray) + page, errWithCode := paging.ParseIDPage(c, + 1, // min limit + 80, // max limit + 40, // default limit + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + resp, errWithCode := m.processor.Conversations().GetAll( + c.Request.Context(), + authed.Account, + page, + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + if resp.LinkHeader != "" { + c.Header("Link", resp.LinkHeader) + } + + apiutil.JSON(c, http.StatusOK, resp.Items) } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index bb910f3e6f..b522037960 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -60,6 +60,8 @@ func (c *Caches) Init() { c.initBlockIDs() c.initBoostOfIDs() c.initClient() + c.initConversation() + c.initConversationIDs() c.initDomainAllow() c.initDomainBlock() c.initEmoji() diff --git a/internal/cache/db.go b/internal/cache/db.go index d0fe77649c..569aeb60a0 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -56,6 +56,12 @@ type GTSCaches struct { // Client provides access to the gtsmodel Client database cache. Client StructCache[*gtsmodel.Client] + // Conversation provides access to the gtsmodel Conversation database cache. + Conversation StructCache[*gtsmodel.Conversation] + + // ConversationIDs provides access to the conversation IDs database cache. + ConversationIDs SliceCache[string] + // DomainAllow provides access to the domain allow database cache. DomainAllow *domain.Cache @@ -423,6 +429,51 @@ func (c *Caches) initClient() { }) } +func (c *Caches) initConversation() { + cap := calculateResultCacheMax( + sizeofConversation(), // model in-mem size. + config.GetCacheConversationMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(c1 *gtsmodel.Conversation) *gtsmodel.Conversation { + c2 := new(gtsmodel.Conversation) + *c2 = *c1 + + // Don't include ptr fields that + // will be populated separately. + // See internal/db/bundb/conversation.go. + c2.Account = nil + c2.OtherAccounts = nil + c2.LastStatus = nil + + return c2 + } + + c.GTS.Conversation.Init(structr.CacheConfig[*gtsmodel.Conversation]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "ThreadID,AccountID,OtherAccountsKey"}, + {Fields: "AccountID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + Copy: copyF, + Invalidate: c.OnInvalidateConversation, + }) +} + +func (c *Caches) initConversationIDs() { + cap := calculateSliceCacheMax( + config.GetCacheConversationIDsMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + c.GTS.ConversationIDs.Init(0, cap) +} + func (c *Caches) initDomainAllow() { c.GTS.DomainAllow = new(domain.Cache) } diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 088e7f91f0..204a2af19f 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -83,6 +83,11 @@ func (c *Caches) OnInvalidateClient(client *gtsmodel.Client) { c.GTS.Token.Invalidate("ClientID", client.ID) } +func (c *Caches) OnInvalidateConversation(conversation *gtsmodel.Conversation) { + // Invalidate source account's conversation lists. + c.GTS.ConversationIDs.Invalidate(conversation.AccountID) +} + func (c *Caches) OnInvalidateEmojiCategory(category *gtsmodel.EmojiCategory) { // Invalidate any emoji in this category. c.GTS.Emoji.Invalidate("CategoryID", category.ID) diff --git a/internal/cache/size.go b/internal/cache/size.go index fb1f165c2e..bc09f48c51 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -19,6 +19,7 @@ package cache import ( "crypto/rsa" + "strings" "time" "unsafe" @@ -319,6 +320,20 @@ func sizeofClient() uintptr { })) } +func sizeofConversation() uintptr { + return uintptr(size.Of(>smodel.Conversation{ + ID: exampleID, + CreatedAt: exampleTime, + UpdatedAt: exampleTime, + AccountID: exampleID, + OtherAccountIDs: []string{exampleID, exampleID, exampleID}, + OtherAccountsKey: strings.Join([]string{exampleID, exampleID, exampleID}, ","), + ThreadID: exampleID, + LastStatusID: exampleID, + Read: func() *bool { ok := true; return &ok }(), + })) +} + func sizeofEmoji() uintptr { return uintptr(size.Of(>smodel.Emoji{ ID: exampleID, diff --git a/internal/config/config.go b/internal/config/config.go index 8d410f6acf..e4c2e19290 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -201,6 +201,8 @@ type CacheConfiguration struct { BlockIDsMemRatio float64 `name:"block-ids-mem-ratio"` BoostOfIDsMemRatio float64 `name:"boost-of-ids-mem-ratio"` ClientMemRatio float64 `name:"client-mem-ratio"` + ConversationMemRatio float64 `name:"conversation-mem-ratio"` + ConversationIDsMemRatio float64 `name:"conversation-ids-mem-ratio"` EmojiMemRatio float64 `name:"emoji-mem-ratio"` EmojiCategoryMemRatio float64 `name:"emoji-category-mem-ratio"` FilterMemRatio float64 `name:"filter-mem-ratio"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 8a76cc21a0..94301d696a 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -165,6 +165,8 @@ var Defaults = Configuration{ BlockIDsMemRatio: 3, BoostOfIDsMemRatio: 3, ClientMemRatio: 0.1, + ConversationMemRatio: 2, + ConversationIDsMemRatio: 3, EmojiMemRatio: 3, EmojiCategoryMemRatio: 0.1, FilterMemRatio: 0.5, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 71a77e7538..27b308f535 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -2975,6 +2975,56 @@ func GetCacheClientMemRatio() float64 { return global.GetCacheClientMemRatio() } // SetCacheClientMemRatio safely sets the value for global configuration 'Cache.ClientMemRatio' field func SetCacheClientMemRatio(v float64) { global.SetCacheClientMemRatio(v) } +// GetCacheConversationMemRatio safely fetches the Configuration value for state's 'Cache.ConversationMemRatio' field +func (st *ConfigState) GetCacheConversationMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.ConversationMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheConversationMemRatio safely sets the Configuration value for state's 'Cache.ConversationMemRatio' field +func (st *ConfigState) SetCacheConversationMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.ConversationMemRatio = v + st.reloadToViper() +} + +// CacheConversationMemRatioFlag returns the flag name for the 'Cache.ConversationMemRatio' field +func CacheConversationMemRatioFlag() string { return "cache-conversation-mem-ratio" } + +// GetCacheConversationMemRatio safely fetches the value for global configuration 'Cache.ConversationMemRatio' field +func GetCacheConversationMemRatio() float64 { return global.GetCacheConversationMemRatio() } + +// SetCacheConversationMemRatio safely sets the value for global configuration 'Cache.ConversationMemRatio' field +func SetCacheConversationMemRatio(v float64) { global.SetCacheConversationMemRatio(v) } + +// GetCacheConversationIDsMemRatio safely fetches the Configuration value for state's 'Cache.ConversationIDsMemRatio' field +func (st *ConfigState) GetCacheConversationIDsMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.ConversationIDsMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheConversationIDsMemRatio safely sets the Configuration value for state's 'Cache.ConversationIDsMemRatio' field +func (st *ConfigState) SetCacheConversationIDsMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.ConversationIDsMemRatio = v + st.reloadToViper() +} + +// CacheConversationIDsMemRatioFlag returns the flag name for the 'Cache.ConversationIDsMemRatio' field +func CacheConversationIDsMemRatioFlag() string { return "cache-conversation-ids-mem-ratio" } + +// GetCacheConversationIDsMemRatio safely fetches the value for global configuration 'Cache.ConversationIDsMemRatio' field +func GetCacheConversationIDsMemRatio() float64 { return global.GetCacheConversationIDsMemRatio() } + +// SetCacheConversationIDsMemRatio safely sets the value for global configuration 'Cache.ConversationIDsMemRatio' field +func SetCacheConversationIDsMemRatio(v float64) { global.SetCacheConversationIDsMemRatio(v) } + // GetCacheEmojiMemRatio safely fetches the Configuration value for state's 'Cache.EmojiMemRatio' field func (st *ConfigState) GetCacheEmojiMemRatio() (v float64) { st.mutex.RLock() diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index e7256c2760..a9f4ae0b8d 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -56,6 +56,7 @@ type DBService struct { db.Admin db.Application db.Basic + db.Conversation db.Domain db.Emoji db.HeaderFilter @@ -157,6 +158,7 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { // https://bun.uptrace.dev/orm/many-to-many-relation/ for _, t := range []interface{}{ >smodel.AccountToEmoji{}, + >smodel.ConversationToStatus{}, >smodel.StatusToEmoji{}, >smodel.StatusToTag{}, >smodel.ThreadToStatus{}, @@ -187,6 +189,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { Basic: &basicDB{ db: db, }, + Conversation: &conversationDB{ + db: db, + state: state, + }, Domain: &domainDB{ db: db, state: state, diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go new file mode 100644 index 0000000000..632ae51328 --- /dev/null +++ b/internal/db/bundb/conversation.go @@ -0,0 +1,505 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb + +import ( + "context" + "errors" + "slices" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/paging" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/util" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +type conversationDB struct { + db *bun.DB + state *state.State +} + +func (c *conversationDB) GetConversationByID(ctx context.Context, id string) (*gtsmodel.Conversation, error) { + return c.getConversation( + ctx, + "ID", + func(conversation *gtsmodel.Conversation) error { + return c.db. + NewSelect(). + Model(conversation). + Where("? = ?", bun.Ident("id"), id). + Scan(ctx) + }, + id, + ) +} + +func (c *conversationDB) GetConversationByThreadAndAccountIDs(ctx context.Context, threadID string, accountID string, otherAccountIDs []string) (*gtsmodel.Conversation, error) { + otherAccountsKey := gtsmodel.ConversationOtherAccountsKey(otherAccountIDs) + return c.getConversation( + ctx, + "ThreadID,AccountID,OtherAccountsKey", + func(conversation *gtsmodel.Conversation) error { + return c.db. + NewSelect(). + Model(conversation). + Where("? = ?", bun.Ident("thread_id"), threadID). + Where("? = ?", bun.Ident("account_id"), accountID). + Where("? = ?", bun.Ident("other_accounts_key"), otherAccountsKey). + Scan(ctx) + }, + threadID, + accountID, + otherAccountsKey, + ) +} + +func (c *conversationDB) getConversation( + ctx context.Context, + lookup string, + dbQuery func(conversation *gtsmodel.Conversation) error, + keyParts ...any, +) (*gtsmodel.Conversation, error) { + // Fetch conversation from cache with loader callback + conversation, err := c.state.Caches.GTS.Conversation.LoadOne(lookup, func() (*gtsmodel.Conversation, error) { + var conversation gtsmodel.Conversation + + // Not cached! Perform database query + if err := dbQuery(&conversation); err != nil { + return nil, err + } + + return &conversation, nil + }, keyParts...) + if err != nil { + // already processe + return nil, err + } + + if gtscontext.Barebones(ctx) { + // Only a barebones model was requested. + return conversation, nil + } + + if err := c.populateConversation(ctx, conversation); err != nil { + return nil, err + } + + return conversation, nil +} + +func (c *conversationDB) populateConversation(ctx context.Context, conversation *gtsmodel.Conversation) error { + var ( + errs gtserror.MultiError + err error + ) + + if conversation.Account == nil { + conversation.Account, err = c.state.DB.GetAccountByID( + gtscontext.SetBarebones(ctx), + conversation.AccountID, + ) + if err != nil { + errs.Appendf("error populating conversation owner account: %w", err) + } + } + + if conversation.OtherAccounts == nil { + conversation.OtherAccounts, err = c.state.DB.GetAccountsByIDs( + gtscontext.SetBarebones(ctx), + conversation.OtherAccountIDs, + ) + if err != nil { + errs.Appendf("error populating other conversation accounts: %w", err) + } + } + + if conversation.LastStatus == nil && conversation.LastStatusID != "" { + conversation.LastStatus, err = c.state.DB.GetStatusByID( + gtscontext.SetBarebones(ctx), + conversation.LastStatusID, + ) + if err != nil { + errs.Appendf("error populating conversation last status: %w", err) + } + } + + return errs.Combine() +} + +func (c *conversationDB) GetConversationsByOwnerAccountID(ctx context.Context, accountID string, page *paging.Page) ([]*gtsmodel.Conversation, error) { + conversationIDs, err := c.getAccountConversationIDs(ctx, accountID, page) + if err != nil { + return nil, err + } + return c.getConversationsByIDs(ctx, conversationIDs) +} + +func (c *conversationDB) getAccountConversationIDs(ctx context.Context, accountID string, page *paging.Page) ([]string, error) { + return loadPagedIDs(&c.state.Caches.GTS.ConversationIDs, accountID, page, func() ([]string, error) { + var conversationIDs []string + + // Conversation IDs not in cache. Perform DB query. + if _, err := c.db. + NewSelect(). + TableExpr("?", bun.Ident("conversations")). + ColumnExpr("?", bun.Ident("id")). + Where("? = ?", bun.Ident("account_id"), accountID). + OrderExpr("? DESC", bun.Ident("id")). + Exec(ctx, &conversationIDs); // nocollapse + err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, err + } + + return conversationIDs, nil + }) +} + +func (c *conversationDB) getConversationsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Conversation, error) { + // Load all conversation IDs via cache loader callbacks. + conversations, err := c.state.Caches.GTS.Conversation.LoadIDs("ID", + ids, + func(uncached []string) ([]*gtsmodel.Conversation, error) { + // Preallocate expected length of uncached conversations. + conversations := make([]*gtsmodel.Conversation, 0, len(uncached)) + + // Perform database query scanning the remaining (uncached) IDs. + if err := c.db.NewSelect(). + Model(&conversations). + Where("? IN (?)", bun.Ident("id"), bun.In(uncached)). + Scan(ctx); err != nil { + return nil, err + } + + return conversations, nil + }, + ) + if err != nil { + return nil, err + } + + // Reorder the conversations by their IDs to ensure correct order. + getID := func(b *gtsmodel.Conversation) string { return b.ID } + util.OrderBy(conversations, ids, getID) + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return conversations, nil + } + + // Populate all loaded conversations, removing those we fail to populate. + conversations = slices.DeleteFunc(conversations, func(conversation *gtsmodel.Conversation) bool { + if err := c.populateConversation(ctx, conversation); err != nil { + log.Errorf(ctx, "error populating conversation %s: %v", conversation.ID, err) + return true + } + return false + }) + + return conversations, nil +} + +func (c *conversationDB) UpdateConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error { + // If we're updating by column, ensure "updated_at" is included. + if len(columns) > 0 { + columns = append(columns, "updated_at") + } + + return c.state.Caches.GTS.Conversation.Store(conversation, func() error { + _, err := c.db.NewUpdate(). + Model(conversation). + Column(columns...). + Where("? = ?", bun.Ident("id"), conversation.ID). + Exec(ctx) + return err + }) +} + +func (c *conversationDB) AddStatusToConversation(ctx context.Context, conversation *gtsmodel.Conversation, status *gtsmodel.Status) (*gtsmodel.Conversation, error) { + // Assume that if the conversation owner posted the status, they've already read it. + statusAuthoredByConversationOwner := status.AccountID == conversation.AccountID + + // Update the existing conversation. + // If there is no previous last status or this one is more recently created, set it as the last status. + if conversation.LastStatus == nil || conversation.LastStatus.CreatedAt.Before(status.CreatedAt) { + conversation.LastStatusID = status.ID + conversation.LastStatus = status + } + // If the conversation is unread, leave it marked as unread. + // If the conversation is read but this status might not have been, mark the conversation as unread. + if !statusAuthoredByConversationOwner { + conversation.Read = util.Ptr(false) + } + + // Link the conversation to the status. + conversationToStatus := >smodel.ConversationToStatus{ + ConversationID: conversation.ID, + StatusID: status.ID, + } + + // Upsert the conversation and insert the link, then cache the conversation. + if err := c.state.Caches.GTS.Conversation.Store(conversation, func() error { + return c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + if _, err := tx.NewInsert(). + Model(conversationToStatus). + Exec(ctx); // nocollapse + err != nil { + return err + } + + if _, err := NewUpsert(tx). + Model(conversation). + Constraint("id"). + Column("last_status_id", "read", "updated_at"). + Exec(ctx); // nocollapse + err != nil { + return err + } + + return nil + }) + }); err != nil { + return nil, err + } + + return conversation, nil +} + +func (c *conversationDB) DeleteConversationByID(ctx context.Context, id string) error { + // Load conversation into cache before attempting a delete, + // as we need it cached in order to trigger the invalidate + // callback. This in turn invalidates others. + _, err := c.GetConversationByID(gtscontext.SetBarebones(ctx), id) + if err != nil { + if errors.Is(err, db.ErrNoEntries) { + // not an issue. + err = nil + } + return err + } + + // Drop this now-cached conversation on return after delete. + defer c.state.Caches.GTS.Conversation.Invalidate("ID", id) + + // Finally delete conversation from DB. + _, err = c.db.NewDelete(). + Model((*gtsmodel.Conversation)(nil)). + Where("? = ?", bun.Ident("id"), id). + Exec(ctx) + return err +} + +func (c *conversationDB) DeleteConversationsByOwnerAccountID(ctx context.Context, accountID string) error { + defer func() { + // Invalidate any cached conversations and conversation IDs owned by this account on return. + // Conversation invalidate hooks only invalidate the conversation ID cache, + // so we don't need to load all conversations into the cache to run invalidation hooks, + // as with some other object types (blocks, for example). + c.state.Caches.GTS.Conversation.Invalidate("AccountID", accountID) + // In case there were no cached conversations, explicitly invalidate conversation ID cache. + c.state.Caches.GTS.ConversationIDs.Invalidate(accountID) + }() + + return c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Delete conversations matching the account ID. + deletedConversationIDs := []string{} + if err := c.db.NewDelete(). + Model((*gtsmodel.Conversation)(nil)). + Where("? = ?", bun.Ident("account_id"), accountID). + Returning("?", bun.Ident("id")). + Scan(ctx, &deletedConversationIDs); // nocollapse + err != nil { + return err + } + + // Delete any conversation-to-status links matching the deleted conversation IDs. + if _, err := c.db.NewDelete(). + Model((*gtsmodel.ConversationToStatus)(nil)). + Where("? IN (?)", bun.Ident("conversation_id"), bun.In(deletedConversationIDs)). + Exec(ctx); // nocollapse + err != nil { + return err + } + + return nil + }) +} + +func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, statusID string) error { + // SQL returning the current time. + var nowSQL string + switch c.db.Dialect().Name() { + case dialect.SQLite: + nowSQL = "DATE('now')" + case dialect.PG: + nowSQL = "NOW()" + default: + log.Panicf(nil, "db conn %s was neither pg nor sqlite", c.db) + } + + updatedConversationIDs := []string{} + deletedConversationIDs := []string{} + + if err := c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Delete this status from conversation-to-status links. + if _, err := tx.NewDelete(). + Model((*gtsmodel.ConversationToStatus)(nil)). + Where("? = ?", bun.Ident("status_id"), statusID). + Exec(ctx); // nocollapse + err != nil { + return err + } + + // Create a temporary table with all statuses other than the deleted status + // in each conversation for which the deleted status is the last status + // (if there are such statuses). + if _, err := tx.NewRaw( + ` + CREATE TEMPORARY TABLE conversation_statuses AS + SELECT + conversations.id conversation_id, + conversation_to_statuses.status_id id, + statuses.created_at + FROM conversations + LEFT JOIN conversation_to_statuses + ON conversations.id = conversation_to_statuses.conversation_id + AND conversation_to_statuses.status_id != ?0 + LEFT JOIN statuses + ON conversation_to_statuses.status_id = statuses.id + WHERE conversations.last_status_id = ?0 + `, + statusID, + ).Exec(ctx); // nocollapse + err != nil { + return err + } + + // Create a temporary table with the most recently created status in each conversation + // for which the deleted status is the last status (if there is such a status). + if _, err := tx.NewRaw( + ` + CREATE TEMPORARY TABLE latest_conversation_statuses AS + SELECT + conversation_statuses.conversation_id, + conversation_statuses.id + FROM conversation_statuses + LEFT JOIN conversation_statuses later_statuses + ON conversation_statuses.conversation_id = later_statuses.conversation_id + AND later_statuses.created_at > conversation_statuses.created_at + WHERE later_statuses.id IS NULL + `, + ).Exec(ctx); // nocollapse + err != nil { + return err + } + + // For every conversation where the given status was the last one, + // reset its last status to the most recently created in the conversation other than that one, + // if there is such a status. + // Return conversation IDs for invalidation. + if err := tx.NewRaw( + ` + UPDATE conversations + SET + last_status_id = latest_conversation_statuses.id, + updated_at = ? + FROM latest_conversation_statuses + WHERE conversations.id = latest_conversation_statuses.conversation_id + AND latest_conversation_statuses.id IS NOT NULL + RETURNING id + `, + bun.Safe(nowSQL), + ).Scan(ctx, &updatedConversationIDs); // nocollapse + err != nil { + return err + } + + // If there is no such status, delete the conversation. + // Return conversation IDs for invalidation. + if err := tx.NewRaw( + ` + DELETE FROM conversations + WHERE id IN ( + SELECT conversation_id + FROM latest_conversation_statuses + WHERE id IS NULL + ) + RETURNING id + `, + ).Scan(ctx, &deletedConversationIDs); // nocollapse + err != nil { + return err + } + + return nil + }); err != nil { + return err + } + + conversationIDs := append(updatedConversationIDs, deletedConversationIDs...) + c.state.Caches.GTS.Conversation.InvalidateIDs("ID", conversationIDs) + + return nil +} + +func (c *conversationDB) MigrateConversations(ctx context.Context, migrateStatus func(ctx context.Context, status *gtsmodel.Status) error) error { + log.Info(ctx, "migrating DMs to conversations…") + + batchSize := 100 + statuses := make([]*gtsmodel.Status, 0, batchSize) + minID := id.Lowest + for { + if err := c.db. + NewSelect(). + Model(&statuses). + Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityDirect). + Where("? > ?", bun.Ident("id"), minID). + Order("id ASC"). + Limit(batchSize). + Scan(ctx); err != nil { + return err + } + + if len(statuses) == 0 { + break + } + log.Infof(ctx, "migrating %d DMs starting past %s", len(statuses), minID) + minID = statuses[len(statuses)-1].ID + + for _, status := range statuses { + if err := c.state.DB.PopulateStatus(ctx, status); err != nil { + log.Errorf(ctx, "couldn't populate DM %s for conversation migration: %v", status.ID, err) + continue + } + if err := migrateStatus(ctx, status); err != nil { + log.Errorf(ctx, "couldn't process DM %s for conversation migration: %v", status.ID, err) + continue + } + } + } + + log.Info(ctx, "finished migrating DMs to conversations.") + + return nil +} diff --git a/internal/db/bundb/conversation_test.go b/internal/db/bundb/conversation_test.go new file mode 100644 index 0000000000..3588fe60d1 --- /dev/null +++ b/internal/db/bundb/conversation_test.go @@ -0,0 +1,192 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type ConversationTestSuite struct { + BunDBStandardTestSuite + + // account is the owner of statuses and conversations in these tests (must be local). + account *gtsmodel.Account + // now is the timestamp used as a base for creating new statuses in any given test. + now time.Time + // threadID is the thread used for statuses in any given test. + threadID string +} + +func (suite *ConversationTestSuite) SetupSuite() { + suite.BunDBStandardTestSuite.SetupSuite() + + suite.account = suite.testAccounts["local_account_1"] +} + +func (suite *ConversationTestSuite) SetupTest() { + suite.BunDBStandardTestSuite.SetupTest() + + suite.now = time.Now() + suite.threadID = id.NewULID() +} + +// newStatus creates a new status in the DB that would be eligible for a conversation, optionally replying to a previous status. +func (suite *ConversationTestSuite) newStatus(nowOffset time.Duration, inReplyTo *gtsmodel.Status) *gtsmodel.Status { + statusID := id.NewULID() + createdAt := suite.now.Add(nowOffset) + status := >smodel.Status{ + ID: statusID, + CreatedAt: createdAt, + UpdatedAt: createdAt, + URI: "http://localhost:8080/users/" + suite.account.Username + "/statuses/" + statusID, + AccountID: suite.account.ID, + AccountURI: suite.account.URI, + Local: util.Ptr(true), + ThreadID: suite.threadID, + Visibility: gtsmodel.VisibilityDirect, + ActivityStreamsType: ap.ObjectNote, + Federated: util.Ptr(true), + Boostable: util.Ptr(true), + Replyable: util.Ptr(true), + Likeable: util.Ptr(true), + } + if inReplyTo != nil { + status.InReplyToID = inReplyTo.ID + status.InReplyToURI = inReplyTo.URI + status.InReplyToAccountID = inReplyTo.AccountID + } + if err := suite.db.PutStatus(context.Background(), status); err != nil { + suite.FailNow(err.Error()) + } + + return status +} + +// newConversation creates a new conversation not yet in the DB. +func (suite *ConversationTestSuite) newConversation() *gtsmodel.Conversation { + return >smodel.Conversation{ + ID: id.NewULID(), + AccountID: suite.account.ID, + ThreadID: suite.threadID, + Read: util.Ptr(true), + } +} + +// addStatus adds a status to a conversation and ends the test if that fails. +func (suite *ConversationTestSuite) addStatus( + conversation *gtsmodel.Conversation, + status *gtsmodel.Status, +) *gtsmodel.Conversation { + conversation, err := suite.db.AddStatusToConversation(context.Background(), conversation, status) + if err != nil { + suite.FailNow(err.Error()) + } + return conversation +} + +// deleteStatus deletes a status from conversations and ends the test if that fails. +func (suite *ConversationTestSuite) deleteStatus(statusID string) { + err := suite.db.DeleteStatusFromConversations(context.Background(), statusID) + if err != nil { + suite.FailNow(err.Error()) + } +} + +// getConversation fetches a conversation by ID and ends the test if that fails. +func (suite *ConversationTestSuite) getConversation(conversationID string) *gtsmodel.Conversation { + conversation, err := suite.db.GetConversationByID(context.Background(), conversationID) + if err != nil { + suite.FailNow(err.Error()) + } + return conversation +} + +// Adding a status to a new conversation should set the last status. +func (suite *ConversationTestSuite) TestAddStatusToNewConversation() { + initial := suite.newStatus(0, nil) + conversation := suite.addStatus(suite.newConversation(), initial) + suite.Equal(initial.ID, conversation.LastStatusID) + if suite.NotNil(conversation.Read) { + // In this test suite, the author of the statuses is also the owner of the conversation, + // so the conversation should be marked as read. + suite.True(*conversation.Read) + } +} + +// Adding a newer status to an existing conversation should update the last status. +func (suite *ConversationTestSuite) TestAddStatusToExistingConversation() { + initial := suite.newStatus(0, nil) + conversation := suite.addStatus(suite.newConversation(), initial) + + reply := suite.newStatus(1, initial) + conversation = suite.addStatus(conversation, reply) + suite.Equal(reply.ID, conversation.LastStatusID) + if suite.NotNil(conversation.Read) { + suite.True(*conversation.Read) + } +} + +// If we delete a status that is in a conversation but not the last status, +// the conversation's last status should not change. +func (suite *ConversationTestSuite) TestDeleteNonLastStatus() { + initial := suite.newStatus(0, nil) + conversation := suite.addStatus(suite.newConversation(), initial) + reply := suite.newStatus(1, initial) + conversation = suite.addStatus(conversation, reply) + + suite.deleteStatus(initial.ID) + conversation = suite.getConversation(conversation.ID) + suite.Equal(reply.ID, conversation.LastStatusID) +} + +// If we delete the last status in a conversation that has other statuses, +// a previous status should become the new last status. +func (suite *ConversationTestSuite) TestDeleteLastStatus() { + initial := suite.newStatus(0, nil) + conversation := suite.addStatus(suite.newConversation(), initial) + reply := suite.newStatus(1, initial) + conversation = suite.addStatus(conversation, reply) + + suite.deleteStatus(reply.ID) + conversation = suite.getConversation(conversation.ID) + suite.Equal(initial.ID, conversation.LastStatusID) +} + +// If we delete the only status in a conversation, +// the conversation should be deleted as well. +func (suite *ConversationTestSuite) TestDeleteOnlyStatus() { + initial := suite.newStatus(0, nil) + conversation := suite.addStatus(suite.newConversation(), initial) + + suite.deleteStatus(initial.ID) + _, err := suite.db.GetConversationByID(context.Background(), conversation.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func TestConversationTestSuite(t *testing.T) { + suite.Run(t, new(ConversationTestSuite)) +} diff --git a/internal/db/bundb/migrations/20240611190733_add_conversations.go b/internal/db/bundb/migrations/20240611190733_add_conversations.go new file mode 100644 index 0000000000..37f3de5c23 --- /dev/null +++ b/internal/db/bundb/migrations/20240611190733_add_conversations.go @@ -0,0 +1,76 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migrations + +import ( + "context" + + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + for _, model := range []interface{}{ + >smodel.Conversation{}, + >smodel.ConversationToStatus{}, + } { + if _, err := tx. + NewCreateTable(). + Model(model). + IfNotExists(). + Exec(ctx); err != nil { + return err + } + } + + // Add indexes to the conversation table. + for index, columns := range map[string][]string{ + "conversations_account_id_idx": { + "account_id", + }, + "conversations_last_status_id_idx": { + "last_status_id", + }, + } { + if _, err := tx. + NewCreateIndex(). + Model(>smodel.Conversation{}). + Index(index). + Column(columns...). + IfNotExists(). + Exec(ctx); err != nil { + return err + } + } + + return nil + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/conversation.go b/internal/db/conversation.go new file mode 100644 index 0000000000..6538a9efe2 --- /dev/null +++ b/internal/db/conversation.go @@ -0,0 +1,56 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/paging" +) + +type Conversation interface { + // GetConversationByID gets a single conversation by ID. + GetConversationByID(ctx context.Context, id string) (*gtsmodel.Conversation, error) + + // GetConversationByThreadAndAccountIDs retrieves a conversation by thread ID and participant account IDs, if it exists. + GetConversationByThreadAndAccountIDs(ctx context.Context, threadID string, accountID string, otherAccountIDs []string) (*gtsmodel.Conversation, error) + + // GetConversationsByOwnerAccountID gets all conversations owned by the given account, with optional paging. + GetConversationsByOwnerAccountID(ctx context.Context, accountID string, page *paging.Page) ([]*gtsmodel.Conversation, error) + + // UpdateConversation updates an existing conversation. + UpdateConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error + + // AddStatusToConversation takes a conversation (which may or may not exist in the DB yet) and a status. + // It will link the status to the conversation, and if the status is newer than the last status, + // it will become the last status. This happens in a transaction. + AddStatusToConversation(ctx context.Context, conversation *gtsmodel.Conversation, status *gtsmodel.Status) (*gtsmodel.Conversation, error) + + // DeleteConversationByID deletes a conversation, removing it from the owning account's conversation list. + DeleteConversationByID(ctx context.Context, id string) error + + // DeleteConversationsByOwnerAccountID deletes all conversations owned by the given account. + DeleteConversationsByOwnerAccountID(ctx context.Context, accountID string) error + + // DeleteStatusFromConversations handles when a status is deleted by nulling out the last status for + // any conversations in which it was the last status. + DeleteStatusFromConversations(ctx context.Context, statusID string) error + + MigrateConversations(ctx context.Context, migrateStatus func(ctx context.Context, status *gtsmodel.Status) error) error +} diff --git a/internal/db/db.go b/internal/db/db.go index 330766306b..31095e3b04 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -28,6 +28,7 @@ type DB interface { Admin Application Basic + Conversation Domain Emoji HeaderFilter diff --git a/internal/gtsmodel/conversation.go b/internal/gtsmodel/conversation.go new file mode 100644 index 0000000000..fe81a68865 --- /dev/null +++ b/internal/gtsmodel/conversation.go @@ -0,0 +1,58 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package gtsmodel + +import ( + "slices" + "strings" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// Conversation represents direct messages between the owner account and a set of other accounts. +type Conversation struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + AccountID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversation_thread_id_account_id_other_accounts_key_uniq"` // Account that owns the conversation + Account *Account `bun:"-"` // + OtherAccountIDs []string `bun:"other_account_ids,array"` // Other accounts participating in the conversation (doesn't include the owner, may be empty in the case of a DM to yourself) + OtherAccounts []*Account `bun:"-"` // + OtherAccountsKey string `bun:",notnull,unique:conversation_thread_id_account_id_other_accounts_key_uniq"` // Denormalized lookup key derived from unique OtherAccountIDs, sorted and concatenated with commas, may be empty in the case of a DM to yourself + ThreadID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversation_thread_id_account_id_other_accounts_key_uniq"` // Thread that the conversation is part of + LastStatusID string `bun:"type:CHAR(26),nullzero,notnull"` // id of the last status in this conversation + LastStatus *Status `bun:"-"` // + Read *bool `bun:",default:false"` // Has the owner read all statuses in this conversation? +} + +// ConversationOtherAccountsKey creates an OtherAccountsKey from a list of OtherAccountIDs. +func ConversationOtherAccountsKey(otherAccountIDs []string) string { + otherAccountIDs = util.UniqueStrings(otherAccountIDs) + slices.Sort(otherAccountIDs) + return strings.Join(otherAccountIDs, ",") +} + +// ConversationToStatus is an intermediate struct to facilitate the many2many relationship between a conversation and its statuses, +// including but not limited to the last status. These are used only when deleting a status from a conversation. +type ConversationToStatus struct { + ConversationID string `bun:"type:CHAR(26),unique:conversation_to_statuses_conversation_id_status_id_uniq,nullzero,notnull"` + Conversation *Conversation `bun:"rel:belongs-to"` + StatusID string `bun:"type:CHAR(26),unique:conversation_to_statuses_conversation_id_status_id_uniq,nullzero,notnull"` + Status *Status `bun:"rel:belongs-to"` +} diff --git a/internal/gtsmodel/emoji.go b/internal/gtsmodel/emoji.go index c80e98ecb1..bb868d0f42 100644 --- a/internal/gtsmodel/emoji.go +++ b/internal/gtsmodel/emoji.go @@ -50,3 +50,8 @@ type Emoji struct { func (e *Emoji) IsLocal() bool { return e.Domain == "" } + +// ShortcodeDomain returns the [shortcode]@[domain] for the emoji. +func (e *Emoji) ShortcodeDomain() string { + return e.Shortcode + "@" + e.Domain +} diff --git a/internal/media/refetch.go b/internal/media/refetch.go index c239655d2e..d02f148728 100644 --- a/internal/media/refetch.go +++ b/internal/media/refetch.go @@ -27,7 +27,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/util" ) type DereferenceMedia func(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error) @@ -68,7 +67,7 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM if refetch, err := m.emojiRequiresRefetch(ctx, emoji); err != nil { // an error here indicates something is wrong with storage, so we should stop - return 0, fmt.Errorf("error checking refetch requirement for emoji %s: %w", util.ShortcodeDomain(emoji), err) + return 0, fmt.Errorf("error checking refetch requirement for emoji %s: %w", emoji.ShortcodeDomain(), err) } else if !refetch { continue } @@ -77,7 +76,7 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM } // Update next maxShortcodeDomain from last emoji - maxShortcodeDomain = util.ShortcodeDomain(emojis[len(emojis)-1]) + maxShortcodeDomain = emojis[len(emojis)-1].ShortcodeDomain() } // bail early if we've got nothing to do @@ -95,7 +94,7 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM // this shouldn't happen--since we know we have the emoji--so return if it does return 0, fmt.Errorf("error getting emoji %s: %w", emojiID, err) } - shortcodeDomain := util.ShortcodeDomain(emoji) + shortcodeDomain := emoji.ShortcodeDomain() if emoji.ImageRemoteURL == "" { log.Errorf(ctx, "remote emoji %s could not be refreshed because it has no ImageRemoteURL set", shortcodeDomain) diff --git a/internal/processing/account/delete.go b/internal/processing/account/delete.go index 075e945440..64ce3ee796 100644 --- a/internal/processing/account/delete.go +++ b/internal/processing/account/delete.go @@ -460,6 +460,13 @@ func (p *Processor) deleteAccountPeripheral(ctx context.Context, account *gtsmod // TODO: add status mutes here when they're implemented. + // Delete all conversations owned by given account. + // Accounts in which the conversation + if err := p.state.DB.DeleteConversationsByOwnerAccountID(ctx, account.ID); // nocollapse + err != nil && !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("error deleting conversations owned by account: %w", err) + } + // Delete all poll votes owned by given account. if err := p.state.DB.DeletePollVotesByAccountID(ctx, account.ID); // nocollapse err != nil && !errors.Is(err, db.ErrNoEntries) { diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index 4d1b464d37..c023fabd83 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -112,9 +112,9 @@ func (p *Processor) EmojisGet( Items: items, Path: "api/v1/admin/custom_emojis", NextMaxIDKey: "max_shortcode_domain", - NextMaxIDValue: util.ShortcodeDomain(emojis[count-1]), + NextMaxIDValue: emojis[count-1].ShortcodeDomain(), PrevMinIDKey: "min_shortcode_domain", - PrevMinIDValue: util.ShortcodeDomain(emojis[0]), + PrevMinIDValue: emojis[0].ShortcodeDomain(), Limit: limit, ExtraQueryParams: []string{ emojisGetFilterParams( diff --git a/internal/processing/conversations/conversations.go b/internal/processing/conversations/conversations.go new file mode 100644 index 0000000000..f4282d618b --- /dev/null +++ b/internal/processing/conversations/conversations.go @@ -0,0 +1,35 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +type Processor struct { + state *state.State + converter *typeutils.Converter +} + +func New(state *state.State, converter *typeutils.Converter) Processor { + return Processor{ + state: state, + converter: converter, + } +} diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go new file mode 100644 index 0000000000..f05b01200a --- /dev/null +++ b/internal/processing/conversations/conversations_test.go @@ -0,0 +1,184 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/email" + "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/processing/conversations" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/util" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type ConversationsTestSuite struct { + // standard suite interfaces + suite.Suite + db db.DB + tc *typeutils.Converter + storage *storage.Driver + state state.State + mediaManager *media.Manager + transportController transport.Controller + federator *federation.Federator + emailSender email.Sender + sentEmails map[string]string + + // standard suite models + testTokens map[string]*gtsmodel.Token + testClients map[string]*gtsmodel.Client + testApplications map[string]*gtsmodel.Application + testUsers map[string]*gtsmodel.User + testAccounts map[string]*gtsmodel.Account + testFollows map[string]*gtsmodel.Follow + testAttachments map[string]*gtsmodel.MediaAttachment + testStatuses map[string]*gtsmodel.Status + + // module being tested + conversationsProcessor conversations.Processor + + // conversation created for test + testAccount *gtsmodel.Account + testConversation *gtsmodel.Conversation +} + +func (suite *ConversationsTestSuite) getClientMsg(timeout time.Duration) (*messages.FromClientAPI, bool) { + ctx := context.Background() + ctx, cncl := context.WithTimeout(ctx, timeout) + defer cncl() + return suite.state.Workers.Client.Queue.PopCtx(ctx) +} + +func (suite *ConversationsTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testFollows = testrig.NewTestFollows() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *ConversationsTestSuite) SetupTest() { + suite.state.Caches.Init() + testrig.StartNoopWorkers(&suite.state) + + testrig.InitTestConfig() + testrig.InitTestLog() + + suite.db = testrig.NewTestDB(&suite.state) + suite.state.DB = suite.db + suite.tc = typeutils.NewConverter(&suite.state) + + testrig.StartTimelines( + &suite.state, + visibility.NewFilter(&suite.state), + suite.tc, + ) + + suite.storage = testrig.NewInMemoryStorage() + suite.state.Storage = suite.storage + suite.mediaManager = testrig.NewTestMediaManager(&suite.state) + + suite.transportController = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")) + suite.federator = testrig.NewTestFederator(&suite.state, suite.transportController, suite.mediaManager) + suite.sentEmails = make(map[string]string) + suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) + + suite.conversationsProcessor = conversations.New(&suite.state, suite.tc) + testrig.StandardDBSetup(suite.db, nil) + testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") + + suite.testAccount = suite.testAccounts["local_account_1"] + suite.testConversation = suite.newTestConversation() +} + +func (suite *ConversationsTestSuite) TearDownTest() { + conversationModels := []interface{}{ + (*gtsmodel.Conversation)(nil), + (*gtsmodel.ConversationToStatus)(nil), + } + for _, model := range conversationModels { + if err := suite.db.DropTable(context.Background(), model); err != nil { + log.Error(context.Background(), err) + } + } + + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) + testrig.StopWorkers(&suite.state) +} + +// newTestConversation creates a new status and adds it to a new unread conversation, returning the conversation. +func (suite *ConversationsTestSuite) newTestConversation() *gtsmodel.Conversation { + statusID := id.NewULID() + createdAt := time.Now() + status := >smodel.Status{ + ID: statusID, + CreatedAt: createdAt, + UpdatedAt: createdAt, + URI: "http://localhost:8080/users/" + suite.testAccount.Username + "/statuses/" + statusID, + AccountID: suite.testAccount.ID, + AccountURI: suite.testAccount.URI, + Local: util.Ptr(true), + ThreadID: id.NewULID(), + Visibility: gtsmodel.VisibilityDirect, + ActivityStreamsType: ap.ObjectNote, + Federated: util.Ptr(true), + Boostable: util.Ptr(true), + Replyable: util.Ptr(true), + Likeable: util.Ptr(true), + } + if err := suite.db.PutStatus(context.Background(), status); err != nil { + suite.FailNow(err.Error()) + } + + conversation := >smodel.Conversation{ + ID: id.NewULID(), + AccountID: suite.testAccount.ID, + ThreadID: status.ThreadID, + Read: util.Ptr(false), + } + conversation, err := suite.db.AddStatusToConversation(context.Background(), conversation, status) + if err != nil { + suite.FailNow(err.Error()) + } + + return conversation +} + +func TestConversationsTestSuite(t *testing.T) { + suite.Run(t, new(ConversationsTestSuite)) +} diff --git a/internal/processing/conversations/delete.go b/internal/processing/conversations/delete.go new file mode 100644 index 0000000000..1cab7346df --- /dev/null +++ b/internal/processing/conversations/delete.go @@ -0,0 +1,53 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "context" + "errors" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (p *Processor) Delete( + ctx context.Context, + requestingAccount *gtsmodel.Account, + id string, +) gtserror.WithCode { + // Get the conversation so that we can check its owning account ID. + conversation, err := p.state.DB.GetConversationByID(gtscontext.SetBarebones(ctx), id) + if err != nil { + if errors.Is(err, db.ErrNoEntries) { + return gtserror.NewErrorNotFound(err) + } + return gtserror.NewErrorInternalError(err) + } + if conversation.AccountID != requestingAccount.ID { + return gtserror.NewErrorNotFound(nil) + } + + // Delete the conversation. + if err := p.state.DB.DeleteConversationByID(ctx, id); err != nil { + return gtserror.NewErrorInternalError(err) + } + + return nil +} diff --git a/internal/util/emoji.go b/internal/processing/conversations/delete_test.go similarity index 75% rename from internal/util/emoji.go rename to internal/processing/conversations/delete_test.go index 7144da1a49..17eb26da13 100644 --- a/internal/util/emoji.go +++ b/internal/processing/conversations/delete_test.go @@ -15,11 +15,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package util +package conversations_test -import "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +import "context" -// ShortcodeDomain returns the [shortcode]@[domain] for the given emoji. -func ShortcodeDomain(emoji *gtsmodel.Emoji) string { - return emoji.Shortcode + "@" + emoji.Domain +func (suite *ConversationsTestSuite) TestDelete() { + err := suite.conversationsProcessor.Delete(context.Background(), suite.testAccount, suite.testConversation.ID) + suite.NoError(err) } diff --git a/internal/processing/conversations/get.go b/internal/processing/conversations/get.go new file mode 100644 index 0000000000..c040c3f3b7 --- /dev/null +++ b/internal/processing/conversations/get.go @@ -0,0 +1,104 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "context" + "errors" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/paging" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// GetAll returns conversations owned by the given account. +// The additional parameters can be used for paging. +func (p *Processor) GetAll( + ctx context.Context, + requestingAccount *gtsmodel.Account, + page *paging.Page, +) (*apimodel.PageableResponse, gtserror.WithCode) { + conversations, err := p.state.DB.GetConversationsByOwnerAccountID( + ctx, + requestingAccount.ID, + page, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, gtserror.NewErrorInternalError(err) + } + + // Check for empty response. + count := len(conversations) + if len(conversations) == 0 { + return util.EmptyPageableResponse(), nil + } + + // Get the lowest and highest ID values, used for paging. + lo := conversations[count-1].ID + hi := conversations[0].ID + + items := make([]interface{}, 0, count) + + filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + + mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), requestingAccount.ID, nil) + if err != nil { + err = gtserror.Newf("couldn't retrieve mutes for account %s: %w", requestingAccount.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + compiledMutes := usermute.NewCompiledUserMuteList(mutes) + + for _, conversation := range conversations { + // Convert conversation to frontend API model. + apiConversation, err := p.converter.ConversationToAPIConversation( + ctx, + conversation, + requestingAccount, + filters, + compiledMutes, + ) + if err != nil { + err = gtserror.Newf( + "couldn't convert conversation %s to API representation for account %s: %w", + conversation.ID, + requestingAccount.ID, + err, + ) + return nil, gtserror.NewErrorInternalError(err) + } + + // Append conversation to return items. + items = append(items, apiConversation) + } + + return paging.PackageResponse(paging.ResponseParams{ + Items: items, + Path: "/api/v1/conversations", + Next: page.Next(lo, hi), + Prev: page.Prev(lo, hi), + }), nil +} diff --git a/internal/processing/conversations/get_test.go b/internal/processing/conversations/get_test.go new file mode 100644 index 0000000000..6aca956484 --- /dev/null +++ b/internal/processing/conversations/get_test.go @@ -0,0 +1,33 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations_test + +import ( + "context" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" +) + +func (suite *ConversationsTestSuite) TestGetAll() { + resp, err := suite.conversationsProcessor.GetAll(context.Background(), suite.testAccount, nil) + if suite.NoError(err) && suite.Len(resp.Items, 1) && suite.IsType((*apimodel.Conversation)(nil), resp.Items[0]) { + apiConversation := resp.Items[0].(*apimodel.Conversation) + suite.Equal(suite.testConversation.ID, apiConversation.ID) + suite.True(apiConversation.Unread) + } +} diff --git a/internal/processing/conversations/read.go b/internal/processing/conversations/read.go new file mode 100644 index 0000000000..183d8440ab --- /dev/null +++ b/internal/processing/conversations/read.go @@ -0,0 +1,84 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "context" + "errors" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +func (p *Processor) Read( + ctx context.Context, + requestingAccount *gtsmodel.Account, + id string, +) (*apimodel.Conversation, gtserror.WithCode) { + // Get the conversation, including participating accounts and last status. + conversation, err := p.state.DB.GetConversationByID(ctx, id) + if err != nil { + if errors.Is(err, db.ErrNoEntries) { + return nil, gtserror.NewErrorNotFound(err) + } + err = gtserror.Newf("db error getting conversation %s: %w", id, err) + return nil, gtserror.NewErrorInternalError(err) + } + if conversation.AccountID != requestingAccount.ID { + return nil, gtserror.NewErrorNotFound(nil) + } + + // Mark the conversation as read. + conversation.Read = util.Ptr(true) + if err := p.state.DB.UpdateConversation(ctx, conversation, "read"); err != nil { + err = gtserror.Newf("db error updating conversation %s: %w", id, err) + return nil, gtserror.NewErrorInternalError(err) + } + + filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + + mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), requestingAccount.ID, nil) + if err != nil { + err = gtserror.Newf("couldn't retrieve mutes for account %s: %w", requestingAccount.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + compiledMutes := usermute.NewCompiledUserMuteList(mutes) + + apiConversation, err := p.converter.ConversationToAPIConversation( + ctx, + conversation, + requestingAccount, + filters, + compiledMutes, + ) + if err != nil { + err = gtserror.Newf("db error converting conversation %s to API representation: %w", id, err) + return nil, gtserror.NewErrorInternalError(err) + } + + return apiConversation, nil +} diff --git a/internal/processing/conversations/read_test.go b/internal/processing/conversations/read_test.go new file mode 100644 index 0000000000..d30e5b45ec --- /dev/null +++ b/internal/processing/conversations/read_test.go @@ -0,0 +1,32 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations_test + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +func (suite *ConversationsTestSuite) TestRead() { + suite.False(util.PtrValueOr(suite.testConversation.Read, false)) + conversation, err := suite.conversationsProcessor.Read(context.Background(), suite.testAccount, suite.testConversation.ID) + if suite.NoError(err) { + suite.False(conversation.Unread) + } +} diff --git a/internal/processing/processor.go b/internal/processing/processor.go index fb6b05d80f..e73d19eec7 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -28,6 +28,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/processing/account" "github.com/superseriousbusiness/gotosocial/internal/processing/admin" "github.com/superseriousbusiness/gotosocial/internal/processing/common" + "github.com/superseriousbusiness/gotosocial/internal/processing/conversations" "github.com/superseriousbusiness/gotosocial/internal/processing/fedi" filtersv1 "github.com/superseriousbusiness/gotosocial/internal/processing/filters/v1" filtersv2 "github.com/superseriousbusiness/gotosocial/internal/processing/filters/v2" @@ -70,22 +71,23 @@ type Processor struct { SUB-PROCESSORS */ - account account.Processor - admin admin.Processor - fedi fedi.Processor - filtersv1 filtersv1.Processor - filtersv2 filtersv2.Processor - list list.Processor - markers markers.Processor - media media.Processor - polls polls.Processor - report report.Processor - search search.Processor - status status.Processor - stream stream.Processor - timeline timeline.Processor - user user.Processor - workers workers.Processor + account account.Processor + admin admin.Processor + conversations conversations.Processor + fedi fedi.Processor + filtersv1 filtersv1.Processor + filtersv2 filtersv2.Processor + list list.Processor + markers markers.Processor + media media.Processor + polls polls.Processor + report report.Processor + search search.Processor + status status.Processor + stream stream.Processor + timeline timeline.Processor + user user.Processor + workers workers.Processor } func (p *Processor) Account() *account.Processor { @@ -96,6 +98,10 @@ func (p *Processor) Admin() *admin.Processor { return &p.admin } +func (p *Processor) Conversations() *conversations.Processor { + return &p.conversations +} + func (p *Processor) Fedi() *fedi.Processor { return &p.fedi } @@ -188,6 +194,7 @@ func NewProcessor( // processors + pin them to this struct. processor.account = account.New(&common, state, converter, mediaManager, federator, filter, parseMentionFunc) processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender) + processor.conversations = conversations.New(state, converter) processor.fedi = fedi.New(state, &common, converter, federator, filter) processor.filtersv1 = filtersv1.New(state, converter, &processor.stream) processor.filtersv2 = filtersv2.New(state, converter, &processor.stream) diff --git a/internal/processing/stream/conversation.go b/internal/processing/stream/conversation.go new file mode 100644 index 0000000000..e08ae449a7 --- /dev/null +++ b/internal/processing/stream/conversation.go @@ -0,0 +1,45 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package stream + +import ( + "context" + "encoding/json" + + "codeberg.org/gruf/go-byteutil" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/stream" +) + +// Conversation streams the given conversation to any open, appropriate streams belonging to the given account. +func (p *Processor) Conversation(ctx context.Context, account *gtsmodel.Account, conversation *apimodel.Conversation) { + b, err := json.Marshal(conversation) + if err != nil { + log.Errorf(ctx, "error marshaling json: %v", err) + return + } + p.streams.Post(ctx, account.ID, stream.Message{ + Payload: byteutil.B2S(b), + Event: stream.EventTypeConversation, + Stream: []string{ + stream.TimelineDirect, + }, + }) +} diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go index d5d4265e12..9d1a627e67 100644 --- a/internal/processing/workers/fromclientapi.go +++ b/internal/processing/workers/fromclientapi.go @@ -245,6 +245,10 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA log.Errorf(ctx, "error timelining and notifying status: %v", err) } + if err := p.surface.UpdateConversationsForStatus(ctx, status, true); err != nil { + log.Errorf(ctx, "error adding status to conversations: %v", err) + } + if status.InReplyToID != "" { // Interaction counts changed on the replied status; // uncache the prepared version from all timelines. diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go index 15be23baf9..b6839955f1 100644 --- a/internal/processing/workers/fromclientapi_test.go +++ b/internal/processing/workers/fromclientapi_test.go @@ -50,6 +50,8 @@ func (suite *FromClientAPITestSuite) newStatus( visibility gtsmodel.Visibility, replyToStatus *gtsmodel.Status, boostOfStatus *gtsmodel.Status, + mentionedAccounts []*gtsmodel.Account, + createThread bool, ) *gtsmodel.Status { var ( protocol = config.GetProtocol() @@ -105,6 +107,39 @@ func (suite *FromClientAPITestSuite) newStatus( newStatus.Visibility = boostOfStatus.Visibility } + for _, mentionedAccount := range mentionedAccounts { + newMention := >smodel.Mention{ + ID: id.NewULID(), + StatusID: newStatus.ID, + Status: newStatus, + OriginAccountID: account.ID, + OriginAccountURI: account.URI, + OriginAccount: account, + TargetAccountID: mentionedAccount.ID, + TargetAccount: mentionedAccount, + Silent: util.Ptr(false), + } + + newStatus.Mentions = append(newStatus.Mentions, newMention) + newStatus.MentionIDs = append(newStatus.MentionIDs, newMention.ID) + + if err := state.DB.PutMention(ctx, newMention); err != nil { + suite.FailNow(err.Error()) + } + } + + if createThread { + newThread := >smodel.Thread{ + ID: id.NewULID(), + } + + newStatus.ThreadID = newThread.ID + + if err := state.DB.PutThread(ctx, newThread); err != nil { + suite.FailNow(err.Error()) + } + } + // Put the status in the db, to mimic what would // have already happened earlier up the flow. if err := state.DB.PutStatus(ctx, newStatus); err != nil { @@ -171,6 +206,31 @@ func (suite *FromClientAPITestSuite) statusJSON( return string(statusJSON) } +func (suite *FromClientAPITestSuite) conversationJSON( + ctx context.Context, + typeConverter *typeutils.Converter, + conversation *gtsmodel.Conversation, + requestingAccount *gtsmodel.Account, +) string { + apiConversation, err := typeConverter.ConversationToAPIConversation( + ctx, + conversation, + requestingAccount, + nil, + nil, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + conversationJSON, err := json.Marshal(apiConversation) + if err != nil { + suite.FailNow(err.Error()) + } + + return string(conversationJSON) +} + func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() { testStructs := suite.SetupTestStructs() defer suite.TearDownTestStructs(testStructs) @@ -197,6 +257,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() { gtsmodel.VisibilityPublic, nil, nil, + nil, + false, ) ) @@ -306,6 +368,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusReply() { gtsmodel.VisibilityPublic, suite.testStatuses["local_account_2_status_1"], nil, + nil, + false, ) ) @@ -365,6 +429,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusReplyMuted() { gtsmodel.VisibilityPublic, suite.testStatuses["local_account_1_status_1"], nil, + nil, + false, ) threadMute = >smodel.ThreadMute{ ID: "01HD3KRMBB1M85QRWHD912QWRE", @@ -423,6 +489,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusBoostMuted() { gtsmodel.VisibilityPublic, nil, suite.testStatuses["local_account_1_status_1"], + nil, + false, ) threadMute = >smodel.ThreadMute{ ID: "01HD3KRMBB1M85QRWHD912QWRE", @@ -486,6 +554,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusListRepliesPolicyLis gtsmodel.VisibilityPublic, suite.testStatuses["local_account_2_status_1"], nil, + nil, + false, ) ) @@ -559,6 +629,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusListRepliesPolicyLis gtsmodel.VisibilityPublic, suite.testStatuses["local_account_2_status_1"], nil, + nil, + false, ) ) @@ -637,6 +709,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusReplyListRepliesPoli gtsmodel.VisibilityPublic, suite.testStatuses["local_account_2_status_1"], nil, + nil, + false, ) ) @@ -707,6 +781,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusBoost() { gtsmodel.VisibilityPublic, nil, suite.testStatuses["local_account_2_status_1"], + nil, + false, ) ) @@ -768,6 +844,8 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusBoostNoReblogs() { gtsmodel.VisibilityPublic, nil, suite.testStatuses["local_account_2_status_1"], + nil, + false, ) ) @@ -810,6 +888,90 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusBoostNoReblogs() { ) } +func (suite *FromClientAPITestSuite) TestProcessCreateStatusWhichBeginsConversation() { + testStructs := suite.SetupTestStructs() + defer suite.TearDownTestStructs(testStructs) + + var ( + ctx = context.Background() + postingAccount = suite.testAccounts["local_account_2"] + receivingAccount = suite.testAccounts["local_account_1"] + streams = suite.openStreams(ctx, + testStructs.Processor, + receivingAccount, + nil, + ) + homeStream = streams[stream.TimelineHome] + directStream = streams[stream.TimelineDirect] + + // turtle posts a new top-level DM mentioning zork. + status = suite.newStatus( + ctx, + testStructs.State, + postingAccount, + gtsmodel.VisibilityDirect, + nil, + nil, + []*gtsmodel.Account{receivingAccount}, + true, + ) + ) + + // Process the new status. + if err := testStructs.Processor.Workers().ProcessFromClientAPI( + ctx, + &messages.FromClientAPI{ + APObjectType: ap.ObjectNote, + APActivityType: ap.ActivityCreate, + GTSModel: status, + Origin: postingAccount, + }, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Locate the conversation which should now exist for zork. + conversation, err := testStructs.State.DB.GetConversationByThreadAndAccountIDs( + ctx, + status.ThreadID, + receivingAccount.ID, + []string{postingAccount.ID}, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Check status in home stream. + suite.checkStreamed( + homeStream, + true, + "", + stream.EventTypeUpdate, + ) + + // Check mention notification in home stream. + suite.checkStreamed( + homeStream, + true, + "", + stream.EventTypeNotification, + ) + + // Check conversation in direct stream. + conversationJSON := suite.conversationJSON( + ctx, + testStructs.TypeConverter, + conversation, + receivingAccount, + ) + suite.checkStreamed( + directStream, + true, + conversationJSON, + stream.EventTypeConversation, + ) +} + func (suite *FromClientAPITestSuite) TestProcessStatusDelete() { testStructs := suite.SetupTestStructs() defer suite.TearDownTestStructs(testStructs) diff --git a/internal/processing/workers/fromfediapi.go b/internal/processing/workers/fromfediapi.go index ac4003f6a4..506820552d 100644 --- a/internal/processing/workers/fromfediapi.go +++ b/internal/processing/workers/fromfediapi.go @@ -232,6 +232,10 @@ func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI) log.Errorf(ctx, "error timelining and notifying status: %v", err) } + if err := p.surface.UpdateConversationsForStatus(ctx, status, true); err != nil { + log.Errorf(ctx, "error adding status to conversations: %v", err) + } + return nil } diff --git a/internal/processing/workers/surfaceconversations.go b/internal/processing/workers/surfaceconversations.go new file mode 100644 index 0000000000..56fe81767f --- /dev/null +++ b/internal/processing/workers/surfaceconversations.go @@ -0,0 +1,203 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package workers + +import ( + "context" + "errors" + + "github.com/superseriousbusiness/gotosocial/internal/db" + statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" + "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// UpdateConversationsForStatus updates conversations that include this status, +// and sends conversation stream events if requested. +func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsmodel.Status, notify bool) error { + if status.Visibility != gtsmodel.VisibilityDirect { + // Only DMs are considered part of conversations. + return nil + } + if status.BoostOfID != "" { + // Boosts can't be part of conversations. + // FUTURE: This may change if we ever implement quote posts. + return nil + } + if status.ThreadID == "" { + // If the status doesn't have a thread ID, it didn't mention a local account, + // and thus can't be part of a conversation. + return nil + } + + // The account which authored the status plus all mentioned accounts. + allParticipantsSet := make(map[string]*gtsmodel.Account, 1+len(status.Mentions)) + allParticipantsSet[status.AccountID] = status.Account + for _, mention := range status.Mentions { + allParticipantsSet[mention.TargetAccountID] = mention.TargetAccount + } + + // Create or update conversations for and send notifications to each local participant. + for _, participant := range allParticipantsSet { + if participant.IsRemote() { + continue + } + localAccount := participant + + // If the status is not visible to this account, skip processing it for this account. + visible, err := s.Filter.StatusVisible(ctx, localAccount, status) + if err != nil { + log.Errorf( + ctx, + "error checking status %s visibility for account %s: %v", + status.ID, + localAccount.ID, + err, + ) + continue + } else if !visible { + continue + } + + // TODO: (Vyr) find a prettier way to do this + // Is the status filtered or muted for this user? + filters, err := s.State.DB.GetFiltersForAccountID(ctx, localAccount.ID) + if err != nil { + log.Errorf(ctx, "error retrieving filters for account %s: %v", localAccount.ID, err) + continue + } + mutes, err := s.State.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), localAccount.ID, nil) + if err != nil { + log.Errorf(ctx, "error retrieving mutes for account %s: %v", localAccount.ID, err) + continue + } + compiledMutes := usermute.NewCompiledUserMuteList(mutes) + // Converting the status to an API status runs the filter/mute checks. + _, err = s.Converter.StatusToAPIStatus( + ctx, + status, + localAccount, + statusfilter.FilterContextNotifications, + filters, + compiledMutes, + ) + if err != nil { + // If the status matched a hide filter, skip processing it for this account. + // If there was another kind of error, log that and skip it anyway. + if !errors.Is(err, statusfilter.ErrHideStatus) { + log.Errorf( + ctx, + "error checking status %s filtering/muting for account %s: %v", + status.ID, + localAccount.ID, + err, + ) + } + continue + } + + // Collect other accounts participating in the conversation. + otherAccounts := make([]*gtsmodel.Account, 0, len(allParticipantsSet)-1) + otherAccountIDs := make([]string, 0, len(allParticipantsSet)-1) + for accountID, account := range allParticipantsSet { + if accountID != localAccount.ID { + otherAccounts = append(otherAccounts, account) + otherAccountIDs = append(otherAccountIDs, accountID) + } + } + + // Check for a previously existing conversation, if there is one. + conversation, err := s.State.DB.GetConversationByThreadAndAccountIDs( + ctx, + status.ThreadID, + localAccount.ID, + otherAccountIDs, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + log.Errorf( + ctx, + "error trying to find a previous conversation for status %s and account %s: %v", + status.ID, + localAccount.ID, + err, + ) + continue + } + + if conversation == nil { + // Create a new conversation. + conversation = >smodel.Conversation{ + ID: id.NewULID(), + AccountID: localAccount.ID, + OtherAccountIDs: otherAccountIDs, + OtherAccounts: otherAccounts, + OtherAccountsKey: gtsmodel.ConversationOtherAccountsKey(otherAccountIDs), + ThreadID: status.ThreadID, + Read: util.Ptr(true), + } + } + + // Create or update the conversation. + conversation, err = s.State.DB.AddStatusToConversation(ctx, conversation, status) + if err != nil { + log.Errorf( + ctx, + "error creating or updating conversation %s for status %s and account %s: %v", + conversation.ID, + status.ID, + localAccount.ID, + err, + ) + continue + } + + // Convert the conversation to API representation. + apiConversation, err := s.Converter.ConversationToAPIConversation( + ctx, + conversation, + localAccount, + filters, + compiledMutes, + ) + if err != nil { + // If the conversation's last status matched a hide filter, skip it. + // If there was another kind of error, log that and skip it anyway. + if !errors.Is(err, statusfilter.ErrHideStatus) { + log.Errorf( + ctx, + "error converting conversation %s to API representation for account %s: %v", + status.ID, + localAccount.ID, + err, + ) + } + continue + } + + if notify { + // Send a conversation notification. + s.Stream.Conversation(ctx, localAccount, apiConversation) + } + } + + return nil +} diff --git a/internal/processing/workers/util.go b/internal/processing/workers/util.go index 780e5ca14d..9153709766 100644 --- a/internal/processing/workers/util.go +++ b/internal/processing/workers/util.go @@ -137,6 +137,11 @@ func (u *utils) wipeStatus( errs.Appendf("error deleting status from timelines: %w", err) } + // delete this status from any conversations that it's part of + if err := u.state.DB.DeleteStatusFromConversations(ctx, statusToDelete.ID); err != nil { + errs.Appendf("error deleting status from conversations: %w", err) + } + // finally, delete the status itself if err := u.state.DB.DeleteStatusByID(ctx, statusToDelete.ID); err != nil { errs.Appendf("error deleting status: %w", err) diff --git a/internal/processing/workers/workers_test.go b/internal/processing/workers/workers_test.go index f66190d750..3093fd93ab 100644 --- a/internal/processing/workers/workers_test.go +++ b/internal/processing/workers/workers_test.go @@ -108,6 +108,7 @@ func (suite *WorkersTestSuite) openStreams(ctx context.Context, processor *proce stream.TimelineHome, stream.TimelinePublic, stream.TimelineNotifications, + stream.TimelineDirect, } { stream, err := processor.Stream().Open(ctx, account, streamType) if err != nil { diff --git a/internal/stream/stream.go b/internal/stream/stream.go index e843a1b760..0a352133af 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -46,6 +46,10 @@ const ( // EventTypeFiltersChanged -- the user's filters // (including keywords and statuses) have changed. EventTypeFiltersChanged = "filters_changed" + + // EventTypeConversation -- a user + // should be shown an updated conversation. + EventTypeConversation = "conversation" ) const ( diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index a8f9b7f8fe..a69d7aef21 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -1618,6 +1618,56 @@ func (c *Converter) NotificationToAPINotification( }, nil } +// ConversationToAPIConversation converts a conversation into its API representation. +// The conversation status will be filtered using the notification filter context, +// and may be nil if the status was hidden. +func (c *Converter) ConversationToAPIConversation( + ctx context.Context, + conversation *gtsmodel.Conversation, + requestingAccount *gtsmodel.Account, + filters []*gtsmodel.Filter, + mutes *usermute.CompiledUserMuteList, +) (*apimodel.Conversation, error) { + apiConversation := &apimodel.Conversation{ + ID: conversation.ID, + Unread: !*conversation.Read, + } + for _, account := range conversation.OtherAccounts { + var apiAccount *apimodel.Account + // TODO: (Vyr) good enough? is there something reusable for this? should we pull in a Filters dependency? + // TODO: (Vyr) duplicated in conversationsget.go + blocked, err := c.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, account.ID) + if err != nil { + return nil, err + } + if blocked || account.IsSuspended() { + apiAccount, err = c.AccountToAPIAccountBlocked(ctx, account) + } else { + apiAccount, err = c.AccountToAPIAccountPublic(ctx, account) + } + if err != nil { + return nil, err + } + apiConversation.Accounts = append(apiConversation.Accounts, *apiAccount) + } + if conversation.LastStatus != nil { + var err error + apiConversation.LastStatus, err = c.StatusToAPIStatus( + ctx, + conversation.LastStatus, + requestingAccount, + statusfilter.FilterContextNotifications, + filters, + mutes, + ) + if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + return nil, err + } + } + + return apiConversation, nil +} + // DomainPermToAPIDomainPerm converts a gts model domin block or allow into an api domain permission. func (c *Converter) DomainPermToAPIDomainPerm( ctx context.Context, diff --git a/test/envparsing.sh b/test/envparsing.sh index a744717a41..30881162f5 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -32,6 +32,8 @@ EXPECT=$(cat << "EOF" "block-mem-ratio": 2, "boost-of-ids-mem-ratio": 3, "client-mem-ratio": 0.1, + "conversation-ids-mem-ratio": 3, + "conversation-mem-ratio": 2, "emoji-category-mem-ratio": 0.1, "emoji-mem-ratio": 3, "filter-keyword-mem-ratio": 0.5, From 31b260dfc9f0ffa90c808cc8bb5f948b00829145 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 2 Jul 2024 11:12:54 -0700 Subject: [PATCH 02/35] Sort and page conversations by last status ID --- docs/api/swagger.yaml | 72 +++++++++++++- .../client/conversations/conversationsget.go | 12 +-- internal/cache/cache.go | 2 +- internal/cache/db.go | 11 ++- internal/cache/invalidate.go | 4 +- internal/cache/wrappers.go | 28 ++++++ internal/config/config.go | 96 +++++++++---------- internal/config/defaults.go | 94 +++++++++--------- internal/config/helpers.gen.go | 30 +++--- internal/db/bundb/conversation.go | 46 +++++---- .../20240611190733_add_conversations.go | 14 ++- internal/db/conversation.go | 3 +- internal/gtsmodel/conversation.go | 24 ++--- .../conversations/conversations_test.go | 21 +++- internal/processing/conversations/get.go | 6 +- internal/processing/conversations/get_test.go | 29 ++++++ test/envparsing.sh | 2 +- 17 files changed, 326 insertions(+), 168 deletions(-) diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index 367dae72fb..5a020dde43 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -6089,11 +6089,43 @@ paths: - read:bookmarks tags: - bookmarks + /api/v1/conversation/{id}/read: + post: + operationId: conversationRead + parameters: + - description: ID of the conversation. + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: Updated conversation. + schema: + $ref: '#/definitions/conversation' + "400": + description: bad request + "401": + description: unauthorized + "404": + description: not found + "406": + description: not acceptable + "422": + description: unprocessable content + "500": + description: internal server error + security: + - OAuth2 Bearer: + - write:conversations + summary: Mark a conversation with the given ID as read. + tags: + - conversations /api/v1/conversations: get: description: |- - NOT IMPLEMENTED YET: Will currently always return an array of length 0. - The next and previous queries can be parsed from the returned Link header. Example: @@ -6102,15 +6134,15 @@ paths: ```` operationId: conversationsGet parameters: - - description: 'Return only conversations *OLDER* than the given max ID. The conversation with the specified ID will not be included in the response. NOTE: the ID is of the internal conversation, use the Link header for pagination.' + - description: 'Return only conversations with last statuses *OLDER* than the given max ID. The conversation with the specified ID will not be included in the response. NOTE: The ID is a status ID. Use the Link header for pagination.' in: query name: max_id type: string - - description: 'Return only conversations *NEWER* than the given since ID. The conversation with the specified ID will not be included in the response. NOTE: the ID is of the internal conversation, use the Link header for pagination.' + - description: 'Return only conversations with last statuses *NEWER* than the given since ID. The conversation with the specified ID will not be included in the response. NOTE: The ID is a status ID. Use the Link header for pagination.' in: query name: since_id type: string - - description: 'Return only conversations *IMMEDIATELY NEWER* than the given min ID. The conversation with the specified ID will not be included in the response. NOTE: the ID is of the internal conversation, use the Link header for pagination.' + - description: 'Return only conversations with last statuses *IMMEDIATELY NEWER* than the given min ID. The conversation with the specified ID will not be included in the response. NOTE: The ID is a status ID. Use the Link header for pagination.' in: query name: min_id type: string @@ -6150,6 +6182,36 @@ paths: summary: Get an array of (direct message) conversations that requesting account is involved in. tags: - conversations + /api/v1/conversations/{id}: + delete: + operationId: conversationDelete + parameters: + - description: ID of the conversation + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: conversation deleted + "400": + description: bad request + "401": + description: unauthorized + "404": + description: not found + "406": + description: not acceptable + "500": + description: internal server error + security: + - OAuth2 Bearer: + - write:conversations + summary: Delete a single conversation with the given ID. + tags: + - conversations /api/v1/custom_emojis: get: operationId: customEmojisGet diff --git a/internal/api/client/conversations/conversationsget.go b/internal/api/client/conversations/conversationsget.go index 28925ae367..663b9a707a 100644 --- a/internal/api/client/conversations/conversationsget.go +++ b/internal/api/client/conversations/conversationsget.go @@ -50,26 +50,26 @@ import ( // name: max_id // type: string // description: >- -// Return only conversations *OLDER* than the given max ID. +// Return only conversations with last statuses *OLDER* than the given max ID. // The conversation with the specified ID will not be included in the response. -// NOTE: the ID is of the internal conversation, use the Link header for pagination. +// NOTE: The ID is a status ID. Use the Link header for pagination. // in: query // required: false // - // name: since_id // type: string // description: >- -// Return only conversations *NEWER* than the given since ID. +// Return only conversations with last statuses *NEWER* than the given since ID. // The conversation with the specified ID will not be included in the response. -// NOTE: the ID is of the internal conversation, use the Link header for pagination. +// NOTE: The ID is a status ID. Use the Link header for pagination. // in: query // - // name: min_id // type: string // description: >- -// Return only conversations *IMMEDIATELY NEWER* than the given min ID. +// Return only conversations with last statuses *IMMEDIATELY NEWER* than the given min ID. // The conversation with the specified ID will not be included in the response. -// NOTE: the ID is of the internal conversation, use the Link header for pagination. +// NOTE: The ID is a status ID. Use the Link header for pagination. // in: query // required: false // - diff --git a/internal/cache/cache.go b/internal/cache/cache.go index b522037960..2eecd31fb0 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -61,7 +61,7 @@ func (c *Caches) Init() { c.initBoostOfIDs() c.initClient() c.initConversation() - c.initConversationIDs() + c.initConversationLastStatusIDs() c.initDomainAllow() c.initDomainBlock() c.initEmoji() diff --git a/internal/cache/db.go b/internal/cache/db.go index 569aeb60a0..1038f7145e 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -59,8 +59,8 @@ type GTSCaches struct { // Conversation provides access to the gtsmodel Conversation database cache. Conversation StructCache[*gtsmodel.Conversation] - // ConversationIDs provides access to the conversation IDs database cache. - ConversationIDs SliceCache[string] + // ConversationLastStatusIDs provides access to the conversation last status IDs database cache. + ConversationLastStatusIDs SliceCache[string] // DomainAllow provides access to the domain allow database cache. DomainAllow *domain.Cache @@ -455,6 +455,7 @@ func (c *Caches) initConversation() { Indices: []structr.IndexConfig{ {Fields: "ID"}, {Fields: "ThreadID,AccountID,OtherAccountsKey"}, + {Fields: "AccountID,LastStatusID"}, {Fields: "AccountID", Multiple: true}, }, MaxSize: cap, @@ -464,14 +465,14 @@ func (c *Caches) initConversation() { }) } -func (c *Caches) initConversationIDs() { +func (c *Caches) initConversationLastStatusIDs() { cap := calculateSliceCacheMax( - config.GetCacheConversationIDsMemRatio(), + config.GetCacheConversationLastStatusIDsMemRatio(), ) log.Infof(nil, "cache size = %d", cap) - c.GTS.ConversationIDs.Init(0, cap) + c.GTS.ConversationLastStatusIDs.Init(0, cap) } func (c *Caches) initDomainAllow() { diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 204a2af19f..2922c0c89b 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -84,8 +84,8 @@ func (c *Caches) OnInvalidateClient(client *gtsmodel.Client) { } func (c *Caches) OnInvalidateConversation(conversation *gtsmodel.Conversation) { - // Invalidate source account's conversation lists. - c.GTS.ConversationIDs.Invalidate(conversation.AccountID) + // Invalidate source account's conversation list. + c.GTS.ConversationLastStatusIDs.Invalidate(conversation.AccountID) } func (c *Caches) OnInvalidateEmojiCategory(category *gtsmodel.EmojiCategory) { diff --git a/internal/cache/wrappers.go b/internal/cache/wrappers.go index edeea9bcde..9cb4fca98d 100644 --- a/internal/cache/wrappers.go +++ b/internal/cache/wrappers.go @@ -158,6 +158,34 @@ func (c *StructCache[T]) LoadIDs(index string, ids []string, load func([]string) }) } +// LoadIDs2Part works as LoadIDs, except using a two-part key, +// where the first part is an ID shared by all the objects, +// and the second part is a list of per-object IDs. +func (c *StructCache[T]) LoadIDs2Part(index string, id1 string, id2s []string, load func(string, []string) ([]T, error)) ([]T, error) { + i := c.index[index] + if i == nil { + // we only perform this check here as + // we're going to use the index before + // passing it to cache in main .Load(). + panic("missing index for cache type") + } + + // Generate cache keys for two-part IDs. + keys := make([]structr.Key, len(id2s)) + for x, id2 := range id2s { + keys[x] = i.Key(id1, id2) + } + + // Pass loader callback with wrapper onto main cache load function. + return c.cache.Load(i, keys, func(uncached []structr.Key) ([]T, error) { + uncachedIDs := make([]string, len(uncached)) + for i := range uncached { + uncachedIDs[i] = uncached[i].Values()[1].(string) + } + return load(id1, uncachedIDs) + }) +} + // Store: see structr.Cache{}.Store(). func (c *StructCache[T]) Store(value T, store func() error) error { return c.cache.Store(value, store) diff --git a/internal/config/config.go b/internal/config/config.go index e4c2e19290..9ff0b0f2ea 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -191,54 +191,54 @@ type HTTPClientConfiguration struct { } type CacheConfiguration struct { - MemoryTarget bytesize.Size `name:"memory-target"` - AccountMemRatio float64 `name:"account-mem-ratio"` - AccountNoteMemRatio float64 `name:"account-note-mem-ratio"` - AccountSettingsMemRatio float64 `name:"account-settings-mem-ratio"` - AccountStatsMemRatio float64 `name:"account-stats-mem-ratio"` - ApplicationMemRatio float64 `name:"application-mem-ratio"` - BlockMemRatio float64 `name:"block-mem-ratio"` - BlockIDsMemRatio float64 `name:"block-ids-mem-ratio"` - BoostOfIDsMemRatio float64 `name:"boost-of-ids-mem-ratio"` - ClientMemRatio float64 `name:"client-mem-ratio"` - ConversationMemRatio float64 `name:"conversation-mem-ratio"` - ConversationIDsMemRatio float64 `name:"conversation-ids-mem-ratio"` - EmojiMemRatio float64 `name:"emoji-mem-ratio"` - EmojiCategoryMemRatio float64 `name:"emoji-category-mem-ratio"` - FilterMemRatio float64 `name:"filter-mem-ratio"` - FilterKeywordMemRatio float64 `name:"filter-keyword-mem-ratio"` - FilterStatusMemRatio float64 `name:"filter-status-mem-ratio"` - FollowMemRatio float64 `name:"follow-mem-ratio"` - FollowIDsMemRatio float64 `name:"follow-ids-mem-ratio"` - FollowRequestMemRatio float64 `name:"follow-request-mem-ratio"` - FollowRequestIDsMemRatio float64 `name:"follow-request-ids-mem-ratio"` - InReplyToIDsMemRatio float64 `name:"in-reply-to-ids-mem-ratio"` - InstanceMemRatio float64 `name:"instance-mem-ratio"` - ListMemRatio float64 `name:"list-mem-ratio"` - ListEntryMemRatio float64 `name:"list-entry-mem-ratio"` - MarkerMemRatio float64 `name:"marker-mem-ratio"` - MediaMemRatio float64 `name:"media-mem-ratio"` - MentionMemRatio float64 `name:"mention-mem-ratio"` - MoveMemRatio float64 `name:"move-mem-ratio"` - NotificationMemRatio float64 `name:"notification-mem-ratio"` - PollMemRatio float64 `name:"poll-mem-ratio"` - PollVoteMemRatio float64 `name:"poll-vote-mem-ratio"` - PollVoteIDsMemRatio float64 `name:"poll-vote-ids-mem-ratio"` - ReportMemRatio float64 `name:"report-mem-ratio"` - StatusMemRatio float64 `name:"status-mem-ratio"` - StatusBookmarkMemRatio float64 `name:"status-bookmark-mem-ratio"` - StatusBookmarkIDsMemRatio float64 `name:"status-bookmark-ids-mem-ratio"` - StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"` - StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"` - TagMemRatio float64 `name:"tag-mem-ratio"` - ThreadMuteMemRatio float64 `name:"thread-mute-mem-ratio"` - TokenMemRatio float64 `name:"token-mem-ratio"` - TombstoneMemRatio float64 `name:"tombstone-mem-ratio"` - UserMemRatio float64 `name:"user-mem-ratio"` - UserMuteMemRatio float64 `name:"user-mute-mem-ratio"` - UserMuteIDsMemRatio float64 `name:"user-mute-ids-mem-ratio"` - WebfingerMemRatio float64 `name:"webfinger-mem-ratio"` - VisibilityMemRatio float64 `name:"visibility-mem-ratio"` + MemoryTarget bytesize.Size `name:"memory-target"` + AccountMemRatio float64 `name:"account-mem-ratio"` + AccountNoteMemRatio float64 `name:"account-note-mem-ratio"` + AccountSettingsMemRatio float64 `name:"account-settings-mem-ratio"` + AccountStatsMemRatio float64 `name:"account-stats-mem-ratio"` + ApplicationMemRatio float64 `name:"application-mem-ratio"` + BlockMemRatio float64 `name:"block-mem-ratio"` + BlockIDsMemRatio float64 `name:"block-ids-mem-ratio"` + BoostOfIDsMemRatio float64 `name:"boost-of-ids-mem-ratio"` + ClientMemRatio float64 `name:"client-mem-ratio"` + ConversationMemRatio float64 `name:"conversation-mem-ratio"` + ConversationLastStatusIDsMemRatio float64 `name:"conversation-last-status-ids-mem-ratio"` + EmojiMemRatio float64 `name:"emoji-mem-ratio"` + EmojiCategoryMemRatio float64 `name:"emoji-category-mem-ratio"` + FilterMemRatio float64 `name:"filter-mem-ratio"` + FilterKeywordMemRatio float64 `name:"filter-keyword-mem-ratio"` + FilterStatusMemRatio float64 `name:"filter-status-mem-ratio"` + FollowMemRatio float64 `name:"follow-mem-ratio"` + FollowIDsMemRatio float64 `name:"follow-ids-mem-ratio"` + FollowRequestMemRatio float64 `name:"follow-request-mem-ratio"` + FollowRequestIDsMemRatio float64 `name:"follow-request-ids-mem-ratio"` + InReplyToIDsMemRatio float64 `name:"in-reply-to-ids-mem-ratio"` + InstanceMemRatio float64 `name:"instance-mem-ratio"` + ListMemRatio float64 `name:"list-mem-ratio"` + ListEntryMemRatio float64 `name:"list-entry-mem-ratio"` + MarkerMemRatio float64 `name:"marker-mem-ratio"` + MediaMemRatio float64 `name:"media-mem-ratio"` + MentionMemRatio float64 `name:"mention-mem-ratio"` + MoveMemRatio float64 `name:"move-mem-ratio"` + NotificationMemRatio float64 `name:"notification-mem-ratio"` + PollMemRatio float64 `name:"poll-mem-ratio"` + PollVoteMemRatio float64 `name:"poll-vote-mem-ratio"` + PollVoteIDsMemRatio float64 `name:"poll-vote-ids-mem-ratio"` + ReportMemRatio float64 `name:"report-mem-ratio"` + StatusMemRatio float64 `name:"status-mem-ratio"` + StatusBookmarkMemRatio float64 `name:"status-bookmark-mem-ratio"` + StatusBookmarkIDsMemRatio float64 `name:"status-bookmark-ids-mem-ratio"` + StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"` + StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"` + TagMemRatio float64 `name:"tag-mem-ratio"` + ThreadMuteMemRatio float64 `name:"thread-mute-mem-ratio"` + TokenMemRatio float64 `name:"token-mem-ratio"` + TombstoneMemRatio float64 `name:"tombstone-mem-ratio"` + UserMemRatio float64 `name:"user-mem-ratio"` + UserMuteMemRatio float64 `name:"user-mute-mem-ratio"` + UserMuteIDsMemRatio float64 `name:"user-mute-ids-mem-ratio"` + WebfingerMemRatio float64 `name:"webfinger-mem-ratio"` + VisibilityMemRatio float64 `name:"visibility-mem-ratio"` } // MarshalMap will marshal current Configuration into a map structure (useful for JSON/TOML/YAML). diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 94301d696a..9f22fbb5bd 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -156,53 +156,53 @@ var Defaults = Configuration{ // when TODO items in the size.go source // file have been addressed, these should // be able to make some more sense :D - AccountMemRatio: 5, - AccountNoteMemRatio: 1, - AccountSettingsMemRatio: 0.1, - AccountStatsMemRatio: 2, - ApplicationMemRatio: 0.1, - BlockMemRatio: 2, - BlockIDsMemRatio: 3, - BoostOfIDsMemRatio: 3, - ClientMemRatio: 0.1, - ConversationMemRatio: 2, - ConversationIDsMemRatio: 3, - EmojiMemRatio: 3, - EmojiCategoryMemRatio: 0.1, - FilterMemRatio: 0.5, - FilterKeywordMemRatio: 0.5, - FilterStatusMemRatio: 0.5, - FollowMemRatio: 2, - FollowIDsMemRatio: 4, - FollowRequestMemRatio: 2, - FollowRequestIDsMemRatio: 2, - InReplyToIDsMemRatio: 3, - InstanceMemRatio: 1, - ListMemRatio: 1, - ListEntryMemRatio: 2, - MarkerMemRatio: 0.5, - MediaMemRatio: 4, - MentionMemRatio: 2, - MoveMemRatio: 0.1, - NotificationMemRatio: 2, - PollMemRatio: 1, - PollVoteMemRatio: 2, - PollVoteIDsMemRatio: 2, - ReportMemRatio: 1, - StatusMemRatio: 5, - StatusBookmarkMemRatio: 0.5, - StatusBookmarkIDsMemRatio: 2, - StatusFaveMemRatio: 2, - StatusFaveIDsMemRatio: 3, - TagMemRatio: 2, - ThreadMuteMemRatio: 0.2, - TokenMemRatio: 0.75, - TombstoneMemRatio: 0.5, - UserMemRatio: 0.25, - UserMuteMemRatio: 2, - UserMuteIDsMemRatio: 3, - WebfingerMemRatio: 0.1, - VisibilityMemRatio: 2, + AccountMemRatio: 5, + AccountNoteMemRatio: 1, + AccountSettingsMemRatio: 0.1, + AccountStatsMemRatio: 2, + ApplicationMemRatio: 0.1, + BlockMemRatio: 2, + BlockIDsMemRatio: 3, + BoostOfIDsMemRatio: 3, + ClientMemRatio: 0.1, + ConversationMemRatio: 2, + ConversationLastStatusIDsMemRatio: 3, + EmojiMemRatio: 3, + EmojiCategoryMemRatio: 0.1, + FilterMemRatio: 0.5, + FilterKeywordMemRatio: 0.5, + FilterStatusMemRatio: 0.5, + FollowMemRatio: 2, + FollowIDsMemRatio: 4, + FollowRequestMemRatio: 2, + FollowRequestIDsMemRatio: 2, + InReplyToIDsMemRatio: 3, + InstanceMemRatio: 1, + ListMemRatio: 1, + ListEntryMemRatio: 2, + MarkerMemRatio: 0.5, + MediaMemRatio: 4, + MentionMemRatio: 2, + MoveMemRatio: 0.1, + NotificationMemRatio: 2, + PollMemRatio: 1, + PollVoteMemRatio: 2, + PollVoteIDsMemRatio: 2, + ReportMemRatio: 1, + StatusMemRatio: 5, + StatusBookmarkMemRatio: 0.5, + StatusBookmarkIDsMemRatio: 2, + StatusFaveMemRatio: 2, + StatusFaveIDsMemRatio: 3, + TagMemRatio: 2, + ThreadMuteMemRatio: 0.2, + TokenMemRatio: 0.75, + TombstoneMemRatio: 0.5, + UserMemRatio: 0.25, + UserMuteMemRatio: 2, + UserMuteIDsMemRatio: 3, + WebfingerMemRatio: 0.1, + VisibilityMemRatio: 2, }, HTTPClient: HTTPClientConfiguration{ diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 27b308f535..45ff28f899 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -3000,30 +3000,36 @@ func GetCacheConversationMemRatio() float64 { return global.GetCacheConversation // SetCacheConversationMemRatio safely sets the value for global configuration 'Cache.ConversationMemRatio' field func SetCacheConversationMemRatio(v float64) { global.SetCacheConversationMemRatio(v) } -// GetCacheConversationIDsMemRatio safely fetches the Configuration value for state's 'Cache.ConversationIDsMemRatio' field -func (st *ConfigState) GetCacheConversationIDsMemRatio() (v float64) { +// GetCacheConversationLastStatusIDsMemRatio safely fetches the Configuration value for state's 'Cache.ConversationLastStatusIDsMemRatio' field +func (st *ConfigState) GetCacheConversationLastStatusIDsMemRatio() (v float64) { st.mutex.RLock() - v = st.config.Cache.ConversationIDsMemRatio + v = st.config.Cache.ConversationLastStatusIDsMemRatio st.mutex.RUnlock() return } -// SetCacheConversationIDsMemRatio safely sets the Configuration value for state's 'Cache.ConversationIDsMemRatio' field -func (st *ConfigState) SetCacheConversationIDsMemRatio(v float64) { +// SetCacheConversationLastStatusIDsMemRatio safely sets the Configuration value for state's 'Cache.ConversationLastStatusIDsMemRatio' field +func (st *ConfigState) SetCacheConversationLastStatusIDsMemRatio(v float64) { st.mutex.Lock() defer st.mutex.Unlock() - st.config.Cache.ConversationIDsMemRatio = v + st.config.Cache.ConversationLastStatusIDsMemRatio = v st.reloadToViper() } -// CacheConversationIDsMemRatioFlag returns the flag name for the 'Cache.ConversationIDsMemRatio' field -func CacheConversationIDsMemRatioFlag() string { return "cache-conversation-ids-mem-ratio" } +// CacheConversationLastStatusIDsMemRatioFlag returns the flag name for the 'Cache.ConversationLastStatusIDsMemRatio' field +func CacheConversationLastStatusIDsMemRatioFlag() string { + return "cache-conversation-last-status-ids-mem-ratio" +} -// GetCacheConversationIDsMemRatio safely fetches the value for global configuration 'Cache.ConversationIDsMemRatio' field -func GetCacheConversationIDsMemRatio() float64 { return global.GetCacheConversationIDsMemRatio() } +// GetCacheConversationLastStatusIDsMemRatio safely fetches the value for global configuration 'Cache.ConversationLastStatusIDsMemRatio' field +func GetCacheConversationLastStatusIDsMemRatio() float64 { + return global.GetCacheConversationLastStatusIDsMemRatio() +} -// SetCacheConversationIDsMemRatio safely sets the value for global configuration 'Cache.ConversationIDsMemRatio' field -func SetCacheConversationIDsMemRatio(v float64) { global.SetCacheConversationIDsMemRatio(v) } +// SetCacheConversationLastStatusIDsMemRatio safely sets the value for global configuration 'Cache.ConversationLastStatusIDsMemRatio' field +func SetCacheConversationLastStatusIDsMemRatio(v float64) { + global.SetCacheConversationLastStatusIDsMemRatio(v) +} // GetCacheEmojiMemRatio safely fetches the Configuration value for state's 'Cache.EmojiMemRatio' field func (st *ConfigState) GetCacheEmojiMemRatio() (v float64) { diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 632ae51328..6a3a30a009 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -149,45 +149,52 @@ func (c *conversationDB) populateConversation(ctx context.Context, conversation } func (c *conversationDB) GetConversationsByOwnerAccountID(ctx context.Context, accountID string, page *paging.Page) ([]*gtsmodel.Conversation, error) { - conversationIDs, err := c.getAccountConversationIDs(ctx, accountID, page) + conversationLastStatusIDs, err := c.getAccountConversationLastStatusIDs(ctx, accountID, page) if err != nil { return nil, err } - return c.getConversationsByIDs(ctx, conversationIDs) + return c.getConversationsByLastStatusIDs(ctx, accountID, conversationLastStatusIDs) } -func (c *conversationDB) getAccountConversationIDs(ctx context.Context, accountID string, page *paging.Page) ([]string, error) { - return loadPagedIDs(&c.state.Caches.GTS.ConversationIDs, accountID, page, func() ([]string, error) { - var conversationIDs []string +func (c *conversationDB) getAccountConversationLastStatusIDs(ctx context.Context, accountID string, page *paging.Page) ([]string, error) { + return loadPagedIDs(&c.state.Caches.GTS.ConversationLastStatusIDs, accountID, page, func() ([]string, error) { + var conversationLastStatusIDs []string - // Conversation IDs not in cache. Perform DB query. + // Conversation last status IDs not in cache. Perform DB query. if _, err := c.db. NewSelect(). TableExpr("?", bun.Ident("conversations")). - ColumnExpr("?", bun.Ident("id")). + ColumnExpr("?", bun.Ident("last_status_id")). Where("? = ?", bun.Ident("account_id"), accountID). - OrderExpr("? DESC", bun.Ident("id")). - Exec(ctx, &conversationIDs); // nocollapse + OrderExpr("? DESC", bun.Ident("last_status_id")). + Exec(ctx, &conversationLastStatusIDs); // nocollapse err != nil && !errors.Is(err, db.ErrNoEntries) { return nil, err } - return conversationIDs, nil + return conversationLastStatusIDs, nil }) } -func (c *conversationDB) getConversationsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Conversation, error) { +func (c *conversationDB) getConversationsByLastStatusIDs( + ctx context.Context, + accountID string, + conversationLastStatusIDs []string, +) ([]*gtsmodel.Conversation, error) { // Load all conversation IDs via cache loader callbacks. - conversations, err := c.state.Caches.GTS.Conversation.LoadIDs("ID", - ids, - func(uncached []string) ([]*gtsmodel.Conversation, error) { + conversations, err := c.state.Caches.GTS.Conversation.LoadIDs2Part( + "AccountID,LastStatusID", + accountID, + conversationLastStatusIDs, + func(accountID string, uncached []string) ([]*gtsmodel.Conversation, error) { // Preallocate expected length of uncached conversations. conversations := make([]*gtsmodel.Conversation, 0, len(uncached)) // Perform database query scanning the remaining (uncached) IDs. if err := c.db.NewSelect(). Model(&conversations). - Where("? IN (?)", bun.Ident("id"), bun.In(uncached)). + Where("? = ?", bun.Ident("last_status_id"), accountID). + Where("? IN (?)", bun.Ident("last_status_id"), bun.In(uncached)). Scan(ctx); err != nil { return nil, err } @@ -199,9 +206,9 @@ func (c *conversationDB) getConversationsByIDs(ctx context.Context, ids []string return nil, err } - // Reorder the conversations by their IDs to ensure correct order. + // Reorder the conversations by their last status IDs to ensure correct order. getID := func(b *gtsmodel.Conversation) string { return b.ID } - util.OrderBy(conversations, ids, getID) + util.OrderBy(conversations, conversationLastStatusIDs, getID) if gtscontext.Barebones(ctx) { // no need to fully populate. @@ -317,8 +324,9 @@ func (c *conversationDB) DeleteConversationsByOwnerAccountID(ctx context.Context // so we don't need to load all conversations into the cache to run invalidation hooks, // as with some other object types (blocks, for example). c.state.Caches.GTS.Conversation.Invalidate("AccountID", accountID) - // In case there were no cached conversations, explicitly invalidate conversation ID cache. - c.state.Caches.GTS.ConversationIDs.Invalidate(accountID) + // In case there were no cached conversations, + // explicitly invalidate the user's conversation last status ID cache. + c.state.Caches.GTS.ConversationLastStatusIDs.Invalidate(accountID) }() return c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { diff --git a/internal/db/bundb/migrations/20240611190733_add_conversations.go b/internal/db/bundb/migrations/20240611190733_add_conversations.go index 37f3de5c23..ff654c6190 100644 --- a/internal/db/bundb/migrations/20240611190733_add_conversations.go +++ b/internal/db/bundb/migrations/20240611190733_add_conversations.go @@ -40,7 +40,7 @@ func init() { } } - // Add indexes to the conversation table. + // Add indexes to the conversations table. for index, columns := range map[string][]string{ "conversations_account_id_idx": { "account_id", @@ -60,6 +60,18 @@ func init() { } } + // Add additional uniqueness constraints to the conversations table. + if _, err := tx. + NewCreateIndex(). + Model(>smodel.Conversation{}). + Index("conversations_account_id_last_status_id_uniq"). + Column("account_id", "last_status_id"). + Unique(). + IfNotExists(). + Exec(ctx); err != nil { + return err + } + return nil }) } diff --git a/internal/db/conversation.go b/internal/db/conversation.go index 6538a9efe2..80e953f168 100644 --- a/internal/db/conversation.go +++ b/internal/db/conversation.go @@ -31,7 +31,8 @@ type Conversation interface { // GetConversationByThreadAndAccountIDs retrieves a conversation by thread ID and participant account IDs, if it exists. GetConversationByThreadAndAccountIDs(ctx context.Context, threadID string, accountID string, otherAccountIDs []string) (*gtsmodel.Conversation, error) - // GetConversationsByOwnerAccountID gets all conversations owned by the given account, with optional paging. + // GetConversationsByOwnerAccountID gets all conversations owned by the given account, + // with optional paging based on last status ID. GetConversationsByOwnerAccountID(ctx context.Context, accountID string, page *paging.Page) ([]*gtsmodel.Conversation, error) // UpdateConversation updates an existing conversation. diff --git a/internal/gtsmodel/conversation.go b/internal/gtsmodel/conversation.go index fe81a68865..7d384507d3 100644 --- a/internal/gtsmodel/conversation.go +++ b/internal/gtsmodel/conversation.go @@ -27,18 +27,18 @@ import ( // Conversation represents direct messages between the owner account and a set of other accounts. type Conversation struct { - ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database - CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated - AccountID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversation_thread_id_account_id_other_accounts_key_uniq"` // Account that owns the conversation - Account *Account `bun:"-"` // - OtherAccountIDs []string `bun:"other_account_ids,array"` // Other accounts participating in the conversation (doesn't include the owner, may be empty in the case of a DM to yourself) - OtherAccounts []*Account `bun:"-"` // - OtherAccountsKey string `bun:",notnull,unique:conversation_thread_id_account_id_other_accounts_key_uniq"` // Denormalized lookup key derived from unique OtherAccountIDs, sorted and concatenated with commas, may be empty in the case of a DM to yourself - ThreadID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversation_thread_id_account_id_other_accounts_key_uniq"` // Thread that the conversation is part of - LastStatusID string `bun:"type:CHAR(26),nullzero,notnull"` // id of the last status in this conversation - LastStatus *Status `bun:"-"` // - Read *bool `bun:",default:false"` // Has the owner read all statuses in this conversation? + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + AccountID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` // Account that owns the conversation + Account *Account `bun:"-"` // + OtherAccountIDs []string `bun:"other_account_ids,array"` // Other accounts participating in the conversation (doesn't include the owner, may be empty in the case of a DM to yourself) + OtherAccounts []*Account `bun:"-"` // + OtherAccountsKey string `bun:",notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` // Denormalized lookup key derived from unique OtherAccountIDs, sorted and concatenated with commas, may be empty in the case of a DM to yourself + ThreadID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` // Thread that the conversation is part of + LastStatusID string `bun:"type:CHAR(26),nullzero,notnull"` // id of the last status in this conversation + LastStatus *Status `bun:"-"` // + Read *bool `bun:",default:false"` // Has the owner read all statuses in this conversation? } // ConversationOtherAccountsKey creates an OtherAccountsKey from a list of OtherAccountIDs. diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go index f05b01200a..4a78ebb606 100644 --- a/internal/processing/conversations/conversations_test.go +++ b/internal/processing/conversations/conversations_test.go @@ -71,6 +71,7 @@ type ConversationsTestSuite struct { // conversation created for test testAccount *gtsmodel.Account testConversation *gtsmodel.Conversation + testNow time.Time } func (suite *ConversationsTestSuite) getClientMsg(timeout time.Duration) (*messages.FromClientAPI, bool) { @@ -121,8 +122,9 @@ func (suite *ConversationsTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") + suite.testNow = time.Now() suite.testAccount = suite.testAccounts["local_account_1"] - suite.testConversation = suite.newTestConversation() + suite.testConversation = suite.newTestConversation(0) } func (suite *ConversationsTestSuite) TearDownTest() { @@ -141,10 +143,9 @@ func (suite *ConversationsTestSuite) TearDownTest() { testrig.StopWorkers(&suite.state) } -// newTestConversation creates a new status and adds it to a new unread conversation, returning the conversation. -func (suite *ConversationsTestSuite) newTestConversation() *gtsmodel.Conversation { +func (suite *ConversationsTestSuite) newTestStatus(threadID string, nowOffset time.Duration, inReplyToStatus *gtsmodel.Status) *gtsmodel.Status { statusID := id.NewULID() - createdAt := time.Now() + createdAt := suite.testNow.Add(nowOffset) status := >smodel.Status{ ID: statusID, CreatedAt: createdAt, @@ -153,7 +154,7 @@ func (suite *ConversationsTestSuite) newTestConversation() *gtsmodel.Conversatio AccountID: suite.testAccount.ID, AccountURI: suite.testAccount.URI, Local: util.Ptr(true), - ThreadID: id.NewULID(), + ThreadID: threadID, Visibility: gtsmodel.VisibilityDirect, ActivityStreamsType: ap.ObjectNote, Federated: util.Ptr(true), @@ -161,10 +162,20 @@ func (suite *ConversationsTestSuite) newTestConversation() *gtsmodel.Conversatio Replyable: util.Ptr(true), Likeable: util.Ptr(true), } + if inReplyToStatus != nil { + status.InReplyToID = inReplyToStatus.ID + status.InReplyToURI = inReplyToStatus.URI + status.InReplyToAccountID = inReplyToStatus.AccountID + } if err := suite.db.PutStatus(context.Background(), status); err != nil { suite.FailNow(err.Error()) } + return status +} +// newTestConversation creates a new status and adds it to a new unread conversation, returning the conversation. +func (suite *ConversationsTestSuite) newTestConversation(nowOffset time.Duration) *gtsmodel.Conversation { + status := suite.newTestStatus(id.NewULID(), nowOffset, nil) conversation := >smodel.Conversation{ ID: id.NewULID(), AccountID: suite.testAccount.ID, diff --git a/internal/processing/conversations/get.go b/internal/processing/conversations/get.go index c040c3f3b7..6f69221b8a 100644 --- a/internal/processing/conversations/get.go +++ b/internal/processing/conversations/get.go @@ -53,9 +53,9 @@ func (p *Processor) GetAll( return util.EmptyPageableResponse(), nil } - // Get the lowest and highest ID values, used for paging. - lo := conversations[count-1].ID - hi := conversations[0].ID + // Get the lowest and highest last status ID values, used for paging. + lo := conversations[count-1].LastStatusID + hi := conversations[0].LastStatusID items := make([]interface{}, 0, count) diff --git a/internal/processing/conversations/get_test.go b/internal/processing/conversations/get_test.go index 6aca956484..3f9a409120 100644 --- a/internal/processing/conversations/get_test.go +++ b/internal/processing/conversations/get_test.go @@ -31,3 +31,32 @@ func (suite *ConversationsTestSuite) TestGetAll() { suite.True(apiConversation.Unread) } } + +// Test that conversations with newer last status IDs are returned earlier. +func (suite *ConversationsTestSuite) TestGetAllOrder() { + // Get our previously created conversation. + conversation1 := suite.testConversation + + // Create a new conversation with a last status newer than conversation1's. + conversation2 := suite.newTestConversation(1) + + // Add an even newer status than that to conversation1. + conversation1Status2 := suite.newTestStatus(conversation1.LastStatus.ThreadID, 2, conversation1.LastStatus) + conversation1, err := suite.db.AddStatusToConversation(context.Background(), conversation1, conversation1Status2) + if err != nil { + suite.FailNow(err.Error()) + } + + resp, err := suite.conversationsProcessor.GetAll(context.Background(), suite.testAccount, nil) + if suite.NoError(err) && suite.Len(resp.Items, 2) { + // conversation1 should be the first conversation returned. + apiConversation1 := resp.Items[0].(*apimodel.Conversation) + suite.Equal(conversation1.ID, apiConversation1.ID) + // It should have the newest status added to it. + suite.Equal(conversation1.LastStatusID, conversation1Status2.ID) + + // conversation2 should be the second conversation returned. + apiConversation2 := resp.Items[1].(*apimodel.Conversation) + suite.Equal(conversation2.ID, apiConversation2.ID) + } +} diff --git a/test/envparsing.sh b/test/envparsing.sh index 30881162f5..cd0d260ba9 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -32,7 +32,7 @@ EXPECT=$(cat << "EOF" "block-mem-ratio": 2, "boost-of-ids-mem-ratio": 3, "client-mem-ratio": 0.1, - "conversation-ids-mem-ratio": 3, + "conversation-last-status-ids-mem-ratio": 3, "conversation-mem-ratio": 2, "emoji-category-mem-ratio": 0.1, "emoji-mem-ratio": 3, From 86262e673b0b8f3577241dce4862db5e912fb6af Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 2 Jul 2024 11:23:36 -0700 Subject: [PATCH 03/35] Appease linter --- internal/db/bundb/conversation.go | 4 ++-- internal/media/manager.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 6a3a30a009..b4b8ce0812 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -465,8 +465,8 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat return err } - conversationIDs := append(updatedConversationIDs, deletedConversationIDs...) - c.state.Caches.GTS.Conversation.InvalidateIDs("ID", conversationIDs) + updatedConversationIDs = append(updatedConversationIDs, deletedConversationIDs...) + c.state.Caches.GTS.Conversation.InvalidateIDs("ID", updatedConversationIDs) return nil } diff --git a/internal/media/manager.go b/internal/media/manager.go index 90a2923b53..ea126e4608 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -308,7 +308,7 @@ func (m *Manager) RefreshEmoji( // paths before they get updated with new // path ID. These are required for later // deleting the old image files on refresh. - shortcodeDomain := util.ShortcodeDomain(emoji) + shortcodeDomain := emoji.ShortcodeDomain() oldStaticPath := emoji.ImageStaticPath oldPath := emoji.ImagePath From 6a3e4d3198ecfa646698c517bc69f002267ebf22 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 2 Jul 2024 13:02:20 -0700 Subject: [PATCH 04/35] Fix deleting conversations and statuses --- internal/db/bundb/conversation.go | 37 +++++++++++++++++++-------- internal/db/conversation.go | 3 +-- internal/processing/account/delete.go | 3 ++- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index b4b8ce0812..fc351638a6 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -332,7 +332,7 @@ func (c *conversationDB) DeleteConversationsByOwnerAccountID(ctx context.Context return c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { // Delete conversations matching the account ID. deletedConversationIDs := []string{} - if err := c.db.NewDelete(). + if err := tx.NewDelete(). Model((*gtsmodel.Conversation)(nil)). Where("? = ?", bun.Ident("account_id"), accountID). Returning("?", bun.Ident("id")). @@ -342,7 +342,7 @@ func (c *conversationDB) DeleteConversationsByOwnerAccountID(ctx context.Context } // Delete any conversation-to-status links matching the deleted conversation IDs. - if _, err := c.db.NewDelete(). + if _, err := tx.NewDelete(). Model((*gtsmodel.ConversationToStatus)(nil)). Where("? IN (?)", bun.Ident("conversation_id"), bun.In(deletedConversationIDs)). Exec(ctx); // nocollapse @@ -382,9 +382,10 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // Create a temporary table with all statuses other than the deleted status // in each conversation for which the deleted status is the last status // (if there are such statuses). + conversationStatusesTempTable := bun.Ident("conversation_statuses_" + id.NewULID()) if _, err := tx.NewRaw( ` - CREATE TEMPORARY TABLE conversation_statuses AS + CREATE TEMPORARY TABLE ?0 AS SELECT conversations.id conversation_id, conversation_to_statuses.status_id id, @@ -392,11 +393,12 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat FROM conversations LEFT JOIN conversation_to_statuses ON conversations.id = conversation_to_statuses.conversation_id - AND conversation_to_statuses.status_id != ?0 + AND conversation_to_statuses.status_id != ?1 LEFT JOIN statuses ON conversation_to_statuses.status_id = statuses.id - WHERE conversations.last_status_id = ?0 + WHERE conversations.last_status_id = ?1 `, + conversationStatusesTempTable, statusID, ).Exec(ctx); // nocollapse err != nil { @@ -405,18 +407,21 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // Create a temporary table with the most recently created status in each conversation // for which the deleted status is the last status (if there is such a status). + latestConversationStatusesTempTable := bun.Ident("latest_conversation_statuses_" + id.NewULID()) if _, err := tx.NewRaw( ` - CREATE TEMPORARY TABLE latest_conversation_statuses AS + CREATE TEMPORARY TABLE ?0 AS SELECT conversation_statuses.conversation_id, conversation_statuses.id - FROM conversation_statuses - LEFT JOIN conversation_statuses later_statuses + FROM ?1 conversation_statuses + LEFT JOIN ?1 later_statuses ON conversation_statuses.conversation_id = later_statuses.conversation_id AND later_statuses.created_at > conversation_statuses.created_at WHERE later_statuses.id IS NULL `, + latestConversationStatusesTempTable, + conversationStatusesTempTable, ).Exec(ctx); // nocollapse err != nil { return err @@ -431,12 +436,13 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat UPDATE conversations SET last_status_id = latest_conversation_statuses.id, - updated_at = ? - FROM latest_conversation_statuses + updated_at = ?1 + FROM ?0 latest_conversation_statuses WHERE conversations.id = latest_conversation_statuses.conversation_id AND latest_conversation_statuses.id IS NOT NULL RETURNING id `, + latestConversationStatusesTempTable, bun.Safe(nowSQL), ).Scan(ctx, &updatedConversationIDs); // nocollapse err != nil { @@ -450,16 +456,25 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat DELETE FROM conversations WHERE id IN ( SELECT conversation_id - FROM latest_conversation_statuses + FROM ?0 WHERE id IS NULL ) RETURNING id `, + latestConversationStatusesTempTable, ).Scan(ctx, &deletedConversationIDs); // nocollapse err != nil { return err } + // Clean up. + if _, err := tx.NewRaw(`DROP TABLE ?`, conversationStatusesTempTable).Exec(ctx); err != nil { + return err + } + if _, err := tx.NewRaw(`DROP TABLE ?`, latestConversationStatusesTempTable).Exec(ctx); err != nil { + return err + } + return nil }); err != nil { return err diff --git a/internal/db/conversation.go b/internal/db/conversation.go index 80e953f168..0e2fb09e74 100644 --- a/internal/db/conversation.go +++ b/internal/db/conversation.go @@ -49,8 +49,7 @@ type Conversation interface { // DeleteConversationsByOwnerAccountID deletes all conversations owned by the given account. DeleteConversationsByOwnerAccountID(ctx context.Context, accountID string) error - // DeleteStatusFromConversations handles when a status is deleted by nulling out the last status for - // any conversations in which it was the last status. + // DeleteStatusFromConversations handles when a status is deleted by updating or deleting conversations for which it was the last status. DeleteStatusFromConversations(ctx context.Context, statusID string) error MigrateConversations(ctx context.Context, migrateStatus func(ctx context.Context, status *gtsmodel.Status) error) error diff --git a/internal/processing/account/delete.go b/internal/processing/account/delete.go index 64ce3ee796..702b46cdac 100644 --- a/internal/processing/account/delete.go +++ b/internal/processing/account/delete.go @@ -461,7 +461,8 @@ func (p *Processor) deleteAccountPeripheral(ctx context.Context, account *gtsmod // TODO: add status mutes here when they're implemented. // Delete all conversations owned by given account. - // Accounts in which the conversation + // Conversations in which it has only participated will be retained; + // they can always be deleted by their owners. if err := p.state.DB.DeleteConversationsByOwnerAccountID(ctx, account.ID); // nocollapse err != nil && !errors.Is(err, db.ErrNoEntries) { return gtserror.Newf("error deleting conversations owned by account: %w", err) From a388920de895d7afb55c035182e0c5b1b15c0c74 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 11 Jul 2024 18:52:33 -0700 Subject: [PATCH 05/35] Refactor to make migrations automatic --- .../action/debug/conversations/migrate.go | 86 ------------ cmd/gotosocial/action/server/server.go | 5 + cmd/gotosocial/debug.go | 14 -- internal/db/advancedmigration.go | 29 ++++ internal/db/bundb/advancedmigration.go | 52 +++++++ internal/db/bundb/bundb.go | 5 + internal/db/bundb/conversation.go | 41 ------ .../20240611190733_add_conversations.go | 4 +- .../20240712005536_add_advanced_migrations.go | 49 +++++++ internal/db/bundb/status.go | 32 +++++ internal/db/conversation.go | 2 - internal/db/db.go | 1 + internal/db/status.go | 9 ++ internal/gtsmodel/advancedmigration.go | 48 +++++++ .../processing/conversations/conversations.go | 9 +- .../conversations/conversations_test.go | 32 +++-- .../processing/conversations/delete_test.go | 4 +- internal/processing/conversations/get_test.go | 15 +- internal/processing/conversations/migrate.go | 131 ++++++++++++++++++ .../processing/conversations/migrate_test.go | 85 ++++++++++++ .../processing/conversations/read_test.go | 8 +- .../update.go} | 46 +++--- .../processing/conversations/update_test.go | 54 ++++++++ internal/processing/processor.go | 5 +- internal/processing/stream/conversation.go | 5 +- internal/processing/workers/fromclientapi.go | 4 - internal/processing/workers/fromfediapi.go | 8 -- internal/processing/workers/surface.go | 12 +- .../processing/workers/surfacetimeline.go | 13 +- internal/processing/workers/workers.go | 13 +- 30 files changed, 610 insertions(+), 211 deletions(-) delete mode 100644 cmd/gotosocial/action/debug/conversations/migrate.go create mode 100644 internal/db/advancedmigration.go create mode 100644 internal/db/bundb/advancedmigration.go create mode 100644 internal/db/bundb/migrations/20240712005536_add_advanced_migrations.go create mode 100644 internal/gtsmodel/advancedmigration.go create mode 100644 internal/processing/conversations/migrate.go create mode 100644 internal/processing/conversations/migrate_test.go rename internal/processing/{workers/surfaceconversations.go => conversations/update.go} (80%) create mode 100644 internal/processing/conversations/update_test.go diff --git a/cmd/gotosocial/action/debug/conversations/migrate.go b/cmd/gotosocial/action/debug/conversations/migrate.go deleted file mode 100644 index 7e08ef9c37..0000000000 --- a/cmd/gotosocial/action/debug/conversations/migrate.go +++ /dev/null @@ -1,86 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package conversations - -import ( - "context" - "fmt" - - "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action" - "github.com/superseriousbusiness/gotosocial/internal/db/bundb" - "github.com/superseriousbusiness/gotosocial/internal/email" - "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/oauth" - "github.com/superseriousbusiness/gotosocial/internal/processing/stream" - "github.com/superseriousbusiness/gotosocial/internal/processing/workers" - "github.com/superseriousbusiness/gotosocial/internal/state" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" -) - -func initState(ctx context.Context) (*state.State, error) { - var state state.State - state.Caches.Init() - state.Caches.Start() - - // Set the state DB connection - dbConn, err := bundb.NewBunDBService(ctx, &state) - if err != nil { - return nil, fmt.Errorf("error creating dbConn: %w", err) - } - state.DB = dbConn - - return &state, nil -} - -func stopState(state *state.State) error { - err := state.DB.Close() - state.Caches.Stop() - return err -} - -// Migrate processes every DM to create conversations. -var Migrate action.GTSAction = func(ctx context.Context) (err error) { - state, err := initState(ctx) - if err != nil { - return err - } - - defer func() { - // Ensure state gets stopped on return. - if err := stopState(state); err != nil { - log.Error(ctx, err) - } - }() - - streamProcessor := stream.New(state, oauth.New(ctx, state.DB)) - surface := workers.Surface{ - State: state, - Converter: typeutils.NewConverter(state), - Stream: &streamProcessor, - Filter: visibility.NewFilter(state), - } - if surface.EmailSender, err = email.NewNoopSender(func(toAddress string, message string) {}); err != nil { - return nil - } - - return state.DB.MigrateConversations(ctx, func(ctx context.Context, status *gtsmodel.Status) error { - return surface.UpdateConversationsForStatus(ctx, status, false) - }) -} diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 14db67795f..ae2ddce869 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -286,6 +286,11 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error initializing metrics: %w", err) } + // Run advanced migrations. + if err := processor.Conversations().MigrateDMsToConversations(ctx); err != nil { + return fmt.Errorf("error running conversations advanced migration: %w", err) + } + /* HTTP router initialization */ diff --git a/cmd/gotosocial/debug.go b/cmd/gotosocial/debug.go index f248167fdb..cd59cdf042 100644 --- a/cmd/gotosocial/debug.go +++ b/cmd/gotosocial/debug.go @@ -20,7 +20,6 @@ package main import ( "github.com/spf13/cobra" configaction "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/debug/config" - "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action/debug/conversations" "github.com/superseriousbusiness/gotosocial/internal/config" ) @@ -43,18 +42,5 @@ func debugCommands() *cobra.Command { config.AddServerFlags(debugConfigCmd) debugCmd.AddCommand(debugConfigCmd) - debugMigrateConversationsCmd := &cobra.Command{ - Use: "migrate-conversations", - Short: "process EVERY DM to create conversations", - PreRunE: func(cmd *cobra.Command, args []string) error { - return preRun(preRunArgs{cmd: cmd}) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return run(cmd.Context(), conversations.Migrate) - }, - } - config.AddServerFlags(debugMigrateConversationsCmd) - debugCmd.AddCommand(debugMigrateConversationsCmd) - return debugCmd } diff --git a/internal/db/advancedmigration.go b/internal/db/advancedmigration.go new file mode 100644 index 0000000000..2b4601bdbf --- /dev/null +++ b/internal/db/advancedmigration.go @@ -0,0 +1,29 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type AdvancedMigration interface { + GetAdvancedMigration(ctx context.Context, id string) (*gtsmodel.AdvancedMigration, error) + PutAdvancedMigration(ctx context.Context, advancedMigration *gtsmodel.AdvancedMigration) error +} diff --git a/internal/db/bundb/advancedmigration.go b/internal/db/bundb/advancedmigration.go new file mode 100644 index 0000000000..2a0ec93e61 --- /dev/null +++ b/internal/db/bundb/advancedmigration.go @@ -0,0 +1,52 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/uptrace/bun" +) + +type advancedMigrationDB struct { + db *bun.DB + state *state.State +} + +func (a *advancedMigrationDB) GetAdvancedMigration(ctx context.Context, id string) (*gtsmodel.AdvancedMigration, error) { + var advancedMigration gtsmodel.AdvancedMigration + err := a.db.NewSelect(). + Model(&advancedMigration). + Where("? = ?", bun.Ident("id"), id). + Limit(1). + Scan(ctx) + if err != nil { + return nil, err + } + return &advancedMigration, nil +} + +func (a *advancedMigrationDB) PutAdvancedMigration(ctx context.Context, advancedMigration *gtsmodel.AdvancedMigration) error { + _, err := NewUpsert(a.db). + Model(advancedMigration). + Constraint("id"). + Exec(ctx) + return err +} diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index a9f4ae0b8d..006b33a3cd 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -54,6 +54,7 @@ import ( type DBService struct { db.Account db.Admin + db.AdvancedMigration db.Application db.Basic db.Conversation @@ -182,6 +183,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { db: db, state: state, }, + AdvancedMigration: &advancedMigrationDB{ + db: db, + state: state, + }, Application: &applicationDB{ db: db, state: state, diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index fc351638a6..f5782105b3 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -485,44 +485,3 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat return nil } - -func (c *conversationDB) MigrateConversations(ctx context.Context, migrateStatus func(ctx context.Context, status *gtsmodel.Status) error) error { - log.Info(ctx, "migrating DMs to conversations…") - - batchSize := 100 - statuses := make([]*gtsmodel.Status, 0, batchSize) - minID := id.Lowest - for { - if err := c.db. - NewSelect(). - Model(&statuses). - Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityDirect). - Where("? > ?", bun.Ident("id"), minID). - Order("id ASC"). - Limit(batchSize). - Scan(ctx); err != nil { - return err - } - - if len(statuses) == 0 { - break - } - log.Infof(ctx, "migrating %d DMs starting past %s", len(statuses), minID) - minID = statuses[len(statuses)-1].ID - - for _, status := range statuses { - if err := c.state.DB.PopulateStatus(ctx, status); err != nil { - log.Errorf(ctx, "couldn't populate DM %s for conversation migration: %v", status.ID, err) - continue - } - if err := migrateStatus(ctx, status); err != nil { - log.Errorf(ctx, "couldn't process DM %s for conversation migration: %v", status.ID, err) - continue - } - } - } - - log.Info(ctx, "finished migrating DMs to conversations.") - - return nil -} diff --git a/internal/db/bundb/migrations/20240611190733_add_conversations.go b/internal/db/bundb/migrations/20240611190733_add_conversations.go index ff654c6190..6df7cc8f5c 100644 --- a/internal/db/bundb/migrations/20240611190733_add_conversations.go +++ b/internal/db/bundb/migrations/20240611190733_add_conversations.go @@ -24,6 +24,8 @@ import ( "github.com/uptrace/bun" ) +// Note: this migration has an advanced migration followup. +// See Conversations.MigrateDMs(). func init() { up := func(ctx context.Context, db *bun.DB) error { return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { @@ -60,7 +62,7 @@ func init() { } } - // Add additional uniqueness constraints to the conversations table. + // Add an additional uniqueness constraint to the conversations table. if _, err := tx. NewCreateIndex(). Model(>smodel.Conversation{}). diff --git a/internal/db/bundb/migrations/20240712005536_add_advanced_migrations.go b/internal/db/bundb/migrations/20240712005536_add_advanced_migrations.go new file mode 100644 index 0000000000..1830652851 --- /dev/null +++ b/internal/db/bundb/migrations/20240712005536_add_advanced_migrations.go @@ -0,0 +1,49 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migrations + +import ( + "context" + + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +// Create the advanced migrations table. +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + _, err := tx. + NewCreateTable(). + Model((*gtsmodel.AdvancedMigration)(nil)). + IfNotExists(). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 512730dd52..1568cfae7d 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -677,3 +677,35 @@ func (s *statusDB) getStatusBoostIDs(ctx context.Context, statusID string) ([]st return statusIDs, nil }) } + +func (s *statusDB) MaxDirectStatusID(ctx context.Context) (string, error) { + maxID := "" + if err := s.db. + NewSelect(). + Model((*gtsmodel.Status)(nil)). + ColumnExpr("COALESCE(MAX(?), '')", bun.Ident("id")). + Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityDirect). + Scan(ctx, &maxID); // nocollapse + err != nil { + return "", err + } + return maxID, nil +} + +func (s *statusDB) GetDirectStatusIDsBatch(ctx context.Context, minID string, maxID string, count int) ([]string, error) { + var statusIDs []string + if err := s.db. + NewSelect(). + Model((*gtsmodel.Status)(nil)). + Column("id"). + Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityDirect). + Where("? > ?", bun.Ident("id"), minID). + Where("? <= ?", bun.Ident("id"), maxID). + Order("id ASC"). + Limit(count). + Scan(ctx, &statusIDs); // nocollapse + err != nil { + return nil, err + } + return statusIDs, nil +} diff --git a/internal/db/conversation.go b/internal/db/conversation.go index 0e2fb09e74..7614d497d9 100644 --- a/internal/db/conversation.go +++ b/internal/db/conversation.go @@ -51,6 +51,4 @@ type Conversation interface { // DeleteStatusFromConversations handles when a status is deleted by updating or deleting conversations for which it was the last status. DeleteStatusFromConversations(ctx context.Context, statusID string) error - - MigrateConversations(ctx context.Context, migrateStatus func(ctx context.Context, status *gtsmodel.Status) error) error } diff --git a/internal/db/db.go b/internal/db/db.go index 31095e3b04..0a25a159d7 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -26,6 +26,7 @@ const ( type DB interface { Account Admin + AdvancedMigration Application Basic Conversation diff --git a/internal/db/status.go b/internal/db/status.go index 88ae12a128..01dcdb05a4 100644 --- a/internal/db/status.go +++ b/internal/db/status.go @@ -78,4 +78,13 @@ type Status interface { // GetStatusChildren gets the child statuses of a given status. GetStatusChildren(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) + + // MaxDirectStatusID returns the newest ID across all DM statuses. + // Returns the empty string with no error if there are no DM statuses yet. + // It is used only by the conversation advanced migration. + MaxDirectStatusID(ctx context.Context) (string, error) + + // GetDirectStatusIDsBatch returns up to count DM status IDs strictly greater than minID and less than maxID. + // It is used only by the conversation advanced migration. + GetDirectStatusIDsBatch(ctx context.Context, minID string, maxID string, count int) ([]string, error) } diff --git a/internal/gtsmodel/advancedmigration.go b/internal/gtsmodel/advancedmigration.go new file mode 100644 index 0000000000..ddd99dfda8 --- /dev/null +++ b/internal/gtsmodel/advancedmigration.go @@ -0,0 +1,48 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package gtsmodel + +import ( + "encoding/json" + "time" +) + +// AdvancedMigration stores state for an "advanced migration", which is a migration +// that doesn't fit into the Bun migration framework. +type AdvancedMigration struct { + ID string `bun:",pk,nullzero,notnull,unique"` // id of this migration (preassigned, not a ULID) + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + StateJson string `bun:",nullzero"` // JSON dump of the migration state + Finished *bool `bun:",nullzero,notnull,default:false"` // has this migration finished? +} + +func AdvancedMigrationLoad[State any](a *AdvancedMigration) (State, error) { + var state State + err := json.Unmarshal([]byte(a.StateJson), state) + return state, err +} + +func AdvancedMigrationStore[State any](a *AdvancedMigration, state State) error { + bytes, err := json.Marshal(state) + if err != nil { + return err + } + a.StateJson = string(bytes) + return nil +} diff --git a/internal/processing/conversations/conversations.go b/internal/processing/conversations/conversations.go index f4282d618b..d42126c230 100644 --- a/internal/processing/conversations/conversations.go +++ b/internal/processing/conversations/conversations.go @@ -18,6 +18,7 @@ package conversations import ( + "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -25,11 +26,17 @@ import ( type Processor struct { state *state.State converter *typeutils.Converter + filter *visibility.Filter } -func New(state *state.State, converter *typeutils.Converter) Processor { +func New( + state *state.State, + converter *typeutils.Converter, + filter *visibility.Filter, +) Processor { return Processor{ state: state, converter: converter, + filter: filter, } } diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go index 4a78ebb606..c8bc71c706 100644 --- a/internal/processing/conversations/conversations_test.go +++ b/internal/processing/conversations/conversations_test.go @@ -19,9 +19,11 @@ package conversations_test import ( "context" + "crypto/rand" "testing" "time" + "github.com/oklog/ulid" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -29,7 +31,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" @@ -54,6 +55,7 @@ type ConversationsTestSuite struct { federator *federation.Federator emailSender email.Sender sentEmails map[string]string + filter *visibility.Filter // standard suite models testTokens map[string]*gtsmodel.Token @@ -69,9 +71,8 @@ type ConversationsTestSuite struct { conversationsProcessor conversations.Processor // conversation created for test - testAccount *gtsmodel.Account - testConversation *gtsmodel.Conversation - testNow time.Time + testAccount *gtsmodel.Account + testNow time.Time } func (suite *ConversationsTestSuite) getClientMsg(timeout time.Duration) (*messages.FromClientAPI, bool) { @@ -102,10 +103,11 @@ func (suite *ConversationsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db suite.tc = typeutils.NewConverter(&suite.state) + suite.filter = visibility.NewFilter(&suite.state) testrig.StartTimelines( &suite.state, - visibility.NewFilter(&suite.state), + suite.filter, suite.tc, ) @@ -118,13 +120,12 @@ func (suite *ConversationsTestSuite) SetupTest() { suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) - suite.conversationsProcessor = conversations.New(&suite.state, suite.tc) + suite.conversationsProcessor = conversations.New(&suite.state, suite.tc, suite.filter) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") suite.testNow = time.Now() suite.testAccount = suite.testAccounts["local_account_1"] - suite.testConversation = suite.newTestConversation(0) } func (suite *ConversationsTestSuite) TearDownTest() { @@ -143,8 +144,18 @@ func (suite *ConversationsTestSuite) TearDownTest() { testrig.StopWorkers(&suite.state) } +func (suite *ConversationsTestSuite) newULID(nowOffset time.Duration) string { + ulid, err := ulid.New( + ulid.Timestamp(suite.testNow.Add(nowOffset)), rand.Reader, + ) + if err != nil { + panic(err) + } + return ulid.String() +} + func (suite *ConversationsTestSuite) newTestStatus(threadID string, nowOffset time.Duration, inReplyToStatus *gtsmodel.Status) *gtsmodel.Status { - statusID := id.NewULID() + statusID := suite.newULID(nowOffset) createdAt := suite.testNow.Add(nowOffset) status := >smodel.Status{ ID: statusID, @@ -175,9 +186,10 @@ func (suite *ConversationsTestSuite) newTestStatus(threadID string, nowOffset ti // newTestConversation creates a new status and adds it to a new unread conversation, returning the conversation. func (suite *ConversationsTestSuite) newTestConversation(nowOffset time.Duration) *gtsmodel.Conversation { - status := suite.newTestStatus(id.NewULID(), nowOffset, nil) + threadID := suite.newULID(nowOffset) + status := suite.newTestStatus(threadID, nowOffset, nil) conversation := >smodel.Conversation{ - ID: id.NewULID(), + ID: suite.newULID(nowOffset), AccountID: suite.testAccount.ID, ThreadID: status.ThreadID, Read: util.Ptr(false), diff --git a/internal/processing/conversations/delete_test.go b/internal/processing/conversations/delete_test.go index 17eb26da13..d050a4e8d6 100644 --- a/internal/processing/conversations/delete_test.go +++ b/internal/processing/conversations/delete_test.go @@ -20,6 +20,8 @@ package conversations_test import "context" func (suite *ConversationsTestSuite) TestDelete() { - err := suite.conversationsProcessor.Delete(context.Background(), suite.testAccount, suite.testConversation.ID) + conversation := suite.newTestConversation(0) + + err := suite.conversationsProcessor.Delete(context.Background(), suite.testAccount, conversation.ID) suite.NoError(err) } diff --git a/internal/processing/conversations/get_test.go b/internal/processing/conversations/get_test.go index 3f9a409120..198b6ef764 100644 --- a/internal/processing/conversations/get_test.go +++ b/internal/processing/conversations/get_test.go @@ -19,29 +19,32 @@ package conversations_test import ( "context" + "time" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" ) func (suite *ConversationsTestSuite) TestGetAll() { + conversation := suite.newTestConversation(0) + resp, err := suite.conversationsProcessor.GetAll(context.Background(), suite.testAccount, nil) if suite.NoError(err) && suite.Len(resp.Items, 1) && suite.IsType((*apimodel.Conversation)(nil), resp.Items[0]) { apiConversation := resp.Items[0].(*apimodel.Conversation) - suite.Equal(suite.testConversation.ID, apiConversation.ID) + suite.Equal(conversation.ID, apiConversation.ID) suite.True(apiConversation.Unread) } } // Test that conversations with newer last status IDs are returned earlier. func (suite *ConversationsTestSuite) TestGetAllOrder() { - // Get our previously created conversation. - conversation1 := suite.testConversation + // Create a new conversation. + conversation1 := suite.newTestConversation(0) - // Create a new conversation with a last status newer than conversation1's. - conversation2 := suite.newTestConversation(1) + // Create another new conversation with a last status newer than conversation1's. + conversation2 := suite.newTestConversation(1 * time.Second) // Add an even newer status than that to conversation1. - conversation1Status2 := suite.newTestStatus(conversation1.LastStatus.ThreadID, 2, conversation1.LastStatus) + conversation1Status2 := suite.newTestStatus(conversation1.LastStatus.ThreadID, 2*time.Second, conversation1.LastStatus) conversation1, err := suite.db.AddStatusToConversation(context.Background(), conversation1, conversation1Status2) if err != nil { suite.FailNow(err.Error()) diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go new file mode 100644 index 0000000000..bd34b8c495 --- /dev/null +++ b/internal/processing/conversations/migrate.go @@ -0,0 +1,131 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations + +import ( + "context" + "errors" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +const advancedMigrationID = "20240611190733_add_conversations" +const statusBatchSize = 100 + +type AdvancedMigrationState struct { + MinID string + MaxID string +} + +func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { + advancedMigration, err := p.state.DB.GetAdvancedMigration(ctx, advancedMigrationID) + if err != nil { + if !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("couldn't get advanced migration with ID %s: %w", advancedMigrationID, err) + } + } + state := AdvancedMigrationState{} + if advancedMigration != nil { + // There was a previous migration. + if *advancedMigration.Finished { + // This migration has already been run to completion; we don't need to run it again. + return nil + } + // Otherwise, pick up where we left off. + state, err = gtsmodel.AdvancedMigrationLoad[AdvancedMigrationState](advancedMigration) + } else { + // Start at the beginning. + state.MinID = id.Lowest + + // Find the max ID of all existing statuses. + // This will be the last one we migrate; + // newer ones will be handled by the normal conversation flow. + state.MaxID, err = p.state.DB.MaxDirectStatusID(ctx) + if err != nil { + return gtserror.Newf("couldn't get max DM status ID for migration: %w", err) + } + + // Save a new advanced migration record. + advancedMigration = >smodel.AdvancedMigration{ + ID: advancedMigrationID, + Finished: util.Ptr(false), + } + if err := gtsmodel.AdvancedMigrationStore(advancedMigration, state); err != nil { + // This should never happen. + return gtserror.Newf("couldn't serialize advanced migration state to JSON: %w", err) + } + if err := p.state.DB.PutAdvancedMigration(ctx, advancedMigration); err != nil { + return gtserror.Newf("couldn't save state for advanced migration with ID %s: %w", advancedMigrationID, err) + } + } + + log.Info(ctx, "migrating DMs to conversations…") + + // In batches, get all statuses up to and including the max ID, + // and update conversations for each in order. + for { + // Get status IDs for this batch. + statusIDs, err := p.state.DB.GetDirectStatusIDsBatch(ctx, state.MinID, state.MaxID, statusBatchSize) + if err != nil { + return gtserror.Newf("couldn't get DM status ID batch for migration: %w", err) + } + if len(statusIDs) == 0 { + break + } + log.Infof(ctx, "migrating %d DMs starting after %s", len(statusIDs), state.MinID) + + // Load the batch by IDs. + statuses, err := p.state.DB.GetStatusesByIDs(ctx, statusIDs) + if err != nil { + return gtserror.Newf("couldn't get DM statuses for migration: %w", err) + } + + // Update conversations for each status. Don't generate notifications. + for _, status := range statuses { + if _, err := p.UpdateConversationsForStatus(ctx, status); err != nil { + if err != nil { + return gtserror.Newf("couldn't update conversations for status %s during migration: %w", status.ID, err) + } + } + } + + // Save the migration state with the new min ID. + state.MinID = statusIDs[len(statusIDs)-1] + if err := gtsmodel.AdvancedMigrationStore(advancedMigration, state); err != nil { + // This should never happen. + return gtserror.Newf("couldn't serialize advanced migration state to JSON: %w", err) + } + if err := p.state.DB.PutAdvancedMigration(ctx, advancedMigration); err != nil { + return gtserror.Newf("couldn't save state for advanced migration with ID %s: %w", advancedMigrationID, err) + } + } + + // Mark the migration as finished. + advancedMigration.Finished = util.Ptr(true) + if err := p.state.DB.PutAdvancedMigration(ctx, advancedMigration); err != nil { + return gtserror.Newf("couldn't save state for advanced migration with ID %s: %w", advancedMigrationID, err) + } + + log.Info(ctx, "finished migrating DMs to conversations.") + return nil +} diff --git a/internal/processing/conversations/migrate_test.go b/internal/processing/conversations/migrate_test.go new file mode 100644 index 0000000000..b625e59ba5 --- /dev/null +++ b/internal/processing/conversations/migrate_test.go @@ -0,0 +1,85 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations_test + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// Test that we can migrate DMs to conversations. +// This test assumes that we're using the standard test fixtures, which contain some conversation-eligible DMs. +func (suite *ConversationsTestSuite) TestMigrateDMsToConversations() { + advancedMigrationID := "20240611190733_add_conversations" + ctx := context.Background() + rawDB := (suite.db).(*bundb.DBService).DB() + + // Precondition: we shouldn't have any conversations yet. + numConversations := 0 + if err := rawDB.NewSelect(). + Model((*gtsmodel.Conversation)(nil)). + ColumnExpr("COUNT(*)"). + Scan(ctx, &numConversations); // nocollapse + err != nil { + suite.FailNow(err.Error()) + } + suite.Zero(numConversations) + + // Precondition: there is no record of the conversations advanced migration. + _, err := suite.db.GetAdvancedMigration(ctx, advancedMigrationID) + suite.ErrorIs(err, db.ErrNoEntries) + + // Run the migration, which should not fail. + if err := suite.conversationsProcessor.MigrateDMsToConversations(ctx); err != nil { + suite.FailNow(err.Error()) + } + + // We should now have some conversations. + if err := rawDB.NewSelect(). + Model((*gtsmodel.Conversation)(nil)). + ColumnExpr("COUNT(*)"). + Scan(ctx, &numConversations); // nocollapse + err != nil { + suite.FailNow(err.Error()) + } + suite.NotZero(numConversations) + + // The advanced migration should now be marked as finished. + advancedMigration, err := suite.db.GetAdvancedMigration(ctx, advancedMigrationID) + if err != nil { + suite.FailNow(err.Error()) + } + if suite.NotNil(advancedMigration) && suite.NotNil(advancedMigration.Finished) { + suite.True(*advancedMigration.Finished) + } + + // Run the migration again, which should not fail. + if err := suite.conversationsProcessor.MigrateDMsToConversations(ctx); err != nil { + suite.FailNow(err.Error()) + } + + // However, it shouldn't have done anything, so the advanced migration should not have been updated. + advancedMigration2, err := suite.db.GetAdvancedMigration(ctx, advancedMigrationID) + if err != nil { + suite.FailNow(err.Error()) + } + suite.Equal(advancedMigration.UpdatedAt, advancedMigration2.UpdatedAt) +} diff --git a/internal/processing/conversations/read_test.go b/internal/processing/conversations/read_test.go index d30e5b45ec..d5ec97fc48 100644 --- a/internal/processing/conversations/read_test.go +++ b/internal/processing/conversations/read_test.go @@ -24,9 +24,11 @@ import ( ) func (suite *ConversationsTestSuite) TestRead() { - suite.False(util.PtrValueOr(suite.testConversation.Read, false)) - conversation, err := suite.conversationsProcessor.Read(context.Background(), suite.testAccount, suite.testConversation.ID) + conversation := suite.newTestConversation(0) + + suite.False(util.PtrValueOr(conversation.Read, false)) + apiConversation, err := suite.conversationsProcessor.Read(context.Background(), suite.testAccount, conversation.ID) if suite.NoError(err) { - suite.False(conversation.Unread) + suite.False(apiConversation.Unread) } } diff --git a/internal/processing/workers/surfaceconversations.go b/internal/processing/conversations/update.go similarity index 80% rename from internal/processing/workers/surfaceconversations.go rename to internal/processing/conversations/update.go index 56fe81767f..262244b29c 100644 --- a/internal/processing/workers/surfaceconversations.go +++ b/internal/processing/conversations/update.go @@ -15,12 +15,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package workers +package conversations import ( "context" "errors" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" @@ -31,22 +32,29 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/util" ) -// UpdateConversationsForStatus updates conversations that include this status, -// and sends conversation stream events if requested. -func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsmodel.Status, notify bool) error { +// UpdateConversationsForStatus updates all conversations related to a status, +// and returns a map from local account IDs to conversation notifications that should be sent to them. +func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gtsmodel.Status) (map[string]*apimodel.Conversation, error) { + notifications := map[string]*apimodel.Conversation{} + + // We need accounts to be populated for this. + if err := p.state.DB.PopulateStatus(ctx, status); err != nil { + return nil, err + } + if status.Visibility != gtsmodel.VisibilityDirect { // Only DMs are considered part of conversations. - return nil + return nil, nil } if status.BoostOfID != "" { // Boosts can't be part of conversations. // FUTURE: This may change if we ever implement quote posts. - return nil + return nil, nil } if status.ThreadID == "" { // If the status doesn't have a thread ID, it didn't mention a local account, // and thus can't be part of a conversation. - return nil + return nil, nil } // The account which authored the status plus all mentioned accounts. @@ -64,7 +72,7 @@ func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsm localAccount := participant // If the status is not visible to this account, skip processing it for this account. - visible, err := s.Filter.StatusVisible(ctx, localAccount, status) + visible, err := p.filter.StatusVisible(ctx, localAccount, status) if err != nil { log.Errorf( ctx, @@ -80,19 +88,19 @@ func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsm // TODO: (Vyr) find a prettier way to do this // Is the status filtered or muted for this user? - filters, err := s.State.DB.GetFiltersForAccountID(ctx, localAccount.ID) + filters, err := p.state.DB.GetFiltersForAccountID(ctx, localAccount.ID) if err != nil { log.Errorf(ctx, "error retrieving filters for account %s: %v", localAccount.ID, err) continue } - mutes, err := s.State.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), localAccount.ID, nil) + mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), localAccount.ID, nil) if err != nil { log.Errorf(ctx, "error retrieving mutes for account %s: %v", localAccount.ID, err) continue } compiledMutes := usermute.NewCompiledUserMuteList(mutes) // Converting the status to an API status runs the filter/mute checks. - _, err = s.Converter.StatusToAPIStatus( + _, err = p.converter.StatusToAPIStatus( ctx, status, localAccount, @@ -126,7 +134,7 @@ func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsm } // Check for a previously existing conversation, if there is one. - conversation, err := s.State.DB.GetConversationByThreadAndAccountIDs( + conversation, err := p.state.DB.GetConversationByThreadAndAccountIDs( ctx, status.ThreadID, localAccount.ID, @@ -157,7 +165,7 @@ func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsm } // Create or update the conversation. - conversation, err = s.State.DB.AddStatusToConversation(ctx, conversation, status) + conversation, err = p.state.DB.AddStatusToConversation(ctx, conversation, status) if err != nil { log.Errorf( ctx, @@ -171,7 +179,7 @@ func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsm } // Convert the conversation to API representation. - apiConversation, err := s.Converter.ConversationToAPIConversation( + apiConversation, err := p.converter.ConversationToAPIConversation( ctx, conversation, localAccount, @@ -193,11 +201,13 @@ func (s *Surface) UpdateConversationsForStatus(ctx context.Context, status *gtsm continue } - if notify { - // Send a conversation notification. - s.Stream.Conversation(ctx, localAccount, apiConversation) + // Generate a notification, + // unless the status was authored by the user who would be notified, + // in which case they already know. + if status.AccountID != localAccount.ID { + notifications[localAccount.ID] = apiConversation } } - return nil + return notifications, nil } diff --git a/internal/processing/conversations/update_test.go b/internal/processing/conversations/update_test.go new file mode 100644 index 0000000000..ebd730e4c6 --- /dev/null +++ b/internal/processing/conversations/update_test.go @@ -0,0 +1,54 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package conversations_test + +import ( + "context" +) + +// Test that we can create conversations when a new status comes in. +func (suite *ConversationsTestSuite) TestUpdateConversationsForStatus() { + ctx := context.Background() + + // Precondition: the test user shouldn't have any conversations yet. + conversations, err := suite.db.GetConversationsByOwnerAccountID(ctx, suite.testAccount.ID, nil) + if err != nil { + suite.FailNow(err.Error()) + } + suite.Empty(conversations) + + // Create a status. + threadID := suite.newULID(0) + status := suite.newTestStatus(threadID, 0, nil) + + // Update conversations for it. + notifications, err := suite.conversationsProcessor.UpdateConversationsForStatus(ctx, status) + if err != nil { + suite.FailNow(err.Error()) + } + + // In this test, the user is DMing themself, and should not receive a notification from that. + suite.Empty(notifications) + + // The test user should have a conversation now. + conversations, err = suite.db.GetConversationsByOwnerAccountID(ctx, suite.testAccount.ID, nil) + if err != nil { + suite.FailNow(err.Error()) + } + suite.NotEmpty(conversations) +} diff --git a/internal/processing/processor.go b/internal/processing/processor.go index e73d19eec7..826498cfdf 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -194,7 +194,9 @@ func NewProcessor( // processors + pin them to this struct. processor.account = account.New(&common, state, converter, mediaManager, federator, filter, parseMentionFunc) processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender) - processor.conversations = conversations.New(state, converter) + // The conversations processor doesn't actually need a reference to conversation logic, + // since it only handles reading and deleting them. + processor.conversations = conversations.New(state, converter, filter) processor.fedi = fedi.New(state, &common, converter, federator, filter) processor.filtersv1 = filtersv1.New(state, converter, &processor.stream) processor.filtersv2 = filtersv2.New(state, converter, &processor.stream) @@ -219,6 +221,7 @@ func NewProcessor( &processor.account, &processor.media, &processor.stream, + &processor.conversations, ) return processor diff --git a/internal/processing/stream/conversation.go b/internal/processing/stream/conversation.go index e08ae449a7..a0236c459d 100644 --- a/internal/processing/stream/conversation.go +++ b/internal/processing/stream/conversation.go @@ -23,19 +23,18 @@ import ( "codeberg.org/gruf/go-byteutil" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/stream" ) // Conversation streams the given conversation to any open, appropriate streams belonging to the given account. -func (p *Processor) Conversation(ctx context.Context, account *gtsmodel.Account, conversation *apimodel.Conversation) { +func (p *Processor) Conversation(ctx context.Context, accountID string, conversation *apimodel.Conversation) { b, err := json.Marshal(conversation) if err != nil { log.Errorf(ctx, "error marshaling json: %v", err) return } - p.streams.Post(ctx, account.ID, stream.Message{ + p.streams.Post(ctx, accountID, stream.Message{ Payload: byteutil.B2S(b), Event: stream.EventTypeConversation, Stream: []string{ diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go index 9d1a627e67..d5d4265e12 100644 --- a/internal/processing/workers/fromclientapi.go +++ b/internal/processing/workers/fromclientapi.go @@ -245,10 +245,6 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA log.Errorf(ctx, "error timelining and notifying status: %v", err) } - if err := p.surface.UpdateConversationsForStatus(ctx, status, true); err != nil { - log.Errorf(ctx, "error adding status to conversations: %v", err) - } - if status.InReplyToID != "" { // Interaction counts changed on the replied status; // uncache the prepared version from all timelines. diff --git a/internal/processing/workers/fromfediapi.go b/internal/processing/workers/fromfediapi.go index 506820552d..2844245c39 100644 --- a/internal/processing/workers/fromfediapi.go +++ b/internal/processing/workers/fromfediapi.go @@ -228,14 +228,6 @@ func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI) p.surface.invalidateStatusFromTimelines(ctx, status.InReplyToID) } - if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil { - log.Errorf(ctx, "error timelining and notifying status: %v", err) - } - - if err := p.surface.UpdateConversationsForStatus(ctx, status, true); err != nil { - log.Errorf(ctx, "error adding status to conversations: %v", err) - } - return nil } diff --git a/internal/processing/workers/surface.go b/internal/processing/workers/surface.go index 5ec905ae80..1a7dbbfe5b 100644 --- a/internal/processing/workers/surface.go +++ b/internal/processing/workers/surface.go @@ -20,6 +20,7 @@ package workers import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/processing/conversations" "github.com/superseriousbusiness/gotosocial/internal/processing/stream" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -32,9 +33,10 @@ import ( // - sending a notification to a user // - sending an email type Surface struct { - State *state.State - Converter *typeutils.Converter - Stream *stream.Processor - Filter *visibility.Filter - EmailSender email.Sender + State *state.State + Converter *typeutils.Converter + Stream *stream.Processor + Filter *visibility.Filter + EmailSender email.Sender + Conversations *conversations.Processor } diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go index 41d7f6f2ab..5e1432105c 100644 --- a/internal/processing/workers/surfacetimeline.go +++ b/internal/processing/workers/surfacetimeline.go @@ -36,8 +36,8 @@ import ( // and LIST timelines of accounts that follow the status author. // // It will also handle notifications for any mentions attached to -// the account, and notifications for any local accounts that want -// to know when this account posts. +// the account, notifications for any local accounts that want +// to know when this account posts, and conversations containing the status. func (s *Surface) timelineAndNotifyStatus(ctx context.Context, status *gtsmodel.Status) error { // Ensure status fully populated; including account, mentions, etc. if err := s.State.DB.PopulateStatus(ctx, status); err != nil { @@ -73,6 +73,15 @@ func (s *Surface) timelineAndNotifyStatus(ctx context.Context, status *gtsmodel. return gtserror.Newf("error notifying status mentions for status %s: %w", status.ID, err) } + // Update any conversations containing this status, and send conversation notifications. + notifications, err := s.Conversations.UpdateConversationsForStatus(ctx, status) + if err != nil { + return gtserror.Newf("error updating conversations for status %s: %w", status.ID, err) + } + for localAccountID, apiConversation := range notifications { + s.Stream.Conversation(ctx, localAccountID, apiConversation) + } + return nil } diff --git a/internal/processing/workers/workers.go b/internal/processing/workers/workers.go index 6b4cc07a6e..c7f67b0254 100644 --- a/internal/processing/workers/workers.go +++ b/internal/processing/workers/workers.go @@ -22,6 +22,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/processing/account" + "github.com/superseriousbusiness/gotosocial/internal/processing/conversations" "github.com/superseriousbusiness/gotosocial/internal/processing/media" "github.com/superseriousbusiness/gotosocial/internal/processing/stream" "github.com/superseriousbusiness/gotosocial/internal/state" @@ -44,6 +45,7 @@ func New( account *account.Processor, media *media.Processor, stream *stream.Processor, + conversations *conversations.Processor, ) Processor { // Init federate logic // wrapper struct. @@ -56,11 +58,12 @@ func New( // Init surface logic // wrapper struct. surface := &Surface{ - State: state, - Converter: converter, - Stream: stream, - Filter: filter, - EmailSender: emailSender, + State: state, + Converter: converter, + Stream: stream, + Filter: filter, + EmailSender: emailSender, + Conversations: conversations, } // Init shared util funcs. From 480744b71cc4aac299b302b3954e4fc8465e5b90 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 11 Jul 2024 19:04:07 -0700 Subject: [PATCH 06/35] Lint --- internal/gtsmodel/advancedmigration.go | 6 +++--- internal/processing/conversations/migrate.go | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/gtsmodel/advancedmigration.go b/internal/gtsmodel/advancedmigration.go index ddd99dfda8..52de3e44f9 100644 --- a/internal/gtsmodel/advancedmigration.go +++ b/internal/gtsmodel/advancedmigration.go @@ -28,13 +28,13 @@ type AdvancedMigration struct { ID string `bun:",pk,nullzero,notnull,unique"` // id of this migration (preassigned, not a ULID) CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated - StateJson string `bun:",nullzero"` // JSON dump of the migration state + StateJSON string `bun:",nullzero"` // JSON dump of the migration state Finished *bool `bun:",nullzero,notnull,default:false"` // has this migration finished? } func AdvancedMigrationLoad[State any](a *AdvancedMigration) (State, error) { var state State - err := json.Unmarshal([]byte(a.StateJson), state) + err := json.Unmarshal([]byte(a.StateJSON), state) return state, err } @@ -43,6 +43,6 @@ func AdvancedMigrationStore[State any](a *AdvancedMigration, state State) error if err != nil { return err } - a.StateJson = string(bytes) + a.StateJSON = string(bytes) return nil } diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go index bd34b8c495..d82d62165e 100644 --- a/internal/processing/conversations/migrate.go +++ b/internal/processing/conversations/migrate.go @@ -53,6 +53,10 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { } // Otherwise, pick up where we left off. state, err = gtsmodel.AdvancedMigrationLoad[AdvancedMigrationState](advancedMigration) + if err != nil { + // This should never happen. + return gtserror.Newf("couldn't deserialize advanced migration state from JSON: %w", err) + } } else { // Start at the beginning. state.MinID = id.Lowest From 80a2d6230855bdef8bbc12872fc1648b036774ec Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 11 Jul 2024 19:28:11 -0700 Subject: [PATCH 07/35] Update tests post-merge --- internal/db/bundb/conversation_test.go | 3 --- internal/processing/conversations/conversations_test.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/internal/db/bundb/conversation_test.go b/internal/db/bundb/conversation_test.go index 3588fe60d1..8df8bb4caa 100644 --- a/internal/db/bundb/conversation_test.go +++ b/internal/db/bundb/conversation_test.go @@ -70,9 +70,6 @@ func (suite *ConversationTestSuite) newStatus(nowOffset time.Duration, inReplyTo Visibility: gtsmodel.VisibilityDirect, ActivityStreamsType: ap.ObjectNote, Federated: util.Ptr(true), - Boostable: util.Ptr(true), - Replyable: util.Ptr(true), - Likeable: util.Ptr(true), } if inReplyTo != nil { status.InReplyToID = inReplyTo.ID diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go index c8bc71c706..cd36490270 100644 --- a/internal/processing/conversations/conversations_test.go +++ b/internal/processing/conversations/conversations_test.go @@ -169,9 +169,6 @@ func (suite *ConversationsTestSuite) newTestStatus(threadID string, nowOffset ti Visibility: gtsmodel.VisibilityDirect, ActivityStreamsType: ap.ObjectNote, Federated: util.Ptr(true), - Boostable: util.Ptr(true), - Replyable: util.Ptr(true), - Likeable: util.Ptr(true), } if inReplyToStatus != nil { status.InReplyToID = inReplyToStatus.ID From 3b6c59417a424e7a45773bc9455cef05e40fe95a Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Fri, 12 Jul 2024 10:28:02 -0700 Subject: [PATCH 08/35] Fixes from live-fire testing --- internal/db/bundb/conversation.go | 20 +++++++++---------- internal/gtsmodel/advancedmigration.go | 2 +- internal/processing/conversations/update.go | 3 ++- internal/processing/processor.go | 2 -- internal/processing/workers/fromfediapi.go | 4 ++++ .../processing/workers/surfacenotify_test.go | 11 +++++----- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index f5782105b3..23ffc1f05b 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -272,7 +272,7 @@ func (c *conversationDB) AddStatusToConversation(ctx context.Context, conversati Model(conversationToStatus). Exec(ctx); // nocollapse err != nil { - return err + return gtserror.Newf("error creating conversation-to-status link between conversation %s and status %s: %w", conversation.ID, status.ID, err) } if _, err := NewUpsert(tx). @@ -281,7 +281,7 @@ func (c *conversationDB) AddStatusToConversation(ctx context.Context, conversati Column("last_status_id", "read", "updated_at"). Exec(ctx); // nocollapse err != nil { - return err + return gtserror.Newf("error upserting conversation %s and status %s: %w", conversation.ID, err) } return nil @@ -338,7 +338,7 @@ func (c *conversationDB) DeleteConversationsByOwnerAccountID(ctx context.Context Returning("?", bun.Ident("id")). Scan(ctx, &deletedConversationIDs); // nocollapse err != nil { - return err + return gtserror.Newf("error deleting conversations for account %s: %w", accountID, err) } // Delete any conversation-to-status links matching the deleted conversation IDs. @@ -347,7 +347,7 @@ func (c *conversationDB) DeleteConversationsByOwnerAccountID(ctx context.Context Where("? IN (?)", bun.Ident("conversation_id"), bun.In(deletedConversationIDs)). Exec(ctx); // nocollapse err != nil { - return err + return gtserror.Newf("error deleting conversation-to-status links for account %s: %w", accountID, err) } return nil @@ -376,7 +376,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat Where("? = ?", bun.Ident("status_id"), statusID). Exec(ctx); // nocollapse err != nil { - return err + return gtserror.Newf("error deleting conversation-to-status links while deleting status %s: %w", statusID, err) } // Create a temporary table with all statuses other than the deleted status @@ -402,7 +402,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat statusID, ).Exec(ctx); // nocollapse err != nil { - return err + return gtserror.Newf("error creating conversationStatusesTempTable while deleting status %s: %w", statusID, err) } // Create a temporary table with the most recently created status in each conversation @@ -424,7 +424,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat conversationStatusesTempTable, ).Exec(ctx); // nocollapse err != nil { - return err + return gtserror.Newf("error creating latestConversationStatusesTempTable while deleting status %s: %w", statusID, err) } // For every conversation where the given status was the last one, @@ -440,13 +440,13 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat FROM ?0 latest_conversation_statuses WHERE conversations.id = latest_conversation_statuses.conversation_id AND latest_conversation_statuses.id IS NOT NULL - RETURNING id + RETURNING conversations.id `, latestConversationStatusesTempTable, bun.Safe(nowSQL), ).Scan(ctx, &updatedConversationIDs); // nocollapse err != nil { - return err + return gtserror.Newf("error rolling back last status for conversation while deleting status %s: %w", statusID, err) } // If there is no such status, delete the conversation. @@ -464,7 +464,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat latestConversationStatusesTempTable, ).Scan(ctx, &deletedConversationIDs); // nocollapse err != nil { - return err + return gtserror.Newf("error deleting conversation while deleting status %s: %w", statusID, err) } // Clean up. diff --git a/internal/gtsmodel/advancedmigration.go b/internal/gtsmodel/advancedmigration.go index 52de3e44f9..cb459f5029 100644 --- a/internal/gtsmodel/advancedmigration.go +++ b/internal/gtsmodel/advancedmigration.go @@ -34,7 +34,7 @@ type AdvancedMigration struct { func AdvancedMigrationLoad[State any](a *AdvancedMigration) (State, error) { var state State - err := json.Unmarshal([]byte(a.StateJSON), state) + err := json.Unmarshal([]byte(a.StateJSON), &state) return state, err } diff --git a/internal/processing/conversations/update.go b/internal/processing/conversations/update.go index 262244b29c..695477e43e 100644 --- a/internal/processing/conversations/update.go +++ b/internal/processing/conversations/update.go @@ -165,12 +165,13 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt } // Create or update the conversation. + conversationID := conversation.ID conversation, err = p.state.DB.AddStatusToConversation(ctx, conversation, status) if err != nil { log.Errorf( ctx, "error creating or updating conversation %s for status %s and account %s: %v", - conversation.ID, + conversationID, status.ID, localAccount.ID, err, diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 826498cfdf..b019af4131 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -194,8 +194,6 @@ func NewProcessor( // processors + pin them to this struct. processor.account = account.New(&common, state, converter, mediaManager, federator, filter, parseMentionFunc) processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender) - // The conversations processor doesn't actually need a reference to conversation logic, - // since it only handles reading and deleting them. processor.conversations = conversations.New(state, converter, filter) processor.fedi = fedi.New(state, &common, converter, federator, filter) processor.filtersv1 = filtersv1.New(state, converter, &processor.stream) diff --git a/internal/processing/workers/fromfediapi.go b/internal/processing/workers/fromfediapi.go index 2844245c39..ac4003f6a4 100644 --- a/internal/processing/workers/fromfediapi.go +++ b/internal/processing/workers/fromfediapi.go @@ -228,6 +228,10 @@ func (p *fediAPI) CreateStatus(ctx context.Context, fMsg *messages.FromFediAPI) p.surface.invalidateStatusFromTimelines(ctx, status.InReplyToID) } + if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil { + log.Errorf(ctx, "error timelining and notifying status: %v", err) + } + return nil } diff --git a/internal/processing/workers/surfacenotify_test.go b/internal/processing/workers/surfacenotify_test.go index 18d0277ae2..937ddeca20 100644 --- a/internal/processing/workers/surfacenotify_test.go +++ b/internal/processing/workers/surfacenotify_test.go @@ -39,11 +39,12 @@ func (suite *SurfaceNotifyTestSuite) TestSpamNotifs() { defer suite.TearDownTestStructs(testStructs) surface := &workers.Surface{ - State: testStructs.State, - Converter: testStructs.TypeConverter, - Stream: testStructs.Processor.Stream(), - Filter: visibility.NewFilter(testStructs.State), - EmailSender: testStructs.EmailSender, + State: testStructs.State, + Converter: testStructs.TypeConverter, + Stream: testStructs.Processor.Stream(), + Filter: visibility.NewFilter(testStructs.State), + EmailSender: testStructs.EmailSender, + Conversations: testStructs.Processor.Conversations(), } var ( From aafd0b9c8954ebb813977200001f42256e736e35 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Fri, 12 Jul 2024 10:33:44 -0700 Subject: [PATCH 09/35] Linter caught a format problem --- internal/db/bundb/conversation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 23ffc1f05b..6dad6394f0 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -281,7 +281,7 @@ func (c *conversationDB) AddStatusToConversation(ctx context.Context, conversati Column("last_status_id", "read", "updated_at"). Exec(ctx); // nocollapse err != nil { - return gtserror.Newf("error upserting conversation %s and status %s: %w", conversation.ID, err) + return gtserror.Newf("error upserting conversation %s: %w", conversation.ID, err) } return nil From 43856e60230b181d349c2e547f4633cc868924ba Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Fri, 12 Jul 2024 10:35:47 -0700 Subject: [PATCH 10/35] Refactor tests, fix cache --- cmd/gotosocial/debug.go | 1 - internal/cache/invalidate.go | 2 +- internal/db/bundb/conversation.go | 61 ++------- internal/db/bundb/conversation_test.go | 116 +++-------------- internal/db/conversation.go | 10 +- internal/db/test/conversation.go | 122 ++++++++++++++++++ .../conversations/conversations_test.go | 71 ++-------- .../processing/conversations/delete_test.go | 2 +- internal/processing/conversations/get_test.go | 12 +- internal/processing/conversations/read.go | 2 +- .../processing/conversations/read_test.go | 2 +- internal/processing/conversations/update.go | 32 ++++- .../processing/conversations/update_test.go | 4 +- internal/typeutils/internaltofrontend.go | 2 - 14 files changed, 210 insertions(+), 229 deletions(-) create mode 100644 internal/db/test/conversation.go diff --git a/cmd/gotosocial/debug.go b/cmd/gotosocial/debug.go index cd59cdf042..c01baeb8b1 100644 --- a/cmd/gotosocial/debug.go +++ b/cmd/gotosocial/debug.go @@ -41,6 +41,5 @@ func debugCommands() *cobra.Command { } config.AddServerFlags(debugConfigCmd) debugCmd.AddCommand(debugConfigCmd) - return debugCmd } diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 2922c0c89b..987a6eb64c 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -84,7 +84,7 @@ func (c *Caches) OnInvalidateClient(client *gtsmodel.Client) { } func (c *Caches) OnInvalidateConversation(conversation *gtsmodel.Conversation) { - // Invalidate source account's conversation list. + // Invalidate owning account's conversation list. c.GTS.ConversationLastStatusIDs.Invalidate(conversation.AccountID) } diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 6dad6394f0..6185c82f50 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -193,7 +193,7 @@ func (c *conversationDB) getConversationsByLastStatusIDs( // Perform database query scanning the remaining (uncached) IDs. if err := c.db.NewSelect(). Model(&conversations). - Where("? = ?", bun.Ident("last_status_id"), accountID). + Where("? = ?", bun.Ident("account_id"), accountID). Where("? IN (?)", bun.Ident("last_status_id"), bun.In(uncached)). Scan(ctx); err != nil { return nil, err @@ -227,70 +227,35 @@ func (c *conversationDB) getConversationsByLastStatusIDs( return conversations, nil } -func (c *conversationDB) UpdateConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error { +func (c *conversationDB) PutConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error { // If we're updating by column, ensure "updated_at" is included. if len(columns) > 0 { columns = append(columns, "updated_at") } return c.state.Caches.GTS.Conversation.Store(conversation, func() error { - _, err := c.db.NewUpdate(). + _, err := NewUpsert(c.db). Model(conversation). + Constraint("id"). Column(columns...). - Where("? = ?", bun.Ident("id"), conversation.ID). Exec(ctx) return err }) } -func (c *conversationDB) AddStatusToConversation(ctx context.Context, conversation *gtsmodel.Conversation, status *gtsmodel.Status) (*gtsmodel.Conversation, error) { - // Assume that if the conversation owner posted the status, they've already read it. - statusAuthoredByConversationOwner := status.AccountID == conversation.AccountID - - // Update the existing conversation. - // If there is no previous last status or this one is more recently created, set it as the last status. - if conversation.LastStatus == nil || conversation.LastStatus.CreatedAt.Before(status.CreatedAt) { - conversation.LastStatusID = status.ID - conversation.LastStatus = status - } - // If the conversation is unread, leave it marked as unread. - // If the conversation is read but this status might not have been, mark the conversation as unread. - if !statusAuthoredByConversationOwner { - conversation.Read = util.Ptr(false) - } - - // Link the conversation to the status. +func (c *conversationDB) LinkConversationToStatus(ctx context.Context, conversationID string, statusID string) error { conversationToStatus := >smodel.ConversationToStatus{ - ConversationID: conversation.ID, - StatusID: status.ID, + ConversationID: conversationID, + StatusID: statusID, } - // Upsert the conversation and insert the link, then cache the conversation. - if err := c.state.Caches.GTS.Conversation.Store(conversation, func() error { - return c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - if _, err := tx.NewInsert(). - Model(conversationToStatus). - Exec(ctx); // nocollapse - err != nil { - return gtserror.Newf("error creating conversation-to-status link between conversation %s and status %s: %w", conversation.ID, status.ID, err) - } - - if _, err := NewUpsert(tx). - Model(conversation). - Constraint("id"). - Column("last_status_id", "read", "updated_at"). - Exec(ctx); // nocollapse - err != nil { - return gtserror.Newf("error upserting conversation %s: %w", conversation.ID, err) - } - - return nil - }) - }); err != nil { - return nil, err + if _, err := c.db.NewInsert(). + Model(conversationToStatus). + Exec(ctx); // nocollapse + err != nil { + return err } - - return conversation, nil + return nil } func (c *conversationDB) DeleteConversationByID(ctx context.Context, id string) error { diff --git a/internal/db/bundb/conversation_test.go b/internal/db/bundb/conversation_test.go index 8df8bb4caa..24d35d4827 100644 --- a/internal/db/bundb/conversation_test.go +++ b/internal/db/bundb/conversation_test.go @@ -23,20 +23,18 @@ import ( "time" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/db/test" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/util" ) type ConversationTestSuite struct { BunDBStandardTestSuite - // account is the owner of statuses and conversations in these tests (must be local). - account *gtsmodel.Account - // now is the timestamp used as a base for creating new statuses in any given test. - now time.Time + cf test.ConversationFactory + + // testAccount is the owner of statuses and conversations in these tests (must be local). + testAccount *gtsmodel.Account // threadID is the thread used for statuses in any given test. threadID string } @@ -44,65 +42,17 @@ type ConversationTestSuite struct { func (suite *ConversationTestSuite) SetupSuite() { suite.BunDBStandardTestSuite.SetupSuite() - suite.account = suite.testAccounts["local_account_1"] + suite.cf.SetupSuite(suite) + + suite.testAccount = suite.testAccounts["local_account_1"] } func (suite *ConversationTestSuite) SetupTest() { suite.BunDBStandardTestSuite.SetupTest() - suite.now = time.Now() - suite.threadID = id.NewULID() -} - -// newStatus creates a new status in the DB that would be eligible for a conversation, optionally replying to a previous status. -func (suite *ConversationTestSuite) newStatus(nowOffset time.Duration, inReplyTo *gtsmodel.Status) *gtsmodel.Status { - statusID := id.NewULID() - createdAt := suite.now.Add(nowOffset) - status := >smodel.Status{ - ID: statusID, - CreatedAt: createdAt, - UpdatedAt: createdAt, - URI: "http://localhost:8080/users/" + suite.account.Username + "/statuses/" + statusID, - AccountID: suite.account.ID, - AccountURI: suite.account.URI, - Local: util.Ptr(true), - ThreadID: suite.threadID, - Visibility: gtsmodel.VisibilityDirect, - ActivityStreamsType: ap.ObjectNote, - Federated: util.Ptr(true), - } - if inReplyTo != nil { - status.InReplyToID = inReplyTo.ID - status.InReplyToURI = inReplyTo.URI - status.InReplyToAccountID = inReplyTo.AccountID - } - if err := suite.db.PutStatus(context.Background(), status); err != nil { - suite.FailNow(err.Error()) - } - - return status -} + suite.cf.SetupTest(suite.db) -// newConversation creates a new conversation not yet in the DB. -func (suite *ConversationTestSuite) newConversation() *gtsmodel.Conversation { - return >smodel.Conversation{ - ID: id.NewULID(), - AccountID: suite.account.ID, - ThreadID: suite.threadID, - Read: util.Ptr(true), - } -} - -// addStatus adds a status to a conversation and ends the test if that fails. -func (suite *ConversationTestSuite) addStatus( - conversation *gtsmodel.Conversation, - status *gtsmodel.Status, -) *gtsmodel.Conversation { - conversation, err := suite.db.AddStatusToConversation(context.Background(), conversation, status) - if err != nil { - suite.FailNow(err.Error()) - } - return conversation + suite.threadID = suite.cf.NewULID(0) } // deleteStatus deletes a status from conversations and ends the test if that fails. @@ -122,38 +72,13 @@ func (suite *ConversationTestSuite) getConversation(conversationID string) *gtsm return conversation } -// Adding a status to a new conversation should set the last status. -func (suite *ConversationTestSuite) TestAddStatusToNewConversation() { - initial := suite.newStatus(0, nil) - conversation := suite.addStatus(suite.newConversation(), initial) - suite.Equal(initial.ID, conversation.LastStatusID) - if suite.NotNil(conversation.Read) { - // In this test suite, the author of the statuses is also the owner of the conversation, - // so the conversation should be marked as read. - suite.True(*conversation.Read) - } -} - -// Adding a newer status to an existing conversation should update the last status. -func (suite *ConversationTestSuite) TestAddStatusToExistingConversation() { - initial := suite.newStatus(0, nil) - conversation := suite.addStatus(suite.newConversation(), initial) - - reply := suite.newStatus(1, initial) - conversation = suite.addStatus(conversation, reply) - suite.Equal(reply.ID, conversation.LastStatusID) - if suite.NotNil(conversation.Read) { - suite.True(*conversation.Read) - } -} - // If we delete a status that is in a conversation but not the last status, // the conversation's last status should not change. func (suite *ConversationTestSuite) TestDeleteNonLastStatus() { - initial := suite.newStatus(0, nil) - conversation := suite.addStatus(suite.newConversation(), initial) - reply := suite.newStatus(1, initial) - conversation = suite.addStatus(conversation, reply) + conversation := suite.cf.NewTestConversation(suite.testAccount, 0) + initial := conversation.LastStatus + reply := suite.cf.NewTestStatus(suite.testAccount, conversation.ThreadID, 1*time.Second, initial) + conversation = suite.cf.SetLastStatus(conversation, reply) suite.deleteStatus(initial.ID) conversation = suite.getConversation(conversation.ID) @@ -163,10 +88,11 @@ func (suite *ConversationTestSuite) TestDeleteNonLastStatus() { // If we delete the last status in a conversation that has other statuses, // a previous status should become the new last status. func (suite *ConversationTestSuite) TestDeleteLastStatus() { - initial := suite.newStatus(0, nil) - conversation := suite.addStatus(suite.newConversation(), initial) - reply := suite.newStatus(1, initial) - conversation = suite.addStatus(conversation, reply) + conversation := suite.cf.NewTestConversation(suite.testAccount, 0) + initial := conversation.LastStatus + reply := suite.cf.NewTestStatus(suite.testAccount, conversation.ThreadID, 1*time.Second, initial) + conversation = suite.cf.SetLastStatus(conversation, reply) + conversation = suite.getConversation(conversation.ID) suite.deleteStatus(reply.ID) conversation = suite.getConversation(conversation.ID) @@ -176,8 +102,8 @@ func (suite *ConversationTestSuite) TestDeleteLastStatus() { // If we delete the only status in a conversation, // the conversation should be deleted as well. func (suite *ConversationTestSuite) TestDeleteOnlyStatus() { - initial := suite.newStatus(0, nil) - conversation := suite.addStatus(suite.newConversation(), initial) + conversation := suite.cf.NewTestConversation(suite.testAccount, 0) + initial := conversation.LastStatus suite.deleteStatus(initial.ID) _, err := suite.db.GetConversationByID(context.Background(), conversation.ID) diff --git a/internal/db/conversation.go b/internal/db/conversation.go index 7614d497d9..fef5f2e333 100644 --- a/internal/db/conversation.go +++ b/internal/db/conversation.go @@ -35,13 +35,11 @@ type Conversation interface { // with optional paging based on last status ID. GetConversationsByOwnerAccountID(ctx context.Context, accountID string, page *paging.Page) ([]*gtsmodel.Conversation, error) - // UpdateConversation updates an existing conversation. - UpdateConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error + // PutConversation creates or updates a conversation. + PutConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error - // AddStatusToConversation takes a conversation (which may or may not exist in the DB yet) and a status. - // It will link the status to the conversation, and if the status is newer than the last status, - // it will become the last status. This happens in a transaction. - AddStatusToConversation(ctx context.Context, conversation *gtsmodel.Conversation, status *gtsmodel.Status) (*gtsmodel.Conversation, error) + // LinkConversationToStatus creates a conversation-to-status link. + LinkConversationToStatus(ctx context.Context, statusID string, conversationID string) error // DeleteConversationByID deletes a conversation, removing it from the owning account's conversation list. DeleteConversationByID(ctx context.Context, id string) error diff --git a/internal/db/test/conversation.go b/internal/db/test/conversation.go new file mode 100644 index 0000000000..d9ebba967d --- /dev/null +++ b/internal/db/test/conversation.go @@ -0,0 +1,122 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test + +import ( + "context" + "crypto/rand" + "time" + + "github.com/oklog/ulid" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type testSuite interface { + FailNow(string, ...interface{}) bool +} + +// ConversationFactory can be embedded or included by test suites that want to generate statuses and conversations. +type ConversationFactory struct { + // Test suite, or at least the methods from it that we care about. + suite testSuite + // Test DB. + db db.DB + + // TestStart is the timestamp used as a base for timestamps and ULIDs in any given test. + TestStart time.Time +} + +// SetupSuite should be called by the SetupSuite of test suites that use this mixin. +func (f *ConversationFactory) SetupSuite(suite testSuite) { + f.suite = suite +} + +// SetupTest should be called by the SetupTest of test suites that use this mixin. +func (f *ConversationFactory) SetupTest(db db.DB) { + f.db = db + f.TestStart = time.Now() +} + +// NewULID is a version of id.NewULID that uses the test start time and an offset instead of the real time. +func (f *ConversationFactory) NewULID(offset time.Duration) string { + ulid, err := ulid.New( + ulid.Timestamp(f.TestStart.Add(offset)), rand.Reader, + ) + if err != nil { + panic(err) + } + return ulid.String() +} + +func (f *ConversationFactory) NewTestStatus(localAccount *gtsmodel.Account, threadID string, nowOffset time.Duration, inReplyToStatus *gtsmodel.Status) *gtsmodel.Status { + statusID := f.NewULID(nowOffset) + createdAt := f.TestStart.Add(nowOffset) + status := >smodel.Status{ + ID: statusID, + CreatedAt: createdAt, + UpdatedAt: createdAt, + URI: "http://localhost:8080/users/" + localAccount.Username + "/statuses/" + statusID, + AccountID: localAccount.ID, + AccountURI: localAccount.URI, + Local: util.Ptr(true), + ThreadID: threadID, + Visibility: gtsmodel.VisibilityDirect, + ActivityStreamsType: ap.ObjectNote, + Federated: util.Ptr(true), + } + if inReplyToStatus != nil { + status.InReplyToID = inReplyToStatus.ID + status.InReplyToURI = inReplyToStatus.URI + status.InReplyToAccountID = inReplyToStatus.AccountID + } + if err := f.db.PutStatus(context.Background(), status); err != nil { + f.suite.FailNow(err.Error()) + } + return status +} + +// NewTestConversation creates a new status and adds it to a new unread conversation, returning the conversation. +func (f *ConversationFactory) NewTestConversation(localAccount *gtsmodel.Account, nowOffset time.Duration) *gtsmodel.Conversation { + threadID := f.NewULID(nowOffset) + status := f.NewTestStatus(localAccount, threadID, nowOffset, nil) + conversation := >smodel.Conversation{ + ID: f.NewULID(nowOffset), + AccountID: localAccount.ID, + ThreadID: status.ThreadID, + Read: util.Ptr(false), + } + f.SetLastStatus(conversation, status) + return conversation +} + +// SetLastStatus sets an already stored status as the last status of a new or already stored conversation, +// and returns the updated conversation. +func (f *ConversationFactory) SetLastStatus(conversation *gtsmodel.Conversation, status *gtsmodel.Status) *gtsmodel.Conversation { + conversation.LastStatusID = status.ID + conversation.LastStatus = status + if err := f.db.PutConversation(context.Background(), conversation, "last_status_id"); err != nil { + f.suite.FailNow(err.Error()) + } + if err := f.db.LinkConversationToStatus(context.Background(), conversation.ID, status.ID); err != nil { + f.suite.FailNow(err.Error()) + } + return conversation +} diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go index cd36490270..cc7ec617e5 100644 --- a/internal/processing/conversations/conversations_test.go +++ b/internal/processing/conversations/conversations_test.go @@ -19,14 +19,12 @@ package conversations_test import ( "context" - "crypto/rand" "testing" "time" - "github.com/oklog/ulid" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" + dbtest "github.com/superseriousbusiness/gotosocial/internal/db/test" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" @@ -39,7 +37,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/internal/util" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -70,9 +67,11 @@ type ConversationsTestSuite struct { // module being tested conversationsProcessor conversations.Processor - // conversation created for test + // Owner of test conversations testAccount *gtsmodel.Account - testNow time.Time + + // Mixin for conversation tests + dbtest.ConversationFactory } func (suite *ConversationsTestSuite) getClientMsg(timeout time.Duration) (*messages.FromClientAPI, bool) { @@ -91,6 +90,8 @@ func (suite *ConversationsTestSuite) SetupSuite() { suite.testFollows = testrig.NewTestFollows() suite.testAttachments = testrig.NewTestAttachments() suite.testStatuses = testrig.NewTestStatuses() + + suite.ConversationFactory.SetupSuite(suite) } func (suite *ConversationsTestSuite) SetupTest() { @@ -124,7 +125,8 @@ func (suite *ConversationsTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") - suite.testNow = time.Now() + suite.ConversationFactory.SetupTest(suite.db) + suite.testAccount = suite.testAccounts["local_account_1"] } @@ -144,61 +146,6 @@ func (suite *ConversationsTestSuite) TearDownTest() { testrig.StopWorkers(&suite.state) } -func (suite *ConversationsTestSuite) newULID(nowOffset time.Duration) string { - ulid, err := ulid.New( - ulid.Timestamp(suite.testNow.Add(nowOffset)), rand.Reader, - ) - if err != nil { - panic(err) - } - return ulid.String() -} - -func (suite *ConversationsTestSuite) newTestStatus(threadID string, nowOffset time.Duration, inReplyToStatus *gtsmodel.Status) *gtsmodel.Status { - statusID := suite.newULID(nowOffset) - createdAt := suite.testNow.Add(nowOffset) - status := >smodel.Status{ - ID: statusID, - CreatedAt: createdAt, - UpdatedAt: createdAt, - URI: "http://localhost:8080/users/" + suite.testAccount.Username + "/statuses/" + statusID, - AccountID: suite.testAccount.ID, - AccountURI: suite.testAccount.URI, - Local: util.Ptr(true), - ThreadID: threadID, - Visibility: gtsmodel.VisibilityDirect, - ActivityStreamsType: ap.ObjectNote, - Federated: util.Ptr(true), - } - if inReplyToStatus != nil { - status.InReplyToID = inReplyToStatus.ID - status.InReplyToURI = inReplyToStatus.URI - status.InReplyToAccountID = inReplyToStatus.AccountID - } - if err := suite.db.PutStatus(context.Background(), status); err != nil { - suite.FailNow(err.Error()) - } - return status -} - -// newTestConversation creates a new status and adds it to a new unread conversation, returning the conversation. -func (suite *ConversationsTestSuite) newTestConversation(nowOffset time.Duration) *gtsmodel.Conversation { - threadID := suite.newULID(nowOffset) - status := suite.newTestStatus(threadID, nowOffset, nil) - conversation := >smodel.Conversation{ - ID: suite.newULID(nowOffset), - AccountID: suite.testAccount.ID, - ThreadID: status.ThreadID, - Read: util.Ptr(false), - } - conversation, err := suite.db.AddStatusToConversation(context.Background(), conversation, status) - if err != nil { - suite.FailNow(err.Error()) - } - - return conversation -} - func TestConversationsTestSuite(t *testing.T) { suite.Run(t, new(ConversationsTestSuite)) } diff --git a/internal/processing/conversations/delete_test.go b/internal/processing/conversations/delete_test.go index d050a4e8d6..23b4f1c1ac 100644 --- a/internal/processing/conversations/delete_test.go +++ b/internal/processing/conversations/delete_test.go @@ -20,7 +20,7 @@ package conversations_test import "context" func (suite *ConversationsTestSuite) TestDelete() { - conversation := suite.newTestConversation(0) + conversation := suite.NewTestConversation(suite.testAccount, 0) err := suite.conversationsProcessor.Delete(context.Background(), suite.testAccount, conversation.ID) suite.NoError(err) diff --git a/internal/processing/conversations/get_test.go b/internal/processing/conversations/get_test.go index 198b6ef764..bf2d5d660c 100644 --- a/internal/processing/conversations/get_test.go +++ b/internal/processing/conversations/get_test.go @@ -25,7 +25,7 @@ import ( ) func (suite *ConversationsTestSuite) TestGetAll() { - conversation := suite.newTestConversation(0) + conversation := suite.NewTestConversation(suite.testAccount, 0) resp, err := suite.conversationsProcessor.GetAll(context.Background(), suite.testAccount, nil) if suite.NoError(err) && suite.Len(resp.Items, 1) && suite.IsType((*apimodel.Conversation)(nil), resp.Items[0]) { @@ -38,15 +38,15 @@ func (suite *ConversationsTestSuite) TestGetAll() { // Test that conversations with newer last status IDs are returned earlier. func (suite *ConversationsTestSuite) TestGetAllOrder() { // Create a new conversation. - conversation1 := suite.newTestConversation(0) + conversation1 := suite.NewTestConversation(suite.testAccount, 0) // Create another new conversation with a last status newer than conversation1's. - conversation2 := suite.newTestConversation(1 * time.Second) + conversation2 := suite.NewTestConversation(suite.testAccount, 1*time.Second) // Add an even newer status than that to conversation1. - conversation1Status2 := suite.newTestStatus(conversation1.LastStatus.ThreadID, 2*time.Second, conversation1.LastStatus) - conversation1, err := suite.db.AddStatusToConversation(context.Background(), conversation1, conversation1Status2) - if err != nil { + conversation1Status2 := suite.NewTestStatus(suite.testAccount, conversation1.LastStatus.ThreadID, 2*time.Second, conversation1.LastStatus) + conversation1.LastStatusID = conversation1Status2.ID + if err := suite.db.PutConversation(context.Background(), conversation1, "last_status_id"); err != nil { suite.FailNow(err.Error()) } diff --git a/internal/processing/conversations/read.go b/internal/processing/conversations/read.go index 183d8440ab..06c0f37851 100644 --- a/internal/processing/conversations/read.go +++ b/internal/processing/conversations/read.go @@ -50,7 +50,7 @@ func (p *Processor) Read( // Mark the conversation as read. conversation.Read = util.Ptr(true) - if err := p.state.DB.UpdateConversation(ctx, conversation, "read"); err != nil { + if err := p.state.DB.PutConversation(ctx, conversation, "read"); err != nil { err = gtserror.Newf("db error updating conversation %s: %w", id, err) return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/processing/conversations/read_test.go b/internal/processing/conversations/read_test.go index d5ec97fc48..e062fcd857 100644 --- a/internal/processing/conversations/read_test.go +++ b/internal/processing/conversations/read_test.go @@ -24,7 +24,7 @@ import ( ) func (suite *ConversationsTestSuite) TestRead() { - conversation := suite.newTestConversation(0) + conversation := suite.NewTestConversation(suite.testAccount, 0) suite.False(util.PtrValueOr(conversation.Read, false)) apiConversation, err := suite.conversationsProcessor.Read(context.Background(), suite.testAccount, conversation.ID) diff --git a/internal/processing/conversations/update.go b/internal/processing/conversations/update.go index 695477e43e..b07d393c12 100644 --- a/internal/processing/conversations/update.go +++ b/internal/processing/conversations/update.go @@ -164,14 +164,28 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt } } + // Assume that if the conversation owner posted the status, they've already read it. + statusAuthoredByConversationOwner := status.AccountID == conversation.AccountID + + // Update the conversation. + // If there is no previous last status or this one is more recently created, set it as the last status. + if conversation.LastStatus == nil || conversation.LastStatus.CreatedAt.Before(status.CreatedAt) { + conversation.LastStatusID = status.ID + conversation.LastStatus = status + } + // If the conversation is unread, leave it marked as unread. + // If the conversation is read but this status might not have been, mark the conversation as unread. + if !statusAuthoredByConversationOwner { + conversation.Read = util.Ptr(false) + } + // Create or update the conversation. - conversationID := conversation.ID - conversation, err = p.state.DB.AddStatusToConversation(ctx, conversation, status) + err = p.state.DB.PutConversation(ctx, conversation) if err != nil { log.Errorf( ctx, "error creating or updating conversation %s for status %s and account %s: %v", - conversationID, + conversation.ID, status.ID, localAccount.ID, err, @@ -179,6 +193,18 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt continue } + // Link the conversation to the status. + if err := p.state.DB.LinkConversationToStatus(ctx, conversation.ID, status.ID); err != nil { + log.Errorf( + ctx, + "error linking conversation %s to status %s: %v", + conversation.ID, + status.ID, + err, + ) + continue + } + // Convert the conversation to API representation. apiConversation, err := p.converter.ConversationToAPIConversation( ctx, diff --git a/internal/processing/conversations/update_test.go b/internal/processing/conversations/update_test.go index ebd730e4c6..8ba2800fe4 100644 --- a/internal/processing/conversations/update_test.go +++ b/internal/processing/conversations/update_test.go @@ -33,8 +33,8 @@ func (suite *ConversationsTestSuite) TestUpdateConversationsForStatus() { suite.Empty(conversations) // Create a status. - threadID := suite.newULID(0) - status := suite.newTestStatus(threadID, 0, nil) + threadID := suite.NewULID(0) + status := suite.NewTestStatus(suite.testAccount, threadID, 0, nil) // Update conversations for it. notifications, err := suite.conversationsProcessor.UpdateConversationsForStatus(ctx, status) diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 6276153363..93ef2ed395 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -1672,8 +1672,6 @@ func (c *Converter) ConversationToAPIConversation( } for _, account := range conversation.OtherAccounts { var apiAccount *apimodel.Account - // TODO: (Vyr) good enough? is there something reusable for this? should we pull in a Filters dependency? - // TODO: (Vyr) duplicated in conversationsget.go blocked, err := c.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, account.ID) if err != nil { return nil, err From 9b725208f2c6756380d471a72b667fced2ddeed0 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Fri, 12 Jul 2024 13:46:59 -0700 Subject: [PATCH 11/35] Negative test for non-DMs --- .../processing/workers/fromclientapi_test.go | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go index 6b758ced86..35c2c31b71 100644 --- a/internal/processing/workers/fromclientapi_test.go +++ b/internal/processing/workers/fromclientapi_test.go @@ -885,6 +885,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusBoostNoReblogs() { ) } +// A DM to a local user should create a conversation and accompanying notification. func (suite *FromClientAPITestSuite) TestProcessCreateStatusWhichBeginsConversation() { testStructs := suite.SetupTestStructs() defer suite.TearDownTestStructs(testStructs) @@ -969,6 +970,74 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWhichBeginsConversat ) } +// A public message to a local user should not result in a conversation notification. +func (suite *FromClientAPITestSuite) TestProcessCreateStatusWhichShouldNotCreateConversation() { + testStructs := suite.SetupTestStructs() + defer suite.TearDownTestStructs(testStructs) + + var ( + ctx = context.Background() + postingAccount = suite.testAccounts["local_account_2"] + receivingAccount = suite.testAccounts["local_account_1"] + streams = suite.openStreams(ctx, + testStructs.Processor, + receivingAccount, + nil, + ) + homeStream = streams[stream.TimelineHome] + directStream = streams[stream.TimelineDirect] + + // turtle posts a new top-level public message mentioning zork. + status = suite.newStatus( + ctx, + testStructs.State, + postingAccount, + gtsmodel.VisibilityPublic, + nil, + nil, + []*gtsmodel.Account{receivingAccount}, + true, + ) + ) + + // Process the new status. + if err := testStructs.Processor.Workers().ProcessFromClientAPI( + ctx, + &messages.FromClientAPI{ + APObjectType: ap.ObjectNote, + APActivityType: ap.ActivityCreate, + GTSModel: status, + Origin: postingAccount, + }, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Check status in home stream. + suite.checkStreamed( + homeStream, + true, + "", + stream.EventTypeUpdate, + ) + + // Check mention notification in home stream. + suite.checkStreamed( + homeStream, + true, + "", + stream.EventTypeNotification, + ) + + // Check for absence of conversation notification in direct stream. + suite.checkStreamed( + directStream, + false, + "", + "", + ) +} + func (suite *FromClientAPITestSuite) TestProcessStatusDelete() { testStructs := suite.SetupTestStructs() defer suite.TearDownTestStructs(testStructs) From 2a8453177b0c54989bc561113b351dbfa28b7cac Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:02:08 -0700 Subject: [PATCH 12/35] Run conversations advanced migration on testrig startup as well as regular server startup --- cmd/gotosocial/action/testrig/testrig.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index 7b99a2a13d..603d06981f 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -204,6 +204,11 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error initializing metrics: %w", err) } + // Run advanced migrations. + if err := processor.Conversations().MigrateDMsToConversations(ctx); err != nil { + return fmt.Errorf("error running conversations advanced migration: %w", err) + } + /* HTTP router initialization */ From 7e4f4bb4d728bdd1d240f99b91e69ab514865b6f Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:05:51 -0700 Subject: [PATCH 13/35] Document (lack of) side effects of API method for deleting a conversation --- docs/api/swagger.yaml | 3 +++ internal/api/client/conversations/conversationdelete.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index f724585c57..e88996b63f 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -6216,6 +6216,9 @@ paths: - conversations /api/v1/conversations/{id}: delete: + description: |- + This doesn't delete the actual statuses in the conversation, + nor does it prevent a new conversation from being created later from the same thread and participants. operationId: conversationDelete parameters: - description: ID of the conversation diff --git a/internal/api/client/conversations/conversationdelete.go b/internal/api/client/conversations/conversationdelete.go index 2737e6cab7..6f8f43a94d 100644 --- a/internal/api/client/conversations/conversationdelete.go +++ b/internal/api/client/conversations/conversationdelete.go @@ -30,6 +30,9 @@ import ( // // Delete a single conversation with the given ID. // +// This doesn't delete the actual statuses in the conversation, +// nor does it prevent a new conversation from being created later from the same thread and participants. +// // --- // tags: // - conversations From a8c9b602d92fa31eeec3108ea95d2d770341a3b9 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:15:27 -0700 Subject: [PATCH 14/35] Make not-found check less nested for readability --- internal/processing/conversations/delete.go | 8 ++++---- internal/processing/conversations/read.go | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/processing/conversations/delete.go b/internal/processing/conversations/delete.go index 1cab7346df..64083bd536 100644 --- a/internal/processing/conversations/delete.go +++ b/internal/processing/conversations/delete.go @@ -34,12 +34,12 @@ func (p *Processor) Delete( ) gtserror.WithCode { // Get the conversation so that we can check its owning account ID. conversation, err := p.state.DB.GetConversationByID(gtscontext.SetBarebones(ctx), id) - if err != nil { - if errors.Is(err, db.ErrNoEntries) { - return gtserror.NewErrorNotFound(err) - } + if err != nil && !errors.Is(err, db.ErrNoEntries) { return gtserror.NewErrorInternalError(err) } + if conversation == nil { + return gtserror.NewErrorNotFound(err) + } if conversation.AccountID != requestingAccount.ID { return gtserror.NewErrorNotFound(nil) } diff --git a/internal/processing/conversations/read.go b/internal/processing/conversations/read.go index 06c0f37851..95e2c728a2 100644 --- a/internal/processing/conversations/read.go +++ b/internal/processing/conversations/read.go @@ -37,13 +37,12 @@ func (p *Processor) Read( ) (*apimodel.Conversation, gtserror.WithCode) { // Get the conversation, including participating accounts and last status. conversation, err := p.state.DB.GetConversationByID(ctx, id) - if err != nil { - if errors.Is(err, db.ErrNoEntries) { - return nil, gtserror.NewErrorNotFound(err) - } - err = gtserror.Newf("db error getting conversation %s: %w", id, err) + if err != nil && !errors.Is(err, db.ErrNoEntries) { return nil, gtserror.NewErrorInternalError(err) } + if conversation == nil { + return nil, gtserror.NewErrorNotFound(err) + } if conversation.AccountID != requestingAccount.ID { return nil, gtserror.NewErrorNotFound(nil) } From b7db4a555bd1515bbe2094b0efb5692ad90044a9 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:17:26 -0700 Subject: [PATCH 15/35] Rename PutConversation to UpsertConversation --- internal/db/bundb/conversation.go | 2 +- internal/db/conversation.go | 4 ++-- internal/db/test/conversation.go | 2 +- internal/processing/conversations/get_test.go | 2 +- internal/processing/conversations/read.go | 2 +- internal/processing/conversations/update.go | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 6185c82f50..c15ab7b4b5 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -227,7 +227,7 @@ func (c *conversationDB) getConversationsByLastStatusIDs( return conversations, nil } -func (c *conversationDB) PutConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error { +func (c *conversationDB) UpsertConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error { // If we're updating by column, ensure "updated_at" is included. if len(columns) > 0 { columns = append(columns, "updated_at") diff --git a/internal/db/conversation.go b/internal/db/conversation.go index fef5f2e333..3d0b4213e2 100644 --- a/internal/db/conversation.go +++ b/internal/db/conversation.go @@ -35,8 +35,8 @@ type Conversation interface { // with optional paging based on last status ID. GetConversationsByOwnerAccountID(ctx context.Context, accountID string, page *paging.Page) ([]*gtsmodel.Conversation, error) - // PutConversation creates or updates a conversation. - PutConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error + // UpsertConversation creates or updates a conversation. + UpsertConversation(ctx context.Context, conversation *gtsmodel.Conversation, columns ...string) error // LinkConversationToStatus creates a conversation-to-status link. LinkConversationToStatus(ctx context.Context, statusID string, conversationID string) error diff --git a/internal/db/test/conversation.go b/internal/db/test/conversation.go index d9ebba967d..95713927e4 100644 --- a/internal/db/test/conversation.go +++ b/internal/db/test/conversation.go @@ -112,7 +112,7 @@ func (f *ConversationFactory) NewTestConversation(localAccount *gtsmodel.Account func (f *ConversationFactory) SetLastStatus(conversation *gtsmodel.Conversation, status *gtsmodel.Status) *gtsmodel.Conversation { conversation.LastStatusID = status.ID conversation.LastStatus = status - if err := f.db.PutConversation(context.Background(), conversation, "last_status_id"); err != nil { + if err := f.db.UpsertConversation(context.Background(), conversation, "last_status_id"); err != nil { f.suite.FailNow(err.Error()) } if err := f.db.LinkConversationToStatus(context.Background(), conversation.ID, status.ID); err != nil { diff --git a/internal/processing/conversations/get_test.go b/internal/processing/conversations/get_test.go index bf2d5d660c..7b3d60749c 100644 --- a/internal/processing/conversations/get_test.go +++ b/internal/processing/conversations/get_test.go @@ -46,7 +46,7 @@ func (suite *ConversationsTestSuite) TestGetAllOrder() { // Add an even newer status than that to conversation1. conversation1Status2 := suite.NewTestStatus(suite.testAccount, conversation1.LastStatus.ThreadID, 2*time.Second, conversation1.LastStatus) conversation1.LastStatusID = conversation1Status2.ID - if err := suite.db.PutConversation(context.Background(), conversation1, "last_status_id"); err != nil { + if err := suite.db.UpsertConversation(context.Background(), conversation1, "last_status_id"); err != nil { suite.FailNow(err.Error()) } diff --git a/internal/processing/conversations/read.go b/internal/processing/conversations/read.go index 95e2c728a2..5b3630318c 100644 --- a/internal/processing/conversations/read.go +++ b/internal/processing/conversations/read.go @@ -49,7 +49,7 @@ func (p *Processor) Read( // Mark the conversation as read. conversation.Read = util.Ptr(true) - if err := p.state.DB.PutConversation(ctx, conversation, "read"); err != nil { + if err := p.state.DB.UpsertConversation(ctx, conversation, "read"); err != nil { err = gtserror.Newf("db error updating conversation %s: %w", id, err) return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/processing/conversations/update.go b/internal/processing/conversations/update.go index b07d393c12..d57698be2c 100644 --- a/internal/processing/conversations/update.go +++ b/internal/processing/conversations/update.go @@ -180,7 +180,7 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt } // Create or update the conversation. - err = p.state.DB.PutConversation(ctx, conversation) + err = p.state.DB.UpsertConversation(ctx, conversation) if err != nil { log.Errorf( ctx, From 3308051f688bedf4887490544e2b789b51dcc908 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:19:14 -0700 Subject: [PATCH 16/35] Use util.Ptr instead of IIFE --- internal/cache/size.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cache/size.go b/internal/cache/size.go index d45d790bb9..4c474fa28a 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -331,7 +331,7 @@ func sizeofConversation() uintptr { OtherAccountsKey: strings.Join([]string{exampleID, exampleID, exampleID}, ","), ThreadID: exampleID, LastStatusID: exampleID, - Read: func() *bool { ok := true; return &ok }(), + Read: util.Ptr(true), })) } From cf37096cf6103f20c6bb9e6b82f035f6a619dba2 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:24:09 -0700 Subject: [PATCH 17/35] Reduce cache used by conversations --- internal/config/defaults.go | 4 ++-- test/envparsing.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 6c2beffbde..82ea07e104 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -165,8 +165,8 @@ var Defaults = Configuration{ BlockIDsMemRatio: 3, BoostOfIDsMemRatio: 3, ClientMemRatio: 0.1, - ConversationMemRatio: 2, - ConversationLastStatusIDsMemRatio: 3, + ConversationMemRatio: 1, + ConversationLastStatusIDsMemRatio: 2, EmojiMemRatio: 3, EmojiCategoryMemRatio: 0.1, FilterMemRatio: 0.5, diff --git a/test/envparsing.sh b/test/envparsing.sh index 8f1aee1553..83dfb85fc5 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -32,8 +32,8 @@ EXPECT=$(cat << "EOF" "block-mem-ratio": 2, "boost-of-ids-mem-ratio": 3, "client-mem-ratio": 0.1, - "conversation-last-status-ids-mem-ratio": 3, - "conversation-mem-ratio": 2, + "conversation-last-status-ids-mem-ratio": 2, + "conversation-mem-ratio": 1, "emoji-category-mem-ratio": 0.1, "emoji-mem-ratio": 3, "filter-keyword-mem-ratio": 0.5, From 94fe752189cca3ee9dfcde5859e4e83fb767e9bc Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 15:27:16 -0700 Subject: [PATCH 18/35] Remove unnecessary TableExpr/ColumnExpr --- internal/db/bundb/conversation.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index c15ab7b4b5..3e9b9ca65e 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -163,8 +163,8 @@ func (c *conversationDB) getAccountConversationLastStatusIDs(ctx context.Context // Conversation last status IDs not in cache. Perform DB query. if _, err := c.db. NewSelect(). - TableExpr("?", bun.Ident("conversations")). - ColumnExpr("?", bun.Ident("last_status_id")). + Model((*gtsmodel.Conversation)(nil)). + Column("last_status_id"). Where("? = ?", bun.Ident("account_id"), accountID). OrderExpr("? DESC", bun.Ident("last_status_id")). Exec(ctx, &conversationLastStatusIDs); // nocollapse From f6ced0770d6ff4b5ff9638ab158b46cfcc7a64cf Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 21:23:08 -0700 Subject: [PATCH 19/35] Use struct tags for both unique constraints on Conversation --- .../20240611190733_add_conversations.go | 12 ------ internal/gtsmodel/conversation.go | 43 +++++++++++++------ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/internal/db/bundb/migrations/20240611190733_add_conversations.go b/internal/db/bundb/migrations/20240611190733_add_conversations.go index 6df7cc8f5c..25b226affe 100644 --- a/internal/db/bundb/migrations/20240611190733_add_conversations.go +++ b/internal/db/bundb/migrations/20240611190733_add_conversations.go @@ -62,18 +62,6 @@ func init() { } } - // Add an additional uniqueness constraint to the conversations table. - if _, err := tx. - NewCreateIndex(). - Model(>smodel.Conversation{}). - Index("conversations_account_id_last_status_id_uniq"). - Column("account_id", "last_status_id"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return err - } - return nil }) } diff --git a/internal/gtsmodel/conversation.go b/internal/gtsmodel/conversation.go index 7d384507d3..f03f274587 100644 --- a/internal/gtsmodel/conversation.go +++ b/internal/gtsmodel/conversation.go @@ -27,18 +27,37 @@ import ( // Conversation represents direct messages between the owner account and a set of other accounts. type Conversation struct { - ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database - CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated - AccountID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` // Account that owns the conversation - Account *Account `bun:"-"` // - OtherAccountIDs []string `bun:"other_account_ids,array"` // Other accounts participating in the conversation (doesn't include the owner, may be empty in the case of a DM to yourself) - OtherAccounts []*Account `bun:"-"` // - OtherAccountsKey string `bun:",notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` // Denormalized lookup key derived from unique OtherAccountIDs, sorted and concatenated with commas, may be empty in the case of a DM to yourself - ThreadID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` // Thread that the conversation is part of - LastStatusID string `bun:"type:CHAR(26),nullzero,notnull"` // id of the last status in this conversation - LastStatus *Status `bun:"-"` // - Read *bool `bun:",default:false"` // Has the owner read all statuses in this conversation? + // ID of this item in the database. + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` + + // When was this item created? + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` + + // When was this item last updated? + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` + + // Account that owns the conversation. + AccountID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq,unique:conversations_account_id_last_status_id_uniq"` + Account *Account `bun:"-"` + + // Other accounts participating in the conversation. + // Doesn't include the owner. May be empty in the case of a DM to yourself. + OtherAccountIDs []string `bun:"other_account_ids,array"` + OtherAccounts []*Account `bun:"-"` + + // Denormalized lookup key derived from unique OtherAccountIDs, sorted and concatenated with commas. + // May be empty in the case of a DM to yourself. + OtherAccountsKey string `bun:",notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` + + // Thread that the conversation is part of. + ThreadID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_thread_id_account_id_other_accounts_key_uniq"` + + // ID of the last status in this conversation. + LastStatusID string `bun:"type:CHAR(26),nullzero,notnull,unique:conversations_account_id_last_status_id_uniq"` + LastStatus *Status `bun:"-"` + + // Has the owner read all statuses in this conversation? + Read *bool `bun:",default:false"` } // ConversationOtherAccountsKey creates an OtherAccountsKey from a list of OtherAccountIDs. From 75a9ed8cab860b9b81b755f298ce66a3d86544af Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 21:28:27 -0700 Subject: [PATCH 20/35] Make it clear how paging with GetDirectStatusIDsBatch should be used --- internal/db/bundb/status.go | 4 ++-- internal/db/status.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index cc11e99a33..b0ed32e0ed 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -697,7 +697,7 @@ func (s *statusDB) MaxDirectStatusID(ctx context.Context) (string, error) { return maxID, nil } -func (s *statusDB) GetDirectStatusIDsBatch(ctx context.Context, minID string, maxID string, count int) ([]string, error) { +func (s *statusDB) GetDirectStatusIDsBatch(ctx context.Context, minID string, maxIDInclusive string, count int) ([]string, error) { var statusIDs []string if err := s.db. NewSelect(). @@ -705,7 +705,7 @@ func (s *statusDB) GetDirectStatusIDsBatch(ctx context.Context, minID string, ma Column("id"). Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityDirect). Where("? > ?", bun.Ident("id"), minID). - Where("? <= ?", bun.Ident("id"), maxID). + Where("? <= ?", bun.Ident("id"), maxIDInclusive). Order("id ASC"). Limit(count). Scan(ctx, &statusIDs); // nocollapse diff --git a/internal/db/status.go b/internal/db/status.go index 01dcdb05a4..ade9007281 100644 --- a/internal/db/status.go +++ b/internal/db/status.go @@ -84,7 +84,10 @@ type Status interface { // It is used only by the conversation advanced migration. MaxDirectStatusID(ctx context.Context) (string, error) - // GetDirectStatusIDsBatch returns up to count DM status IDs strictly greater than minID and less than maxID. + // GetDirectStatusIDsBatch returns up to count DM status IDs strictly greater than minID + // and less than or equal to maxIDInclusive. Note that this is different from most of our paging, + // which uses a maxID and returns IDs strictly less than that, because it's called with the result of + // MaxDirectStatusID, and expects to eventually return the status with that ID. // It is used only by the conversation advanced migration. - GetDirectStatusIDsBatch(ctx context.Context, minID string, maxID string, count int) ([]string, error) + GetDirectStatusIDsBatch(ctx context.Context, minID string, maxIDInclusive string, count int) ([]string, error) } From 67b7ccae03efebcf2f8341aacd9b48c482d9cdb4 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 21:38:45 -0700 Subject: [PATCH 21/35] Let conversation paging skip conversations it can't render --- internal/processing/conversations/get.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/processing/conversations/get.go b/internal/processing/conversations/get.go index 6f69221b8a..36d0c33aed 100644 --- a/internal/processing/conversations/get.go +++ b/internal/processing/conversations/get.go @@ -27,6 +27,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/paging" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -82,13 +83,13 @@ func (p *Processor) GetAll( compiledMutes, ) if err != nil { - err = gtserror.Newf( - "couldn't convert conversation %s to API representation for account %s: %w", + log.Errorf( + ctx, + "error converting conversation %s to API representation: %v", conversation.ID, - requestingAccount.ID, err, ) - return nil, gtserror.NewErrorInternalError(err) + continue } // Append conversation to return items. From 10abcf98528507947ed3621d5707012b162f8f91 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 22:10:09 -0700 Subject: [PATCH 22/35] Use Bun NewDropTable --- internal/db/bundb/conversation.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 3e9b9ca65e..7428f27dc5 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -347,7 +347,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // Create a temporary table with all statuses other than the deleted status // in each conversation for which the deleted status is the last status // (if there are such statuses). - conversationStatusesTempTable := bun.Ident("conversation_statuses_" + id.NewULID()) + conversationStatusesTempTable := "conversation_statuses_" + id.NewULID() if _, err := tx.NewRaw( ` CREATE TEMPORARY TABLE ?0 AS @@ -363,7 +363,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat ON conversation_to_statuses.status_id = statuses.id WHERE conversations.last_status_id = ?1 `, - conversationStatusesTempTable, + bun.Ident(conversationStatusesTempTable), statusID, ).Exec(ctx); // nocollapse err != nil { @@ -372,7 +372,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // Create a temporary table with the most recently created status in each conversation // for which the deleted status is the last status (if there is such a status). - latestConversationStatusesTempTable := bun.Ident("latest_conversation_statuses_" + id.NewULID()) + latestConversationStatusesTempTable := "latest_conversation_statuses_" + id.NewULID() if _, err := tx.NewRaw( ` CREATE TEMPORARY TABLE ?0 AS @@ -385,8 +385,8 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat AND later_statuses.created_at > conversation_statuses.created_at WHERE later_statuses.id IS NULL `, - latestConversationStatusesTempTable, - conversationStatusesTempTable, + bun.Ident(latestConversationStatusesTempTable), + bun.Ident(conversationStatusesTempTable), ).Exec(ctx); // nocollapse err != nil { return gtserror.Newf("error creating latestConversationStatusesTempTable while deleting status %s: %w", statusID, err) @@ -407,7 +407,7 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat AND latest_conversation_statuses.id IS NOT NULL RETURNING conversations.id `, - latestConversationStatusesTempTable, + bun.Ident(latestConversationStatusesTempTable), bun.Safe(nowSQL), ).Scan(ctx, &updatedConversationIDs); // nocollapse err != nil { @@ -426,18 +426,20 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat ) RETURNING id `, - latestConversationStatusesTempTable, + bun.Ident(latestConversationStatusesTempTable), ).Scan(ctx, &deletedConversationIDs); // nocollapse err != nil { return gtserror.Newf("error deleting conversation while deleting status %s: %w", statusID, err) } // Clean up. - if _, err := tx.NewRaw(`DROP TABLE ?`, conversationStatusesTempTable).Exec(ctx); err != nil { - return err - } - if _, err := tx.NewRaw(`DROP TABLE ?`, latestConversationStatusesTempTable).Exec(ctx); err != nil { - return err + for _, tempTable := range []string{ + conversationStatusesTempTable, + latestConversationStatusesTempTable, + } { + if _, err := tx.NewDropTable().Table(tempTable).Exec(ctx); err != nil { + return err + } } return nil From 573a81d1f0940bbc4beecc6d2448116235f8d5b1 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 22:22:54 -0700 Subject: [PATCH 23/35] Convert delete raw query to Bun --- internal/db/bundb/conversation.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 7428f27dc5..001a477fb4 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -416,18 +416,18 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // If there is no such status, delete the conversation. // Return conversation IDs for invalidation. - if err := tx.NewRaw( - ` - DELETE FROM conversations - WHERE id IN ( - SELECT conversation_id - FROM ?0 - WHERE id IS NULL - ) - RETURNING id - `, - bun.Ident(latestConversationStatusesTempTable), - ).Scan(ctx, &deletedConversationIDs); // nocollapse + if err := tx.NewDelete(). + Model((*gtsmodel.Conversation)(nil)). + Where( + "? IN (?)", + bun.Ident("id"), + tx.NewSelect(). + Table(latestConversationStatusesTempTable). + Column("conversation_id"). + Where("? IS NULL", bun.Ident("id")), + ). + Returning("?", bun.Ident("id")). + Scan(ctx, &deletedConversationIDs); // nocollapse err != nil { return gtserror.Newf("error deleting conversation while deleting status %s: %w", statusID, err) } From 66520da3e576778ecd3c9d83b2f67cf512d325d9 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 22:50:52 -0700 Subject: [PATCH 24/35] Convert update raw query to Bun --- internal/db/bundb/conversation.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 001a477fb4..95b322377d 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -396,20 +396,15 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // reset its last status to the most recently created in the conversation other than that one, // if there is such a status. // Return conversation IDs for invalidation. - if err := tx.NewRaw( - ` - UPDATE conversations - SET - last_status_id = latest_conversation_statuses.id, - updated_at = ?1 - FROM ?0 latest_conversation_statuses - WHERE conversations.id = latest_conversation_statuses.conversation_id - AND latest_conversation_statuses.id IS NOT NULL - RETURNING conversations.id - `, - bun.Ident(latestConversationStatusesTempTable), - bun.Safe(nowSQL), - ).Scan(ctx, &updatedConversationIDs); // nocollapse + if err := tx.NewUpdate(). + Model((*gtsmodel.Conversation)(nil)). + SetColumn("last_status_id", "?", bun.Ident("latest_conversation_statuses.id")). + SetColumn("updated_at", "?", bun.Safe(nowSQL)). + TableExpr("? AS ?", bun.Ident(latestConversationStatusesTempTable), bun.Ident("latest_conversation_statuses")). + Where("?TableAlias.? = ?", bun.Ident("id"), bun.Ident("latest_conversation_statuses.conversation_id")). + Where("? IS NOT NULL", bun.Ident("latest_conversation_statuses.id")). + Returning("?TableName.?", bun.Ident("id")). + Scan(ctx, &updatedConversationIDs); // nocollapse err != nil { return gtserror.Newf("error rolling back last status for conversation while deleting status %s: %w", statusID, err) } From 45114ffe7e9b3ebc3a81f011db2332173cda4284 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 23:32:21 -0700 Subject: [PATCH 25/35] Convert latestConversationStatusesTempTable raw query partially to Bun --- internal/db/bundb/conversation.go | 44 ++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 95b322377d..592f7b2920 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -344,6 +344,8 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat return gtserror.Newf("error deleting conversation-to-status links while deleting status %s: %w", statusID, err) } + // Note: Bun doesn't currently support CREATE TABLE … AS SELECT … so we need to use raw queries here. + // Create a temporary table with all statuses other than the deleted status // in each conversation for which the deleted status is the last status // (if there are such statuses). @@ -374,20 +376,36 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // for which the deleted status is the last status (if there is such a status). latestConversationStatusesTempTable := "latest_conversation_statuses_" + id.NewULID() if _, err := tx.NewRaw( - ` - CREATE TEMPORARY TABLE ?0 AS - SELECT - conversation_statuses.conversation_id, - conversation_statuses.id - FROM ?1 conversation_statuses - LEFT JOIN ?1 later_statuses - ON conversation_statuses.conversation_id = later_statuses.conversation_id - AND later_statuses.created_at > conversation_statuses.created_at - WHERE later_statuses.id IS NULL - `, + "CREATE TEMPORARY TABLE ? AS ?", bun.Ident(latestConversationStatusesTempTable), - bun.Ident(conversationStatusesTempTable), - ).Exec(ctx); // nocollapse + tx.NewSelect(). + Column( + "conversation_statuses.conversation_id", + "conversation_statuses.id", + ). + TableExpr( + "? AS ?", + bun.Ident(conversationStatusesTempTable), + bun.Ident("conversation_statuses"), + ). + Join( + "LEFT JOIN ? AS ?", + bun.Ident(conversationStatusesTempTable), + bun.Ident("later_statuses"), + ). + JoinOn( + "? = ?", + bun.Ident("conversation_statuses.conversation_id"), + bun.Ident("later_statuses.conversation_id"), + ). + JoinOn( + "? > ?", + bun.Ident("later_statuses.created_at"), + bun.Ident("conversation_statuses.created_at"), + ). + Where("? IS NULL", bun.Ident("later_statuses.id")), + ). + Exec(ctx); // nocollapse err != nil { return gtserror.Newf("error creating latestConversationStatusesTempTable while deleting status %s: %w", statusID, err) } From 041584bdf5dce561586ebadd94b2087cdabb0c55 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sun, 14 Jul 2024 23:50:01 -0700 Subject: [PATCH 26/35] Convert conversationStatusesTempTable raw query partially to Bun --- internal/db/bundb/conversation.go | 54 ++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index 592f7b2920..a49b59ca38 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -351,23 +351,45 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat // (if there are such statuses). conversationStatusesTempTable := "conversation_statuses_" + id.NewULID() if _, err := tx.NewRaw( - ` - CREATE TEMPORARY TABLE ?0 AS - SELECT - conversations.id conversation_id, - conversation_to_statuses.status_id id, - statuses.created_at - FROM conversations - LEFT JOIN conversation_to_statuses - ON conversations.id = conversation_to_statuses.conversation_id - AND conversation_to_statuses.status_id != ?1 - LEFT JOIN statuses - ON conversation_to_statuses.status_id = statuses.id - WHERE conversations.last_status_id = ?1 - `, + "CREATE TEMPORARY TABLE ? AS ?", bun.Ident(conversationStatusesTempTable), - statusID, - ).Exec(ctx); // nocollapse + tx.NewSelect(). + ColumnExpr( + "? AS ?", + bun.Ident("conversations.id"), + bun.Ident("conversation_id"), + ). + ColumnExpr( + "? AS ?", + bun.Ident("conversation_to_statuses.status_id"), + bun.Ident("id"), + ). + Column("statuses.created_at"). + Table("conversations"). + Join("LEFT JOIN ?", bun.Ident("conversation_to_statuses")). + JoinOn( + "? = ?", + bun.Ident("conversations.id"), + bun.Ident("conversation_to_statuses.conversation_id"), + ). + JoinOn( + "? != ?", + bun.Ident("conversation_to_statuses.status_id"), + statusID, + ). + Join("LEFT JOIN ?", bun.Ident("statuses")). + JoinOn( + "? = ?", + bun.Ident("conversation_to_statuses.status_id"), + bun.Ident("statuses.id"), + ). + Where( + "? = ?", + bun.Ident("conversations.last_status_id"), + statusID, + ), + ). + Exec(ctx); // nocollapse err != nil { return gtserror.Newf("error creating conversationStatusesTempTable while deleting status %s: %w", statusID, err) } From b26d10487e1cd50031cf51767bd2dc43a12d50d5 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Mon, 15 Jul 2024 01:34:49 -0700 Subject: [PATCH 27/35] Rename field used to store result of MaxDirectStatusID --- internal/processing/conversations/migrate.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go index d82d62165e..83d4b8a0ca 100644 --- a/internal/processing/conversations/migrate.go +++ b/internal/processing/conversations/migrate.go @@ -33,8 +33,8 @@ const advancedMigrationID = "20240611190733_add_conversations" const statusBatchSize = 100 type AdvancedMigrationState struct { - MinID string - MaxID string + MinID string + MaxIDInclusive string } func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { @@ -64,7 +64,7 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { // Find the max ID of all existing statuses. // This will be the last one we migrate; // newer ones will be handled by the normal conversation flow. - state.MaxID, err = p.state.DB.MaxDirectStatusID(ctx) + state.MaxIDInclusive, err = p.state.DB.MaxDirectStatusID(ctx) if err != nil { return gtserror.Newf("couldn't get max DM status ID for migration: %w", err) } @@ -89,7 +89,7 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { // and update conversations for each in order. for { // Get status IDs for this batch. - statusIDs, err := p.state.DB.GetDirectStatusIDsBatch(ctx, state.MinID, state.MaxID, statusBatchSize) + statusIDs, err := p.state.DB.GetDirectStatusIDsBatch(ctx, state.MinID, state.MaxIDInclusive, statusBatchSize) if err != nil { return gtserror.Newf("couldn't get DM status ID batch for migration: %w", err) } From 284b7303bb8bc83f8334a608fd2eec716fe04af5 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sat, 20 Jul 2024 08:40:38 -0700 Subject: [PATCH 28/35] Move advanced migrations to their own tiny processor --- cmd/gotosocial/action/server/server.go | 4 +- cmd/gotosocial/action/testrig/testrig.go | 4 +- .../advancedmigrations/advancedmigrations.go | 48 +++++++++++++++++++ internal/processing/processor.go | 43 ++++++++++------- 4 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 internal/processing/advancedmigrations/advancedmigrations.go diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 92e6d79b97..4758042187 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -291,8 +291,8 @@ var Start action.GTSAction = func(ctx context.Context) error { } // Run advanced migrations. - if err := processor.Conversations().MigrateDMsToConversations(ctx); err != nil { - return fmt.Errorf("error running conversations advanced migration: %w", err) + if err := processor.AdvancedMigrations().Migrate(ctx); err != nil { + return err } /* diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index 603d06981f..580cb37d68 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -205,8 +205,8 @@ var Start action.GTSAction = func(ctx context.Context) error { } // Run advanced migrations. - if err := processor.Conversations().MigrateDMsToConversations(ctx); err != nil { - return fmt.Errorf("error running conversations advanced migration: %w", err) + if err := processor.AdvancedMigrations().Migrate(ctx); err != nil { + return err } /* diff --git a/internal/processing/advancedmigrations/advancedmigrations.go b/internal/processing/advancedmigrations/advancedmigrations.go new file mode 100644 index 0000000000..3f18765397 --- /dev/null +++ b/internal/processing/advancedmigrations/advancedmigrations.go @@ -0,0 +1,48 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package advancedmigrations + +import ( + "context" + "fmt" + + "github.com/superseriousbusiness/gotosocial/internal/processing/conversations" +) + +// Processor holds references to any other processor that has migrations to run. +type Processor struct { + conversations *conversations.Processor +} + +func New( + conversations *conversations.Processor, +) Processor { + return Processor{ + conversations: conversations, + } +} + +// Migrate runs all advanced migrations. +// Errors should be in the same format thrown by other server or testrig startup failures. +func (p *Processor) Migrate(ctx context.Context) error { + if err := p.conversations.MigrateDMsToConversations(ctx); err != nil { + return fmt.Errorf("error running conversations advanced migration: %w", err) + } + + return nil +} diff --git a/internal/processing/processor.go b/internal/processing/processor.go index b019af4131..a07df76e17 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -27,6 +27,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing/account" "github.com/superseriousbusiness/gotosocial/internal/processing/admin" + "github.com/superseriousbusiness/gotosocial/internal/processing/advancedmigrations" "github.com/superseriousbusiness/gotosocial/internal/processing/common" "github.com/superseriousbusiness/gotosocial/internal/processing/conversations" "github.com/superseriousbusiness/gotosocial/internal/processing/fedi" @@ -71,23 +72,24 @@ type Processor struct { SUB-PROCESSORS */ - account account.Processor - admin admin.Processor - conversations conversations.Processor - fedi fedi.Processor - filtersv1 filtersv1.Processor - filtersv2 filtersv2.Processor - list list.Processor - markers markers.Processor - media media.Processor - polls polls.Processor - report report.Processor - search search.Processor - status status.Processor - stream stream.Processor - timeline timeline.Processor - user user.Processor - workers workers.Processor + account account.Processor + admin admin.Processor + advancedmigrations advancedmigrations.Processor + conversations conversations.Processor + fedi fedi.Processor + filtersv1 filtersv1.Processor + filtersv2 filtersv2.Processor + list list.Processor + markers markers.Processor + media media.Processor + polls polls.Processor + report report.Processor + search search.Processor + status status.Processor + stream stream.Processor + timeline timeline.Processor + user user.Processor + workers workers.Processor } func (p *Processor) Account() *account.Processor { @@ -98,6 +100,10 @@ func (p *Processor) Admin() *admin.Processor { return &p.admin } +func (p *Processor) AdvancedMigrations() *advancedmigrations.Processor { + return &p.advancedmigrations +} + func (p *Processor) Conversations() *conversations.Processor { return &p.conversations } @@ -207,6 +213,9 @@ func NewProcessor( processor.status = status.New(state, &common, &processor.polls, federator, converter, filter, parseMentionFunc) processor.user = user.New(state, converter, oauthServer, emailSender) + // The advanced migrations processor sequences advanced migrations from all other processors. + processor.advancedmigrations = advancedmigrations.New(&processor.conversations) + // Workers processor handles asynchronous // worker jobs; instantiate it separately // and pass subset of sub processors it needs. From adbd7edbf08eaa6672544b04062bd669b7c149ff Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sat, 20 Jul 2024 09:26:18 -0700 Subject: [PATCH 29/35] Catch up util function name with main --- internal/processing/conversations/read_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/processing/conversations/read_test.go b/internal/processing/conversations/read_test.go index e062fcd857..ebd8f7fe5a 100644 --- a/internal/processing/conversations/read_test.go +++ b/internal/processing/conversations/read_test.go @@ -26,7 +26,7 @@ import ( func (suite *ConversationsTestSuite) TestRead() { conversation := suite.NewTestConversation(suite.testAccount, 0) - suite.False(util.PtrValueOr(conversation.Read, false)) + suite.False(util.PtrOrValue(conversation.Read, false)) apiConversation, err := suite.conversationsProcessor.Read(context.Background(), suite.testAccount, conversation.ID) if suite.NoError(err) { suite.False(apiConversation.Unread) From 1b853a62543a6c2412a070ababc5929ce6f3f715 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 23 Jul 2024 09:05:04 -0700 Subject: [PATCH 30/35] =?UTF-8?q?Remove=20json.=E2=80=A6=20wrappers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/gtsmodel/advancedmigration.go | 18 +----------------- internal/processing/conversations/migrate.go | 8 ++++---- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/internal/gtsmodel/advancedmigration.go b/internal/gtsmodel/advancedmigration.go index cb459f5029..d9ce9d5439 100644 --- a/internal/gtsmodel/advancedmigration.go +++ b/internal/gtsmodel/advancedmigration.go @@ -18,7 +18,6 @@ package gtsmodel import ( - "encoding/json" "time" ) @@ -28,21 +27,6 @@ type AdvancedMigration struct { ID string `bun:",pk,nullzero,notnull,unique"` // id of this migration (preassigned, not a ULID) CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated - StateJSON string `bun:",nullzero"` // JSON dump of the migration state + StateJSON []byte `bun:",nullzero"` // JSON dump of the migration state Finished *bool `bun:",nullzero,notnull,default:false"` // has this migration finished? } - -func AdvancedMigrationLoad[State any](a *AdvancedMigration) (State, error) { - var state State - err := json.Unmarshal([]byte(a.StateJSON), &state) - return state, err -} - -func AdvancedMigrationStore[State any](a *AdvancedMigration, state State) error { - bytes, err := json.Marshal(state) - if err != nil { - return err - } - a.StateJSON = string(bytes) - return nil -} diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go index 83d4b8a0ca..360e31a4c1 100644 --- a/internal/processing/conversations/migrate.go +++ b/internal/processing/conversations/migrate.go @@ -19,6 +19,7 @@ package conversations import ( "context" + "encoding/json" "errors" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -52,8 +53,7 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { return nil } // Otherwise, pick up where we left off. - state, err = gtsmodel.AdvancedMigrationLoad[AdvancedMigrationState](advancedMigration) - if err != nil { + if err := json.Unmarshal(advancedMigration.StateJSON, &state); err != nil { // This should never happen. return gtserror.Newf("couldn't deserialize advanced migration state from JSON: %w", err) } @@ -74,7 +74,7 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { ID: advancedMigrationID, Finished: util.Ptr(false), } - if err := gtsmodel.AdvancedMigrationStore(advancedMigration, state); err != nil { + if advancedMigration.StateJSON, err = json.Marshal(state); err != nil { // This should never happen. return gtserror.Newf("couldn't serialize advanced migration state to JSON: %w", err) } @@ -115,7 +115,7 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { // Save the migration state with the new min ID. state.MinID = statusIDs[len(statusIDs)-1] - if err := gtsmodel.AdvancedMigrationStore(advancedMigration, state); err != nil { + if advancedMigration.StateJSON, err = json.Marshal(state); err != nil { // This should never happen. return gtserror.Newf("couldn't serialize advanced migration state to JSON: %w", err) } From caefe9b31b2c62d63ecca835ea42fe4daf8887e6 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 23 Jul 2024 09:06:53 -0700 Subject: [PATCH 31/35] Remove redundant check --- internal/processing/conversations/migrate.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go index 360e31a4c1..6abe3bef4a 100644 --- a/internal/processing/conversations/migrate.go +++ b/internal/processing/conversations/migrate.go @@ -107,9 +107,7 @@ func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { // Update conversations for each status. Don't generate notifications. for _, status := range statuses { if _, err := p.UpdateConversationsForStatus(ctx, status); err != nil { - if err != nil { - return gtserror.Newf("couldn't update conversations for status %s during migration: %w", status.ID, err) - } + return gtserror.Newf("couldn't update conversations for status %s during migration: %w", status.ID, err) } } From ff4feb908a4104d895a573a023086ec5124d9ced Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 23 Jul 2024 09:08:36 -0700 Subject: [PATCH 32/35] Combine error checks --- internal/processing/conversations/migrate.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go index 6abe3bef4a..959ffcca40 100644 --- a/internal/processing/conversations/migrate.go +++ b/internal/processing/conversations/migrate.go @@ -40,10 +40,8 @@ type AdvancedMigrationState struct { func (p *Processor) MigrateDMsToConversations(ctx context.Context) error { advancedMigration, err := p.state.DB.GetAdvancedMigration(ctx, advancedMigrationID) - if err != nil { - if !errors.Is(err, db.ErrNoEntries) { - return gtserror.Newf("couldn't get advanced migration with ID %s: %w", advancedMigrationID, err) - } + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return gtserror.Newf("couldn't get advanced migration with ID %s: %w", advancedMigrationID, err) } state := AdvancedMigrationState{} if advancedMigration != nil { From cf7a6269f3e16d2fe5b5b6497082e7c79510b283 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 23 Jul 2024 09:31:11 -0700 Subject: [PATCH 33/35] Replace map with slice of structs --- internal/processing/conversations/update.go | 18 ++++++++++++++---- internal/processing/workers/surfacetimeline.go | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/processing/conversations/update.go b/internal/processing/conversations/update.go index d57698be2c..413e9fc00d 100644 --- a/internal/processing/conversations/update.go +++ b/internal/processing/conversations/update.go @@ -32,11 +32,17 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/util" ) +// ConversationNotification carries the arguments to processing/stream.Processor.Conversation. +type ConversationNotification struct { + // AccountID of a local account to deliver the notification to. + AccountID string + // Conversation as the notification payload. + Conversation *apimodel.Conversation +} + // UpdateConversationsForStatus updates all conversations related to a status, // and returns a map from local account IDs to conversation notifications that should be sent to them. -func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gtsmodel.Status) (map[string]*apimodel.Conversation, error) { - notifications := map[string]*apimodel.Conversation{} - +func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gtsmodel.Status) ([]ConversationNotification, error) { // We need accounts to be populated for this. if err := p.state.DB.PopulateStatus(ctx, status); err != nil { return nil, err @@ -65,6 +71,7 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt } // Create or update conversations for and send notifications to each local participant. + notifications := make([]ConversationNotification, 0, len(allParticipantsSet)) for _, participant := range allParticipantsSet { if participant.IsRemote() { continue @@ -232,7 +239,10 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt // unless the status was authored by the user who would be notified, // in which case they already know. if status.AccountID != localAccount.ID { - notifications[localAccount.ID] = apiConversation + notifications = append(notifications, ConversationNotification{ + AccountID: localAccount.ID, + Conversation: apiConversation, + }) } } diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go index 5e1432105c..8ac8293ed4 100644 --- a/internal/processing/workers/surfacetimeline.go +++ b/internal/processing/workers/surfacetimeline.go @@ -78,8 +78,8 @@ func (s *Surface) timelineAndNotifyStatus(ctx context.Context, status *gtsmodel. if err != nil { return gtserror.Newf("error updating conversations for status %s: %w", status.ID, err) } - for localAccountID, apiConversation := range notifications { - s.Stream.Conversation(ctx, localAccountID, apiConversation) + for _, notification := range notifications { + s.Stream.Conversation(ctx, notification.AccountID, notification.Conversation) } return nil From 0367bb3fbe116e8342e9bdb9424f0ed7a0d596bd Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 23 Jul 2024 10:27:58 -0700 Subject: [PATCH 34/35] Address processor/type converter comments - Add context info for errors - Extract some common processor code into shared methods - Move conversation eligibility check ahead of populating conversation --- .../processing/conversations/conversations.go | 84 +++++++++++++++++++ internal/processing/conversations/delete.go | 16 +--- internal/processing/conversations/get.go | 26 +++--- internal/processing/conversations/read.go | 36 ++------ internal/processing/conversations/update.go | 32 +++---- internal/typeutils/internaltofrontend.go | 19 ++++- 6 files changed, 136 insertions(+), 77 deletions(-) diff --git a/internal/processing/conversations/conversations.go b/internal/processing/conversations/conversations.go index d42126c230..d957406054 100644 --- a/internal/processing/conversations/conversations.go +++ b/internal/processing/conversations/conversations.go @@ -18,7 +18,15 @@ package conversations import ( + "context" + "errors" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -40,3 +48,79 @@ func New( filter: filter, } } + +const conversationNotFoundHelpText = "conversation not found" + +// getConversationOwnedBy gets a conversation by ID and checks that it is owned by the given account. +func (p *Processor) getConversationOwnedBy( + ctx context.Context, + id string, + requestingAccount *gtsmodel.Account, +) (*gtsmodel.Conversation, gtserror.WithCode) { + // Get the conversation so that we can check its owning account ID. + conversation, err := p.state.DB.GetConversationByID(ctx, id) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return nil, gtserror.NewErrorInternalError( + gtserror.Newf( + "DB error getting conversation %s for account %s: %w", + id, + requestingAccount.ID, + err, + ), + ) + } + if conversation == nil { + return nil, gtserror.NewErrorNotFound( + gtserror.Newf( + "conversation %s not found: %w", + id, + err, + ), + conversationNotFoundHelpText, + ) + } + if conversation.AccountID != requestingAccount.ID { + return nil, gtserror.NewErrorNotFound( + gtserror.Newf( + "conversation %s not owned by account %s: %w", + id, + requestingAccount.ID, + err, + ), + conversationNotFoundHelpText, + ) + } + + return conversation, nil +} + +// getFiltersAndMutes gets the given account's filters and compiled mute list. +func (p *Processor) getFiltersAndMutes( + ctx context.Context, + requestingAccount *gtsmodel.Account, +) ([]*gtsmodel.Filter, *usermute.CompiledUserMuteList, gtserror.WithCode) { + filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) + if err != nil { + return nil, nil, gtserror.NewErrorInternalError( + gtserror.Newf( + "DB error getting filters for account %s: %w", + requestingAccount.ID, + err, + ), + ) + } + + mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), requestingAccount.ID, nil) + if err != nil { + return nil, nil, gtserror.NewErrorInternalError( + gtserror.Newf( + "DB error getting mutes for account %s: %w", + requestingAccount.ID, + err, + ), + ) + } + compiledMutes := usermute.NewCompiledUserMuteList(mutes) + + return filters, compiledMutes, nil +} diff --git a/internal/processing/conversations/delete.go b/internal/processing/conversations/delete.go index 64083bd536..5cbdd00a52 100644 --- a/internal/processing/conversations/delete.go +++ b/internal/processing/conversations/delete.go @@ -19,9 +19,7 @@ package conversations import ( "context" - "errors" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -33,19 +31,13 @@ func (p *Processor) Delete( id string, ) gtserror.WithCode { // Get the conversation so that we can check its owning account ID. - conversation, err := p.state.DB.GetConversationByID(gtscontext.SetBarebones(ctx), id) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return gtserror.NewErrorInternalError(err) - } - if conversation == nil { - return gtserror.NewErrorNotFound(err) - } - if conversation.AccountID != requestingAccount.ID { - return gtserror.NewErrorNotFound(nil) + conversation, errWithCode := p.getConversationOwnedBy(gtscontext.SetBarebones(ctx), id, requestingAccount) + if errWithCode != nil { + return errWithCode } // Delete the conversation. - if err := p.state.DB.DeleteConversationByID(ctx, id); err != nil { + if err := p.state.DB.DeleteConversationByID(ctx, conversation.ID); err != nil { return gtserror.NewErrorInternalError(err) } diff --git a/internal/processing/conversations/get.go b/internal/processing/conversations/get.go index 36d0c33aed..0c7832caed 100644 --- a/internal/processing/conversations/get.go +++ b/internal/processing/conversations/get.go @@ -23,8 +23,6 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" - "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -45,7 +43,13 @@ func (p *Processor) GetAll( page, ) if err != nil && !errors.Is(err, db.ErrNoEntries) { - return nil, gtserror.NewErrorInternalError(err) + return nil, gtserror.NewErrorInternalError( + gtserror.Newf( + "DB error getting conversations for account %s: %w", + requestingAccount.ID, + err, + ), + ) } // Check for empty response. @@ -60,18 +64,10 @@ func (p *Processor) GetAll( items := make([]interface{}, 0, count) - filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) - if err != nil { - err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) - return nil, gtserror.NewErrorInternalError(err) - } - - mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), requestingAccount.ID, nil) - if err != nil { - err = gtserror.Newf("couldn't retrieve mutes for account %s: %w", requestingAccount.ID, err) - return nil, gtserror.NewErrorInternalError(err) + filters, mutes, errWithCode := p.getFiltersAndMutes(ctx, requestingAccount) + if errWithCode != nil { + return nil, errWithCode } - compiledMutes := usermute.NewCompiledUserMuteList(mutes) for _, conversation := range conversations { // Convert conversation to frontend API model. @@ -80,7 +76,7 @@ func (p *Processor) GetAll( conversation, requestingAccount, filters, - compiledMutes, + mutes, ) if err != nil { log.Errorf( diff --git a/internal/processing/conversations/read.go b/internal/processing/conversations/read.go index 5b3630318c..512a004a3b 100644 --- a/internal/processing/conversations/read.go +++ b/internal/processing/conversations/read.go @@ -19,12 +19,8 @@ package conversations import ( "context" - "errors" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" - "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -36,46 +32,32 @@ func (p *Processor) Read( id string, ) (*apimodel.Conversation, gtserror.WithCode) { // Get the conversation, including participating accounts and last status. - conversation, err := p.state.DB.GetConversationByID(ctx, id) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return nil, gtserror.NewErrorInternalError(err) - } - if conversation == nil { - return nil, gtserror.NewErrorNotFound(err) - } - if conversation.AccountID != requestingAccount.ID { - return nil, gtserror.NewErrorNotFound(nil) + conversation, errWithCode := p.getConversationOwnedBy(ctx, id, requestingAccount) + if errWithCode != nil { + return nil, errWithCode } // Mark the conversation as read. conversation.Read = util.Ptr(true) if err := p.state.DB.UpsertConversation(ctx, conversation, "read"); err != nil { - err = gtserror.Newf("db error updating conversation %s: %w", id, err) + err = gtserror.Newf("DB error updating conversation %s: %w", id, err) return nil, gtserror.NewErrorInternalError(err) } - filters, err := p.state.DB.GetFiltersForAccountID(ctx, requestingAccount.ID) - if err != nil { - err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) - return nil, gtserror.NewErrorInternalError(err) - } - - mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), requestingAccount.ID, nil) - if err != nil { - err = gtserror.Newf("couldn't retrieve mutes for account %s: %w", requestingAccount.ID, err) - return nil, gtserror.NewErrorInternalError(err) + filters, mutes, errWithCode := p.getFiltersAndMutes(ctx, requestingAccount) + if errWithCode != nil { + return nil, errWithCode } - compiledMutes := usermute.NewCompiledUserMuteList(mutes) apiConversation, err := p.converter.ConversationToAPIConversation( ctx, conversation, requestingAccount, filters, - compiledMutes, + mutes, ) if err != nil { - err = gtserror.Newf("db error converting conversation %s to API representation: %w", id, err) + err = gtserror.Newf("error converting conversation %s to API representation: %w", id, err) return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/processing/conversations/update.go b/internal/processing/conversations/update.go index 413e9fc00d..7445994ae6 100644 --- a/internal/processing/conversations/update.go +++ b/internal/processing/conversations/update.go @@ -24,8 +24,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" - "github.com/superseriousbusiness/gotosocial/internal/filter/usermute" - "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -43,11 +42,6 @@ type ConversationNotification struct { // UpdateConversationsForStatus updates all conversations related to a status, // and returns a map from local account IDs to conversation notifications that should be sent to them. func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gtsmodel.Status) ([]ConversationNotification, error) { - // We need accounts to be populated for this. - if err := p.state.DB.PopulateStatus(ctx, status); err != nil { - return nil, err - } - if status.Visibility != gtsmodel.VisibilityDirect { // Only DMs are considered part of conversations. return nil, nil @@ -63,6 +57,11 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt return nil, nil } + // We need accounts to be populated for this. + if err := p.state.DB.PopulateStatus(ctx, status); err != nil { + return nil, gtserror.Newf("DB error populating status %s: %w", status.ID, err) + } + // The account which authored the status plus all mentioned accounts. allParticipantsSet := make(map[string]*gtsmodel.Account, 1+len(status.Mentions)) allParticipantsSet[status.AccountID] = status.Account @@ -93,27 +92,20 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt continue } - // TODO: (Vyr) find a prettier way to do this // Is the status filtered or muted for this user? - filters, err := p.state.DB.GetFiltersForAccountID(ctx, localAccount.ID) - if err != nil { - log.Errorf(ctx, "error retrieving filters for account %s: %v", localAccount.ID, err) - continue - } - mutes, err := p.state.DB.GetAccountMutes(gtscontext.SetBarebones(ctx), localAccount.ID, nil) - if err != nil { - log.Errorf(ctx, "error retrieving mutes for account %s: %v", localAccount.ID, err) + // Converting the status to an API status runs the filter/mute checks. + filters, mutes, errWithCode := p.getFiltersAndMutes(ctx, localAccount) + if errWithCode != nil { + log.Error(ctx, errWithCode) continue } - compiledMutes := usermute.NewCompiledUserMuteList(mutes) - // Converting the status to an API status runs the filter/mute checks. _, err = p.converter.StatusToAPIStatus( ctx, status, localAccount, statusfilter.FilterContextNotifications, filters, - compiledMutes, + mutes, ) if err != nil { // If the status matched a hide filter, skip processing it for this account. @@ -218,7 +210,7 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt conversation, localAccount, filters, - compiledMutes, + mutes, ) if err != nil { // If the conversation's last status matched a hide filter, skip it. diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 8173fea045..cffd5affa1 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -1697,7 +1697,12 @@ func (c *Converter) ConversationToAPIConversation( var apiAccount *apimodel.Account blocked, err := c.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, account.ID) if err != nil { - return nil, err + return nil, gtserror.Newf( + "DB error checking blocks between accounts %s and %s: %w", + requestingAccount.ID, + account.ID, + err, + ) } if blocked || account.IsSuspended() { apiAccount, err = c.AccountToAPIAccountBlocked(ctx, account) @@ -1705,7 +1710,11 @@ func (c *Converter) ConversationToAPIConversation( apiAccount, err = c.AccountToAPIAccountPublic(ctx, account) } if err != nil { - return nil, err + return nil, gtserror.Newf( + "error converting account %s to API representation: %w", + account.ID, + err, + ) } apiConversation.Accounts = append(apiConversation.Accounts, *apiAccount) } @@ -1720,7 +1729,11 @@ func (c *Converter) ConversationToAPIConversation( mutes, ) if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { - return nil, err + return nil, gtserror.Newf( + "error converting status %s to API representation: %w", + conversation.LastStatus.ID, + err, + ) } } From 6b3dfde03e1821a20e855fd702832f449069b3f2 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Tue, 23 Jul 2024 11:39:40 -0700 Subject: [PATCH 35/35] Add error context when dropping temp tables --- internal/db/bundb/conversation.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/db/bundb/conversation.go b/internal/db/bundb/conversation.go index a49b59ca38..1a3958a798 100644 --- a/internal/db/bundb/conversation.go +++ b/internal/db/bundb/conversation.go @@ -473,7 +473,12 @@ func (c *conversationDB) DeleteStatusFromConversations(ctx context.Context, stat latestConversationStatusesTempTable, } { if _, err := tx.NewDropTable().Table(tempTable).Exec(ctx); err != nil { - return err + return gtserror.Newf( + "error dropping temporary table %s after deleting status %s: %w", + tempTable, + statusID, + err, + ) } }