From 18325393b9b858cc9b0b3b8421219c3980d2b85c Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Sat, 20 May 2023 20:57:16 +1000 Subject: [PATCH] #202 channel members --- clienter_mock_test.go | 102 +++++++++++++++++++---------- go.mod | 1 + go.sum | 3 +- internal/chunk/chunktype_string.go | 6 +- slackdump.go | 7 +- stream.go | 47 +++++++++++-- 6 files changed, 117 insertions(+), 49 deletions(-) diff --git a/clienter_mock_test.go b/clienter_mock_test.go index bbb2a66c..69c5ffef 100644 --- a/clienter_mock_test.go +++ b/clienter_mock_test.go @@ -13,31 +13,31 @@ import ( slack "github.com/slack-go/slack" ) -// MockSlacker is a mock of Slacker interface. -type MockSlacker struct { +// Mockslacker is a mock of slacker interface. +type Mockslacker struct { ctrl *gomock.Controller - recorder *MockSlackerMockRecorder + recorder *MockslackerMockRecorder } -// MockSlackerMockRecorder is the mock recorder for MockSlacker. -type MockSlackerMockRecorder struct { - mock *MockSlacker +// MockslackerMockRecorder is the mock recorder for Mockslacker. +type MockslackerMockRecorder struct { + mock *Mockslacker } -// NewMockSlacker creates a new mock instance. -func NewMockSlacker(ctrl *gomock.Controller) *MockSlacker { - mock := &MockSlacker{ctrl: ctrl} - mock.recorder = &MockSlackerMockRecorder{mock} +// NewMockslacker creates a new mock instance. +func NewMockslacker(ctrl *gomock.Controller) *Mockslacker { + mock := &Mockslacker{ctrl: ctrl} + mock.recorder = &MockslackerMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSlacker) EXPECT() *MockSlackerMockRecorder { +func (m *Mockslacker) EXPECT() *MockslackerMockRecorder { return m.recorder } // AuthTestContext mocks base method. -func (m *MockSlacker) AuthTestContext(arg0 context.Context) (*slack.AuthTestResponse, error) { +func (m *Mockslacker) AuthTestContext(arg0 context.Context) (*slack.AuthTestResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AuthTestContext", arg0) ret0, _ := ret[0].(*slack.AuthTestResponse) @@ -46,13 +46,13 @@ func (m *MockSlacker) AuthTestContext(arg0 context.Context) (*slack.AuthTestResp } // AuthTestContext indicates an expected call of AuthTestContext. -func (mr *MockSlackerMockRecorder) AuthTestContext(arg0 interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) AuthTestContext(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthTestContext", reflect.TypeOf((*MockSlacker)(nil).AuthTestContext), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthTestContext", reflect.TypeOf((*Mockslacker)(nil).AuthTestContext), arg0) } // GetConversationHistoryContext mocks base method. -func (m *MockSlacker) GetConversationHistoryContext(ctx context.Context, params *slack.GetConversationHistoryParameters) (*slack.GetConversationHistoryResponse, error) { +func (m *Mockslacker) GetConversationHistoryContext(ctx context.Context, params *slack.GetConversationHistoryParameters) (*slack.GetConversationHistoryResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetConversationHistoryContext", ctx, params) ret0, _ := ret[0].(*slack.GetConversationHistoryResponse) @@ -61,13 +61,13 @@ func (m *MockSlacker) GetConversationHistoryContext(ctx context.Context, params } // GetConversationHistoryContext indicates an expected call of GetConversationHistoryContext. -func (mr *MockSlackerMockRecorder) GetConversationHistoryContext(ctx, params interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) GetConversationHistoryContext(ctx, params interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationHistoryContext", reflect.TypeOf((*MockSlacker)(nil).GetConversationHistoryContext), ctx, params) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationHistoryContext", reflect.TypeOf((*Mockslacker)(nil).GetConversationHistoryContext), ctx, params) } // GetConversationInfoContext mocks base method. -func (m *MockSlacker) GetConversationInfoContext(ctx context.Context, input *slack.GetConversationInfoInput) (*slack.Channel, error) { +func (m *Mockslacker) GetConversationInfoContext(ctx context.Context, input *slack.GetConversationInfoInput) (*slack.Channel, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetConversationInfoContext", ctx, input) ret0, _ := ret[0].(*slack.Channel) @@ -76,13 +76,13 @@ func (m *MockSlacker) GetConversationInfoContext(ctx context.Context, input *sla } // GetConversationInfoContext indicates an expected call of GetConversationInfoContext. -func (mr *MockSlackerMockRecorder) GetConversationInfoContext(ctx, input interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) GetConversationInfoContext(ctx, input interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationInfoContext", reflect.TypeOf((*MockSlacker)(nil).GetConversationInfoContext), ctx, input) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationInfoContext", reflect.TypeOf((*Mockslacker)(nil).GetConversationInfoContext), ctx, input) } // GetConversationRepliesContext mocks base method. -func (m *MockSlacker) GetConversationRepliesContext(ctx context.Context, params *slack.GetConversationRepliesParameters) ([]slack.Message, bool, string, error) { +func (m *Mockslacker) GetConversationRepliesContext(ctx context.Context, params *slack.GetConversationRepliesParameters) ([]slack.Message, bool, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetConversationRepliesContext", ctx, params) ret0, _ := ret[0].([]slack.Message) @@ -93,13 +93,13 @@ func (m *MockSlacker) GetConversationRepliesContext(ctx context.Context, params } // GetConversationRepliesContext indicates an expected call of GetConversationRepliesContext. -func (mr *MockSlackerMockRecorder) GetConversationRepliesContext(ctx, params interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) GetConversationRepliesContext(ctx, params interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationRepliesContext", reflect.TypeOf((*MockSlacker)(nil).GetConversationRepliesContext), ctx, params) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationRepliesContext", reflect.TypeOf((*Mockslacker)(nil).GetConversationRepliesContext), ctx, params) } // GetConversationsContext mocks base method. -func (m *MockSlacker) GetConversationsContext(ctx context.Context, params *slack.GetConversationsParameters) ([]slack.Channel, string, error) { +func (m *Mockslacker) GetConversationsContext(ctx context.Context, params *slack.GetConversationsParameters) ([]slack.Channel, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetConversationsContext", ctx, params) ret0, _ := ret[0].([]slack.Channel) @@ -109,13 +109,13 @@ func (m *MockSlacker) GetConversationsContext(ctx context.Context, params *slack } // GetConversationsContext indicates an expected call of GetConversationsContext. -func (mr *MockSlackerMockRecorder) GetConversationsContext(ctx, params interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) GetConversationsContext(ctx, params interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationsContext", reflect.TypeOf((*MockSlacker)(nil).GetConversationsContext), ctx, params) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationsContext", reflect.TypeOf((*Mockslacker)(nil).GetConversationsContext), ctx, params) } // GetStarredContext mocks base method. -func (m *MockSlacker) GetStarredContext(ctx context.Context, params slack.StarsParameters) ([]slack.StarredItem, *slack.Paging, error) { +func (m *Mockslacker) GetStarredContext(ctx context.Context, params slack.StarsParameters) ([]slack.StarredItem, *slack.Paging, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetStarredContext", ctx, params) ret0, _ := ret[0].([]slack.StarredItem) @@ -125,13 +125,29 @@ func (m *MockSlacker) GetStarredContext(ctx context.Context, params slack.StarsP } // GetStarredContext indicates an expected call of GetStarredContext. -func (mr *MockSlackerMockRecorder) GetStarredContext(ctx, params interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) GetStarredContext(ctx, params interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStarredContext", reflect.TypeOf((*MockSlacker)(nil).GetStarredContext), ctx, params) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStarredContext", reflect.TypeOf((*Mockslacker)(nil).GetStarredContext), ctx, params) +} + +// GetUsersInConversationContext mocks base method. +func (m *Mockslacker) GetUsersInConversationContext(ctx context.Context, params *slack.GetUsersInConversationParameters) ([]string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUsersInConversationContext", ctx, params) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetUsersInConversationContext indicates an expected call of GetUsersInConversationContext. +func (mr *MockslackerMockRecorder) GetUsersInConversationContext(ctx, params interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersInConversationContext", reflect.TypeOf((*Mockslacker)(nil).GetUsersInConversationContext), ctx, params) } // GetUsersPaginated mocks base method. -func (m *MockSlacker) GetUsersPaginated(options ...slack.GetUsersOption) slack.UserPagination { +func (m *Mockslacker) GetUsersPaginated(options ...slack.GetUsersOption) slack.UserPagination { m.ctrl.T.Helper() varargs := []interface{}{} for _, a := range options { @@ -143,13 +159,13 @@ func (m *MockSlacker) GetUsersPaginated(options ...slack.GetUsersOption) slack.U } // GetUsersPaginated indicates an expected call of GetUsersPaginated. -func (mr *MockSlackerMockRecorder) GetUsersPaginated(options ...interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) GetUsersPaginated(options ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersPaginated", reflect.TypeOf((*MockSlacker)(nil).GetUsersPaginated), options...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersPaginated", reflect.TypeOf((*Mockslacker)(nil).GetUsersPaginated), options...) } // ListBookmarks mocks base method. -func (m *MockSlacker) ListBookmarks(channelID string) ([]slack.Bookmark, error) { +func (m *Mockslacker) ListBookmarks(channelID string) ([]slack.Bookmark, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListBookmarks", channelID) ret0, _ := ret[0].([]slack.Bookmark) @@ -158,9 +174,9 @@ func (m *MockSlacker) ListBookmarks(channelID string) ([]slack.Bookmark, error) } // ListBookmarks indicates an expected call of ListBookmarks. -func (mr *MockSlackerMockRecorder) ListBookmarks(channelID interface{}) *gomock.Call { +func (mr *MockslackerMockRecorder) ListBookmarks(channelID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBookmarks", reflect.TypeOf((*MockSlacker)(nil).ListBookmarks), channelID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBookmarks", reflect.TypeOf((*Mockslacker)(nil).ListBookmarks), channelID) } // mockClienter is a mock of clienter interface. @@ -329,6 +345,22 @@ func (mr *mockClienterMockRecorder) GetUsersContext(ctx interface{}, options ... return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersContext", reflect.TypeOf((*mockClienter)(nil).GetUsersContext), varargs...) } +// GetUsersInConversationContext mocks base method. +func (m *mockClienter) GetUsersInConversationContext(ctx context.Context, params *slack.GetUsersInConversationParameters) ([]string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUsersInConversationContext", ctx, params) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetUsersInConversationContext indicates an expected call of GetUsersInConversationContext. +func (mr *mockClienterMockRecorder) GetUsersInConversationContext(ctx, params interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersInConversationContext", reflect.TypeOf((*mockClienter)(nil).GetUsersInConversationContext), ctx, params) +} + // GetUsersPaginated mocks base method. func (m *mockClienter) GetUsersPaginated(options ...slack.GetUsersOption) slack.UserPagination { m.ctrl.T.Helper() diff --git a/go.mod b/go.mod index 560e5638..862775ee 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/schollz/progressbar/v3 v3.13.0 github.com/slack-go/slack v0.12.2 github.com/stretchr/testify v1.8.2 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/term v0.7.0 golang.org/x/text v0.9.0 golang.org/x/time v0.3.0 diff --git a/go.sum b/go.sum index 58efbe80..0e2aee66 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,6 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rusq/chttp v1.0.1 h1:j3WE7+jQE9Rgw0E6mGMje8HxMCv09QRkKvR0oZ1R2vY= -github.com/rusq/chttp v1.0.1/go.mod h1:9H/mMp/iUc4xDkSOY0rL6ecxd/YyaW7zE9GhR+mZfRg= github.com/rusq/chttp v1.0.2 h1:bc8FTKE/l318Kie3sb2KrGi7Fu5tSDQY+JiXMsq4fO8= github.com/rusq/chttp v1.0.2/go.mod h1:bmuoQMUFs9fmigUmT7xbp8s0rHyzUrf7+78yLklr1so= github.com/rusq/dlog v1.4.0 h1:64oHTSzHjzG6TXKvMbPKQzvqADCZRn6XgAWnp7ASr5k= @@ -113,6 +111,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/chunk/chunktype_string.go b/internal/chunk/chunktype_string.go index b6c03f71..7056a21d 100644 --- a/internal/chunk/chunktype_string.go +++ b/internal/chunk/chunktype_string.go @@ -15,11 +15,13 @@ func _() { _ = x[CChannels-4] _ = x[CChannelInfo-5] _ = x[CWorkspaceInfo-6] + _ = x[CStarredItems-7] + _ = x[CBookmarks-8] } -const _ChunkType_name = "MessagesThreadMessagesFilesUsersChannelsChannelInfoWorkspaceInfo" +const _ChunkType_name = "MessagesThreadMessagesFilesUsersChannelsChannelInfoWorkspaceInfoStarredItemsBookmarks" -var _ChunkType_index = [...]uint8{0, 8, 22, 27, 32, 40, 51, 64} +var _ChunkType_index = [...]uint8{0, 8, 22, 27, 32, 40, 51, 64, 76, 85} func (i ChunkType) String() string { if i >= ChunkType(len(_ChunkType_index)-1) { diff --git a/slackdump.go b/slackdump.go index 893a70ac..73c50a8a 100644 --- a/slackdump.go +++ b/slackdump.go @@ -37,8 +37,8 @@ type Session struct { // WorkspaceInfo is an type alias for [slack.AuthTestResponse]. type WorkspaceInfo = slack.AuthTestResponse -// Slacker is the interface with some functions of slack.Client. -type Slacker interface { +// slacker is the interface with some functions of slack.Client. +type slacker interface { AuthTestContext(context.Context) (response *slack.AuthTestResponse, err error) GetConversationInfoContext(ctx context.Context, input *slack.GetConversationInfoInput) (*slack.Channel, error) GetConversationHistoryContext(ctx context.Context, params *slack.GetConversationHistoryParameters) (*slack.GetConversationHistoryResponse, error) @@ -46,13 +46,14 @@ type Slacker interface { GetConversationsContext(ctx context.Context, params *slack.GetConversationsParameters) (channels []slack.Channel, nextCursor string, err error) GetStarredContext(ctx context.Context, params slack.StarsParameters) ([]slack.StarredItem, *slack.Paging, error) GetUsersPaginated(options ...slack.GetUsersOption) slack.UserPagination + GetUsersInConversationContext(ctx context.Context, params *slack.GetUsersInConversationParameters) ([]string, string, error) ListBookmarks(channelID string) ([]slack.Bookmark, error) } // clienter is the interface with some functions of slack.Client with the sole // purpose of mocking in tests (see client_mock.go) type clienter interface { - Slacker + slacker GetFile(downloadURL string, writer io.Writer) error GetUsersContext(ctx context.Context, options ...slack.GetUsersOption) ([]slack.User, error) GetEmojiContext(ctx context.Context) (map[string]string, error) diff --git a/stream.go b/stream.go index 187a6afa..12b62893 100644 --- a/stream.go +++ b/stream.go @@ -9,6 +9,7 @@ import ( "time" "github.com/slack-go/slack" + "golang.org/x/sync/errgroup" "golang.org/x/time/rate" "github.com/rusq/slackdump/v2/internal/chunk/state" @@ -36,7 +37,7 @@ const ( // use. type Stream struct { oldest, latest time.Time - client Slacker + client slacker limits rateLimits chanCache *chanCache resultFn []func(sr StreamResult) error @@ -133,7 +134,7 @@ func OptState(s *state.State) StreamOption { // NewStream creates a new Stream instance that allows to stream different // slack entities. -func NewStream(cl Slacker, l *Limits, opts ...StreamOption) *Stream { +func NewStream(cl slacker, l *Limits, opts ...StreamOption) *Stream { cs := &Stream{ client: cl, limits: limits(l), @@ -551,13 +552,45 @@ func (cs *Stream) channelInfo(ctx context.Context, proc processor.ChannelInforme var info *slack.Channel if info = cs.chanCache.get(channelID); info == nil { if err := network.WithRetry(ctx, cs.limits.channels, cs.limits.tier.Tier3.Retries, func() error { - var err error - info, err = cs.client.GetConversationInfoContext(ctx, &slack.GetConversationInfoInput{ - ChannelID: channelID, + var eg errgroup.Group + + eg.Go(func() error { + var err error + info, err = cs.client.GetConversationInfoContext(ctx, &slack.GetConversationInfoInput{ + ChannelID: channelID, + }) + if err != nil { + return fmt.Errorf("error getting channel information: %w", err) + } + return nil }) - return err + + var uu []string + eg.Go(func() error { + var cursor string + for { + u, next, err := cs.client.GetUsersInConversationContext(ctx, &slack.GetUsersInConversationParameters{ + ChannelID: channelID, + Cursor: cursor, + }) + if err != nil { + return fmt.Errorf("error getting conversation users: %w", err) + } + uu = append(uu, u...) + if next == "" { + break + } + cursor = next + } + return nil + }) + if err := eg.Wait(); err != nil { + return err + } + info.Members = uu + return nil }); err != nil { - return nil, fmt.Errorf("error getting channel information for %s: %w", channelID, err) + return nil, fmt.Errorf("api error: %s: %w", channelID, err) } cs.chanCache.set(channelID, info) }