channel users chunk

config.go

@@ -36,6 +36,8 @@ type Limits struct {
 	Tier2 TierLimit `json:"tier_2,omitempty" yaml:"tier_2,omitempty"`
 	// Tier-3 limits
 	Tier3 TierLimit `json:"tier_3,omitempty" yaml:"tier_3,omitempty"`
+	// Tier-4 limits
+	Tier4 TierLimit `json:"tier_4,omitempty" yaml:"tier_4,omitempty"`
 	// Request Limits
 	Request RequestLimit `json:"per_request,omitempty" yaml:"per_request,omitempty"`
 }
@@ -74,6 +76,11 @@ var DefLimits = Limits{
 		Burst:   1,   // safe value, who would ever want to modify it? I don't know.
 		Retries: 3,   // on Tier 3 this was never a problem, even with limiter-boost=120
 	},
+	Tier4: TierLimit{
+		Boost:   10,
+		Burst:   1,
+		Retries: 3,
+	},
 	Request: RequestLimit{
 		Conversations: 100, // this is the recommended value by Slack. But who listens to them anyway.
 		Channels:      100, // channels are Tier2 rate limited. Slack is greedy and never returns more than 100 per call.
@@ -87,6 +94,7 @@ var NoLimits = Limits{
 	DownloadRetries: 3, // this shouldn't even happen, as we have no limiter on files download.
 	Tier2:           noTierLimits,
 	Tier3:           noTierLimits,
+	Tier4:           noTierLimits,
 	Request: RequestLimit{
 		Conversations: 100, // this is the recommended value by Slack. But who listens to them anyway.
 		Channels:      100, // channels are Tier2 rate limited. Slack is greedy and never returns more than 100 per call.
@@ -133,6 +141,9 @@ func (o *Limits) Apply(other Limits) error {
 	apply(&o.Tier3.Boost, other.Tier3.Boost)
 	apply(&o.Tier3.Burst, other.Tier3.Burst)
 	apply(&o.Tier3.Retries, other.Tier3.Retries)
+	apply(&o.Tier4.Boost, other.Tier4.Boost)
+	apply(&o.Tier4.Burst, other.Tier4.Burst)
+	apply(&o.Tier4.Retries, other.Tier4.Retries)
 	apply(&o.Request.Conversations, other.Request.Conversations)
 	apply(&o.Request.Channels, other.Request.Channels)
 	apply(&o.Request.Replies, other.Request.Replies)

config_test.go

@@ -12,6 +12,7 @@ func TestLimits_Apply(t *testing.T) {
 		DownloadRetries int
 		Tier2           TierLimit
 		Tier3           TierLimit
+		Tier4           TierLimit
 		Request         RequestLimit
 	}
 	type args struct {
@@ -69,6 +70,7 @@ func TestLimits_Validate(t *testing.T) {
 		DownloadRetries int
 		Tier2           TierLimit
 		Tier3           TierLimit
+		Tier4           TierLimit
 		Request         RequestLimit
 	}
 	tests := []struct {

internal/chunk/chunk.go

@@ -21,6 +21,7 @@ const (
 	CChannels
 	CChannelInfo
 	CWorkspaceInfo
+	CChannelUsers
 	CStarredItems
 	CBookmarks
 )
@@ -51,6 +52,8 @@ type Chunk struct {
 	// Files methods.
 	Channel *slack.Channel `json:"ci,omitempty"`
 
+	ChannelUsers []string `json:"cu,omitempty"` // Populated by ChannelUsers
+
 	// Parent is populated in case the chunk is a thread, or a file. Populated
 	// by ThreadMessages and Files methods.
 	Parent *slack.Message `json:"p,omitempty"`
@@ -87,10 +90,11 @@ const (
 	starredChunkID GroupID = "ls"
 	wspInfoChunkID GroupID = "iw"
 
-	threadPrefix   = "t"
-	filePrefix     = "f"
-	chanInfoPrefix = "ic"
-	bookmarkPrefix = "lb"
+	threadPrefix    = "t"
+	filePrefix      = "f"
+	chanInfoPrefix  = "ic"
+	bookmarkPrefix  = "lb"
+	chanUsersPrefix = "lcu"
 )
 
 // Chunk ID categories
@@ -111,6 +115,8 @@ func (c *Chunk) ID() GroupID {
 		return id(filePrefix, c.ChannelID, c.Parent.Timestamp)
 	case CChannelInfo:
 		return channelInfoID(c.ChannelID)
+	case CChannelUsers:
+		return id(chanUsersPrefix, c.ChannelID)
 	case CUsers:
 		return userChunkID // static, one team per chunk file
 	case CChannels:

internal/chunk/directory.go

@@ -112,12 +112,8 @@ func (d *Directory) Channels() ([]slack.Channel, error) {
 	if val := d.cache.channels.Load(); val != nil {
 		return val.([]slack.Channel), nil
 	}
-	// try to open the channels file
-	if fi, err := os.Stat(d.filename(FChannels)); err == nil && !fi.IsDir() {
-		return d.loadChannelsJSON(d.filename(FChannels))
-	}
-	// channel files not found, try to get channel info from the conversation
-	// files.
+	// we are not using the channel.json.gz file because it doesn't contain
+	// members.
 	var ch []slack.Channel
 	if err := filepath.WalkDir(d.dir, func(path string, de fs.DirEntry, err error) error {
 		if err != nil {

internal/chunk/dirproc/conversations.go

@@ -270,6 +270,14 @@ func (cv *Conversations) finalise(ctx context.Context, id chunk.FileID) error {
 	return nil
 }
 
+func (cv *Conversations) ChannelUsers(ctx context.Context, channelID string, threadTS string, cu []string) error {
+	r, err := cv.recorder(chunk.ToFileID(channelID, threadTS, threadTS != ""))
+	if err != nil {
+		return err
+	}
+	return r.ChannelUsers(ctx, channelID, threadTS, cu)
+}
+
 func (cv *Conversations) Close() error {
 	cv.mu.Lock()
 	defer cv.mu.Unlock()

internal/chunk/recorder.go

@@ -221,3 +221,20 @@ func (rec *Recorder) WorkspaceInfo(ctx context.Context, atr *slack.AuthTestRespo
 	}
 	return nil
 }
+
+// ChannelUsers records the channel users
+func (rec *Recorder) ChannelUsers(ctx context.Context, channelID string, threadTS string, users []string) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-rec.errC:
+		return err
+	case rec.chunks <- Chunk{
+		Type:         CChannelUsers,
+		ChannelID:    channelID,
+		Timestamp:    time.Now().UnixNano(),
+		ChannelUsers: users,
+	}:
+	}
+	return nil
+}

internal/format/text.go

@@ -19,8 +19,8 @@ import (
 var _ Formatter = &Text{}
 
 const (
-	defaultNewMsgThreshold = 3 * time.Minute
-	textTimeFmt            = "02/01/2006 15:04:05 Z0700"
+	defaultMsgSplitAfter = 3 * time.Minute
+	textTimeFmt          = "02/01/2006 15:04:05 Z0700"
 )
 
 type Text struct {
@@ -44,7 +44,7 @@ func init() {
 func NewText(opts ...Option) Formatter {
 	settings := options{
 		textOptions: textOptions{
-			msgSplitAfter: defaultNewMsgThreshold,
+			msgSplitAfter: defaultMsgSplitAfter,
 		}}
 	for _, fn := range opts {
 		fn(&settings)

internal/structures/index.go

@@ -66,14 +66,16 @@ func MakeExportIndex(channels []slack.Channel, users []slack.User, currentUserID
 			idx.DMs = append(idx.DMs, DM{
 				ID:      ch.ID,
 				Created: int64(ch.Created),
-				Members: []string{ch.User, currentUserID},
+				Members: ch.Members,
 			})
 		case ch.IsMpIM:
-			fixed, err := FixMpIMmembers(&ch, users)
-			if err != nil {
-				return nil, err
-			}
-			idx.MPIMs = append(idx.MPIMs, *fixed)
+			// TODO: verify this is not needed
+			// fixed, err := FixMpIMmembers(&ch, users)
+			// if err != nil {
+			// 	return nil, err
+			// }
+			// idx.MPIMs = append(idx.MPIMs, *fixed)
+			idx.MPIMs = append(idx.MPIMs, ch)
 		case ch.IsGroup:
 			idx.Groups = append(idx.Groups, ch)
 		default: