adding search support

cmd/slackdump/internal/record/record.go

@@ -3,6 +3,7 @@ package record
 import (
 	"context"
 	_ "embed"
+	"fmt"
 	"strings"
 	"time"
 
@@ -37,8 +38,10 @@ func RunRecord(ctx context.Context, cmd *base.Command, args []string) error {
 		return err
 	}
 
-	// hack
-	cfg.Output = strings.TrimSuffix(cfg.Output, ".zip")
+	if strings.HasSuffix(strings.ToUpper(cfg.Output), ".ZIP") {
+		base.SetExitStatus(base.SInvalidParameters)
+		return fmt.Errorf("record does not support writing to a zip file, please specify a directory")
+	}
 
 	cd, err := chunk.CreateDir(cfg.Output)
 	if err != nil {

internal/chunk/chunk.go

@@ -23,6 +23,8 @@ const (
 	CChannelUsers
 	CStarredItems
 	CBookmarks
+	CSearchMessages
+	CSearchFiles
 )
 
 var ErrUnsupChunkType = fmt.Errorf("unsupported chunk type")
@@ -78,10 +80,16 @@ type Chunk struct {
 	// WorkspaceInfo contains the workspace information as returned by the
 	// API.  Populated by WorkspaceInfo.
 	WorkspaceInfo *slack.AuthTestResponse `json:"w,omitempty"`
-	//
+	// StarredItems contains the starred items.
 	StarredItems []slack.StarredItem `json:"st,omitempty"` // Populated by StarredItems
-	//
+	// Bookmarks contains the bookmarks.
 	Bookmarks []slack.Bookmark `json:"b,omitempty"` // Populated by Bookmarks
+	// SearchQuery contains the search query.
+	SearchQuery string `json:"sq,omitempty"` // Populated by SearchMessages and SearchFiles.
+	// SearchMessages contains the search results.
+	SearchMessages []slack.SearchMessage `json:"sm,omitempty"` // Populated by SearchMessages
+	// SearchFiles contains the search results.
+	SearchFiles []slack.File `json:"sf,omitempty"` // Populated by SearchFiles
 }
 
 // GroupID is a unique ID for a chunk group.  It is used to group chunks of
@@ -90,10 +98,12 @@ type Chunk struct {
 type GroupID string
 
 const (
-	userChunkID    GroupID = "lusr"
-	channelChunkID GroupID = "lch"
-	starredChunkID GroupID = "ls"
-	wspInfoChunkID GroupID = "iw"
+	userChunkID     GroupID = "lusr" // list users
+	channelChunkID  GroupID = "lch"  // list channels
+	starredChunkID  GroupID = "ls"   // list starred
+	wspInfoChunkID  GroupID = "iw"   // info workspace
+	srchMsgChunkID  GroupID = "sm"   // search messages results
+	srchFileChunkID GroupID = "sf"   // search file results
 )
 
 const (
@@ -106,9 +116,10 @@ const (
 
 // Chunk ID categories
 const (
-	catFile = 'f'
-	catInfo = 'i'
-	catList = 'l'
+	catFile   = 'f'
+	catInfo   = 'i'
+	catList   = 'l'
+	catSearch = 's'
 )
 
 // ID returns a Group ID for the chunk.
@@ -134,6 +145,10 @@ func (c *Chunk) ID() GroupID {
 		return starredChunkID // static
 	case CBookmarks:
 		return id(bookmarkPrefix, c.ChannelID)
+	case CSearchMessages:
+		return srchMsgChunkID
+	case CSearchFiles:
+		return srchFileChunkID
 	}
 	return GroupID(fmt.Sprintf("<unknown:%s>", c.Type))
 }
@@ -191,3 +206,8 @@ func (g GroupID) isInfo() bool {
 func (g GroupID) isList() bool {
 	return g[0] == catList
 }
+
+// isSearch returns true, if the chunk is a search chunk.
+func (g GroupID) isSearch() bool {
+	return g[0] == catSearch
+}

internal/chunk/file.go

@@ -430,7 +430,7 @@ func (f *File) offsetTimestamps(ctx context.Context) (offts, error) {
 	var ret = make(offts, f.idx.OffsetCount())
 	for id, offsets := range f.idx {
 		switch id[0] {
-		case catInfo, catFile, catList: // ignoring files, information and list chunks
+		case catInfo, catFile, catList, catSearch: // ignoring files, information and list chunks
 			continue
 		}
 		for _, offset := range offsets {

internal/chunk/recorder.go

@@ -214,3 +214,34 @@ func (rec *Recorder) ChannelUsers(ctx context.Context, channelID string, threadT
 
 	return nil
 }
+
+// SearchMessages records the search messages.
+func (rec *Recorder) SearchMessages(ctx context.Context, query string, sm []slack.SearchMessage) error {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	chunk := Chunk{
+		Type:           CSearchMessages,
+		Timestamp:      time.Now().UnixNano(),
+		SearchQuery:    query,
+		SearchMessages: sm,
+	}
+	if err := rec.enc.Encode(chunk); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (rec *Recorder) SearchFiles(ctx context.Context, query string, sf []slack.File) error {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	chunk := Chunk{
+		Type:        CSearchFiles,
+		Timestamp:   time.Now().UnixNano(),
+		SearchQuery: query,
+		SearchFiles: sf,
+	}
+	if err := rec.enc.Encode(chunk); err != nil {
+		return err
+	}
+	return nil
+}

internal/edge/wrapper.go

@@ -71,3 +71,11 @@ func (w *Wrapper) GetUsersContext(ctx context.Context, options ...slack.GetUsers
 func (w *Wrapper) GetEmojiContext(ctx context.Context) (map[string]string, error) {
 	return w.cl.GetEmojiContext(ctx)
 }
+
+func (w *Wrapper) SearchMessagesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchMessages, error) {
+	return w.cl.SearchMessagesContext(ctx, query, params)
+}
+
+func (w *Wrapper) SearchFilesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchFiles, error) {
+	return w.cl.SearchFilesContext(ctx, query, params)
+}

processor/processor.go

@@ -54,3 +54,10 @@ type Channels interface {
 	// Channels is called for each channel chunk that is retrieved.
 	Channels(ctx context.Context, channels []slack.Channel) error
 }
+
+type Search interface {
+	// SearchMessages is called for each message that is retrieved.
+	SearchMessages(ctx context.Context, query string, messages []slack.SearchMessage) error
+	// SearchFiles is called for each file that is retrieved.
+	SearchFiles(ctx context.Context, query string, files []slack.File) error
+}

slackdump.go

@@ -51,6 +51,9 @@ type Slacker interface {
 	GetConversationsContext(ctx context.Context, params *slack.GetConversationsParameters) (channels []slack.Channel, nextCursor string, err error)
 	GetConversationInfoContext(ctx context.Context, input *slack.GetConversationInfoInput) (*slack.Channel, error)
 	GetUsersInConversationContext(ctx context.Context, params *slack.GetUsersInConversationParameters) ([]string, string, error)
+
+	SearchMessagesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchMessages, error)
+	SearchFilesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchFiles, error)
 }
 
 // clienter is the interface with some functions of slack.Client with the sole

stream.go

@@ -713,3 +713,43 @@ func (cs *Stream) ListChannels(ctx context.Context, proc processor.Channels, p *
 	}
 	return nil
 }
+
+func (cs *Stream) SearchMessages(ctx context.Context, proc processor.Search, query string) error {
+	ctx, task := trace.NewTask(ctx, "SearchMessages")
+	defer task.End()
+
+	lg := logger.FromContext(ctx)
+
+	lim := rate.NewLimiter(rate.Every(3*time.Second), 5)
+	var p = slack.SearchParameters{
+		Sort:          "timestamp",
+		SortDirection: "desc",
+		Count:         100,
+		Cursor:        "*",
+	}
+	for {
+		var (
+			sm  *slack.SearchMessages
+			err error
+		)
+		if err := network.WithRetry(ctx, lim, 3, func() error {
+			sm, err = cs.client.SearchMessagesContext(ctx, query, p)
+			return err
+		}); err != nil {
+			return err
+		}
+		if err := proc.SearchMessages(ctx, query, sm.Matches); err != nil {
+			return err
+		}
+		if sm.NextCursor == "" {
+			lg.Print("no more messages")
+			break
+		}
+		lg.Printf("cursor %s", sm.NextCursor)
+		p.Cursor = sm.NextCursor
+
+		lim.Wait(ctx)
+	}
+
+	return nil
+}