fix edge userinfo method

internal/edge/conversations.go

@@ -53,3 +53,63 @@ func (cl *Client) ConversationsGenericInfo(ctx context.Context, channelID ...str
 	}
 	return r.Channels, nil
 }
+
+type conversationsViewForm struct {
+	BaseRequest
+	CanonicalAvatars             bool   `json:"canonical_avatars"`
+	NoUserProfile                bool   `json:"no_user_profile"`
+	IgnoreReplies                bool   `json:"ignore_replies"`
+	NoSelf                       bool   `json:"no_self"`
+	IncludeFullUsers             bool   `json:"include_full_users"`
+	IncludeUseCases              bool   `json:"include_use_cases"`
+	IncludeStories               bool   `json:"include_stories"`
+	NoMembers                    bool   `json:"no_members"`
+	IncludeMutationTimestamps    bool   `json:"include_mutation_timestamps"`
+	Count                        int    `json:"count"`
+	Channel                      string `json:"channel"`
+	IncludeFreeTeamExtraMessages bool   `json:"include_free_team_extra_messages"`
+	WebClientFields
+}
+
+type ConversationsViewResponse struct {
+	Users  []User            `json:"users"`
+	IM     IM                `json:"im"`
+	Emojis map[string]string `json:"emojis"`
+	// we don't care about the rest of the response
+}
+
+func (cl *Client) ConversationsView(ctx context.Context, channelID string) (ConversationsViewResponse, error) {
+	ctx, task := trace.NewTask(ctx, "ConversationsView")
+	defer task.End()
+	trace.Logf(ctx, "params", "channelID=%v", channelID)
+
+	form := conversationsViewForm{
+		BaseRequest: BaseRequest{
+			Token: cl.token,
+		},
+		CanonicalAvatars:          true,
+		NoUserProfile:             true,
+		IgnoreReplies:             true,
+		NoSelf:                    true,
+		IncludeFullUsers:          false,
+		IncludeUseCases:           false,
+		IncludeStories:            false,
+		NoMembers:                 true,
+		IncludeMutationTimestamps: false,
+		Count:                     50,
+		Channel:                   channelID,
+		WebClientFields:           webclientReason(""),
+	}
+	resp, err := cl.PostForm(ctx, "conversations.view", values(form, true))
+	if err != nil {
+		return ConversationsViewResponse{}, err
+	}
+	var r = struct {
+		BaseResponse
+		ConversationsViewResponse
+	}{}
+	if err := cl.ParseResponse(&r, resp); err != nil {
+		return ConversationsViewResponse{}, err
+	}
+	return r.ConversationsViewResponse, nil
+}

internal/edge/edge.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/rusq/slack"
+	"github.com/rusq/slackauth"
 	"github.com/rusq/slackdump/v3/auth"
 	"github.com/rusq/slackdump/v3/internal/tagmagic"
 	"github.com/rusq/slackdump/v3/logger"
@@ -195,6 +196,15 @@ func (cl *Client) PostJSON(ctx context.Context, path string, req PostRequest) (*
 	return do(ctx, cl.cl, r)
 }
 
+// callEdgeAPI calls the edge API.
+func (cl *Client) callEdgeAPI(ctx context.Context, v any, endpoint string, req PostRequest) error {
+	r, err := cl.PostJSON(ctx, endpoint, req)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return err
+	}
+	return cl.ParseResponse(v, r)
+}
+
 func (cl *Client) PostForm(ctx context.Context, path string, form url.Values) (*http.Response, error) {
 	return cl.PostFormRaw(ctx, cl.webclientAPI+path, form)
 }
@@ -238,7 +248,7 @@ func (cl *Client) ParseResponse(req any, r *http.Response) error {
 func do(ctx context.Context, cl *http.Client, req *http.Request) (*http.Response, error) {
 	lg := logger.FromContext(ctx)
 	req.Header.Set("Accept-Language", "en-NZ,en-AU;q=0.9,en;q=0.8")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
+	req.Header.Set("User-Agent", slackauth.DefaultUserAgent)
 
 	resp, err := cl.Do(req)
 	if err != nil {
@@ -315,8 +325,8 @@ func (e *APIError) Error() string {
 }
 
 type WebClientFields struct {
-	XReason  string `json:"_x_reason"`
-	XMode    string `json:"_x_mode"`
+	XReason  string `json:"_x_reason,omitempty"`
+	XMode    string `json:"_x_mode,omitempty"`
 	XSonic   bool   `json:"_x_sonic"`
 	XAppName string `json:"_x_app_name"`
 }

internal/edge/userlist.go

@@ -3,9 +3,9 @@ package edge
 import (
 	"context"
 	"errors"
-	"io"
 
 	"github.com/rusq/slack"
+	"golang.org/x/sync/errgroup"
 )
 
 type UsersListRequest struct {
@@ -47,6 +47,7 @@ type User struct {
 	Updated                slack.JSONTime `json:"updated"`
 	IsEmailConfirmed       bool           `json:"is_email_confirmed"`
 	WhoCanShareContactCard string         `json:"who_can_share_contact_card"`
+	Has2Fa                 *bool          `json:"has_2fa,omitempty"`
 }
 
 type Profile struct {
@@ -181,46 +182,66 @@ func (cl *Client) ChannelsMembership(ctx context.Context, req *ChannelsMembershi
 	return &um, nil
 }
 
-// callEdgeAPI calls the edge API.
-func (cl *Client) callEdgeAPI(ctx context.Context, v any, endpoint string, req PostRequest) error {
-	r, err := cl.PostJSON(ctx, endpoint, req)
-	if err != nil && !errors.Is(err, io.EOF) {
-		return err
+// UserList lists users in the conversation(s).
+func (cl *Client) UsersList(ctx context.Context, channelIDs ...string) ([]User, error) {
+	if len(channelIDs) == 0 {
+		return nil, errors.New("no channel IDs provided")
 	}
-	return cl.ParseResponse(v, r)
+	channelIDs, dmIDs := splitDMs(channelIDs)
+	var uu []User
+	var eg errgroup.Group
+	if len(channelIDs) > 0 {
+		eg.Go(func() error {
+			u, err := cl.publicUserList(ctx, channelIDs)
+			if err != nil {
+				return err
+			}
+			uu = append(uu, u...)
+			return nil
+		})
+	}
+	if len(dmIDs) > 0 {
+		eg.Go(func() error {
+			u, err := cl.directUserList(ctx, dmIDs)
+			if err != nil {
+				return err
+			}
+			uu = append(uu, u...)
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+	return uu, nil
 }
 
-// UserList lists users in the conversation(s).
-func (cl *Client) UsersList(ctx context.Context, channelIDs ...string) ([]User, error) {
+func (cl *Client) publicUserList(ctx context.Context, channelIDs []string) ([]User, error) {
 	const (
-		everyone           = "everyone AND NOT bots AND NOT apps"
-		people             = "people"
-		usersByDisplayName = "users_by_display_name"
+		// everyone = "everyone AND NOT bots AND NOT apps"
+		everyone = "everyone"
+		filter   = "people"
+		index    = "users_by_display_name"
 
-		perRequest = 50
+		count = 50
 	)
-	if len(channelIDs) == 0 {
-		return nil, errors.New("no channel IDs provided")
-
-	}
 	req := UsersListRequest{
 		Channels:     channelIDs,
 		Filter:       everyone,
-		PresentFirst: true,
-		Index:        usersByDisplayName,
+		PresentFirst: false,
+		Index:        index,
 		Locale:       "en-US",
 		Marker:       "",
-		Count:        perRequest,
+		Count:        count,
 	}
-
+	var uu = make([]User, 0, count)
 	lim := tier3.limiter()
-	var uu = make([]User, 0, perRequest)
 	for {
 		var ur UsersListResponse
 		if err := cl.callEdgeAPI(ctx, &ur, "users/list", &req); err != nil {
 			return nil, err
 		}
-		if len(ur.Results) == 0 {
+		if len(ur.Results) == 0 && ur.NextMarker == "" {
 			break
 		}
 		uu = append(uu, ur.Results...)
@@ -234,3 +255,39 @@ func (cl *Client) UsersList(ctx context.Context, channelIDs ...string) ([]User,
 	}
 	return uu, nil
 }
+
+// directUserList tries to get users from the direct message channels.  It is
+// much slower than getting users from the public channels, as it uses
+// conversations.view endpoint.
+func (cl *Client) directUserList(ctx context.Context, dmIDs []string) ([]User, error) {
+	if len(dmIDs) == 0 {
+		return nil, errors.New("no direct message IDs provided")
+	}
+	var ret []User
+	lim := tier3.limiter()
+	for _, id := range dmIDs {
+		resp, err := cl.ConversationsView(ctx, id)
+		if err != nil {
+			return nil, err
+		}
+		ret = append(ret, resp.Users...)
+		if err := lim.Wait(ctx); err != nil {
+			return nil, err
+		}
+	}
+	return ret, nil
+}
+
+func splitDMs(IDs []string) (chans []string, dms []string) {
+	for _, id := range IDs {
+		if len(id) == 0 {
+			continue
+		}
+		if len(id) > 0 && id[0] == 'D' {
+			dms = append(dms, id)
+		} else {
+			chans = append(chans, id)
+		}
+	}
+	return chans, dms
+}

internal/network/network.go

@@ -59,7 +59,6 @@ func (e *ErrRetryFailed) Is(target error) bool {
 // slack.RateLimitedError, it will delay, and then call it again up to
 // maxAttempts times. It will return an error if it runs out of attempts.
 func WithRetry(ctx context.Context, lim *rate.Limiter, maxAttempts int, fn func() error) error {
-	tracelogf(ctx, "info", "maxAttempts=%d", maxAttempts)
 	var ok bool
 	if maxAttempts == 0 {
 		maxAttempts = defNumAttempts
@@ -78,7 +77,6 @@ func WithRetry(ctx context.Context, lim *rate.Limiter, maxAttempts int, fn func(
 
 		cbErr := fn()
 		if cbErr == nil {
-			tracelogf(ctx, "info", "success")
 			ok = true
 			break
 		}

stream/conversation.go

@@ -269,9 +269,9 @@ func (cs *Stream) thread(ctx context.Context, sl *structures.SlackLink, callback
 	return nil
 }
 
-// procChanMsg processes the messages in the channel and sends
-// thread requests for the threads in the channel, if it discovers messages
-// with threads.  It returns thread count in the mm and error if any.
+// procChanMsg processes the message slice mm, for each threaded message, it
+// sends the thread request on threadC.  It returns thread count in the mm and
+// error if any.
 func procChanMsg(ctx context.Context, proc processor.Conversations, threadC chan<- request, channel *slack.Channel, isLast bool, mm []slack.Message) (int, error) {
 	lg := logger.FromContext(ctx)
 
@@ -334,8 +334,8 @@ func procFiles(ctx context.Context, proc processor.Filer, channel *slack.Channel
 	return nil
 }
 
-// channelInfo fetches the channel info and passes it to the processor.
-func (cs *Stream) channelInfo(ctx context.Context, proc processor.ChannelInformer, channelID string, threadTS string) (*slack.Channel, error) {
+// procChannelInfo fetches the channel info and passes it to the processor.
+func (cs *Stream) procChannelInfo(ctx context.Context, proc processor.ChannelInformer, channelID string, threadTS string) (*slack.Channel, error) {
 	ctx, task := trace.NewTask(ctx, "channelInfo")
 	defer task.End()
 
@@ -367,8 +367,9 @@ func (cs *Stream) channelInfo(ctx context.Context, proc processor.ChannelInforme
 	return info, nil
 }
 
-func (cs *Stream) channelUsers(ctx context.Context, proc processor.ChannelInformer, channelID, threadTS string) ([]string, error) {
-	var uu []string
+func (cs *Stream) procChannelUsers(ctx context.Context, proc processor.ChannelInformer, channelID, threadTS string) ([]string, error) {
+	var users []string
+
 	var cursor string
 	for {
 		var u []string
@@ -383,27 +384,31 @@ func (cs *Stream) channelUsers(ctx context.Context, proc processor.ChannelInform
 		}); err != nil {
 			return nil, fmt.Errorf("error getting conversation users: %w", err)
 		}
+		if len(u) == 0 && next == "" {
+			break
+		}
 		if err := proc.ChannelUsers(ctx, channelID, threadTS, u); err != nil {
 			return nil, err
 		}
-		uu = append(uu, u...)
+		users = append(users, u...)
 		if next == "" {
 			break
 		}
 		cursor = next
 	}
-	return uu, nil
+
+	return users, nil
 }
 
-// channelInfoWithUsers returns the slack channel with members populated from
+// procChannelInfoWithUsers returns the slack channel with members populated from
 // another api.
-func (cs *Stream) channelInfoWithUsers(ctx context.Context, proc processor.ChannelInformer, channelID, threadTS string) (*slack.Channel, error) {
+func (cs *Stream) procChannelInfoWithUsers(ctx context.Context, proc processor.ChannelInformer, channelID, threadTS string) (*slack.Channel, error) {
 	var eg errgroup.Group
 
 	var chC = make(chan slack.Channel, 1)
 	eg.Go(func() error {
 		defer close(chC)
-		ch, err := cs.channelInfo(ctx, proc, channelID, threadTS)
+		ch, err := cs.procChannelInfo(ctx, proc, channelID, threadTS)
 		if err != nil {
 			return err
 		}
@@ -414,7 +419,7 @@ func (cs *Stream) channelInfoWithUsers(ctx context.Context, proc processor.Chann
 	var uC = make(chan []string, 1)
 	eg.Go(func() error {
 		defer close(uC)
-		m, err := cs.channelUsers(ctx, proc, channelID, threadTS)
+		m, err := cs.procChannelUsers(ctx, proc, channelID, threadTS)
 		if err != nil {
 			return err
 		}