Skip to content

Commit

Permalink
brush up listing users and channels
Browse files Browse the repository at this point in the history
  • Loading branch information
@rusq
rusq committed Nov 10, 2024
1 parent 48ea2fd commit cfbe5b5
Show file tree
Hide file tree
Showing 7 changed files with 205 additions and 167 deletions.
152 changes: 87 additions & 65 deletions cmd/slackdump/internal/list/channels.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,13 @@ package list
import (
"context"
"fmt"
"log"
"runtime/trace"
"time"

"github.com/rusq/slack"
"github.com/rusq/slackdump/v3"
"github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap"
"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
"github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base"
"github.com/rusq/slackdump/v3/internal/cache"
Expand All @@ -15,7 +18,7 @@ import (
)

var CmdListChannels = &base.Command{
Run: listChannels,
Run: runListChannels,
UsageLine: "slackdump list channels [flags] [filename]",
PrintFlags: true,
FlagMask: cfg.OmitDownloadFlag,
Expand All @@ -37,90 +40,109 @@ and -chan-cache-retention flags to control the cache behavior.
RequireAuth: true,
}

func init() {
CmdListChannels.Wizard = wizChannels
}

type channelOptions struct {
noResolve bool
cache cacheOpts
}
type (
channelOptions struct {
resolveUsers bool
cache cacheOpts
}

type cacheOpts struct {
Disabled bool
Retention time.Duration
Filename string
}
cacheOpts struct {
Enabled bool
Retention time.Duration
Filename string
}
)

var chanFlags = channelOptions{
noResolve: false,
resolveUsers: false,
cache: cacheOpts{
Disabled: false,
Enabled: false,
Retention: 20 * time.Minute,
Filename: "channels.json",
},
}

func init() {
CmdListChannels.Flag.BoolVar(&chanFlags.cache.Disabled, "no-chan-cache", chanFlags.cache.Disabled, "disable channel cache")
CmdListChannels.Wizard = wizChannels

CmdListChannels.Flag.BoolVar(&chanFlags.cache.Enabled, "no-chan-cache", chanFlags.cache.Enabled, "disable channel cache")
CmdListChannels.Flag.DurationVar(&chanFlags.cache.Retention, "chan-cache-retention", chanFlags.cache.Retention, "channel cache retention time. After this time, the cache is considered stale and will be refreshed.")
CmdListChannels.Flag.BoolVar(&chanFlags.noResolve, "no-resolve", chanFlags.noResolve, "do not resolve user IDs to names")
CmdListChannels.Flag.BoolVar(&chanFlags.resolveUsers, "resolve", chanFlags.resolveUsers, "resolve user IDs to names")
}

func listChannels(ctx context.Context, cmd *base.Command, args []string) error {
if err := list(ctx, func(ctx context.Context, sess *slackdump.Session) (any, string, error) {
ctx, task := trace.NewTask(ctx, "listChannels")
defer task.End()

var filename = makeFilename("channels", sess.Info().TeamID, ".json")
if len(args) > 0 {
filename = args[0]
}
teamID := sess.Info().TeamID
cc, ok := maybeLoadChanCache(cfg.CacheDir(), teamID)
if ok {
// cache hit
trace.Logf(ctx, "cache hit", "teamID=%s", teamID)
return cc, filename, nil
}
// cache miss, load from API
trace.Logf(ctx, "cache miss", "teamID=%s", teamID)
cc, err := sess.GetChannels(ctx)
if err != nil {
return nil, "", fmt.Errorf("error getting channels: %w", err)
}
if err := saveCache(cfg.CacheDir(), teamID, cc); err != nil {
// warn, but don't fail
logger.FromContext(ctx).Printf("failed to save cache: %v", err)
}
return cc, filename, nil
}); err != nil {
func runListChannels(ctx context.Context, cmd *base.Command, args []string) error {
sess, err := bootstrap.SlackdumpSession(ctx)
if err != nil {
base.SetExitStatus(base.SInitializationError)
return err
}

return nil
var l = &channels{
opts: chanFlags,
common: commonFlags,
}

return list(ctx, sess, l, filename)
}

func maybeLoadChanCache(cacheDir string, teamID string) (types.Channels, bool) {
if chanFlags.cache.Disabled {
// channel cache disabled
return nil, false
}
m, err := cache.NewManager(cacheDir)
if err != nil {
return nil, false
}
cc, err := m.LoadChannels(teamID, chanFlags.cache.Retention)
if err != nil {
return nil, false
}
return cc, true
type channels struct {
channels types.Channels
users types.Users

opts channelOptions
common commonOpts
}

func saveCache(cacheDir, teamID string, cc types.Channels) error {
m, err := cache.NewManager(cacheDir)
func (l *channels) Type() string {
return "channels"
}

func (l *channels) Data() types.Channels {
return l.channels
}

func (l *channels) Users() []slack.User {
return l.users
}

func (l *channels) Retrieve(ctx context.Context, sess *slackdump.Session, m *cache.Manager) error {
ctx, task := trace.NewTask(ctx, "channels.List")
defer task.End()
lg := cfg.Log

teamID := sess.Info().TeamID

usersc := make(chan []slack.User)
go func() {
defer close(usersc)
if l.opts.resolveUsers {

lg.Println("getting users to resolve DM names")
u, err := fetchUsers(ctx, sess, m, cfg.NoUserCache, teamID)
if err != nil {
log.Printf("error getting users to resolve DM names (ignored): %s", err)
return
}
usersc <- u
}
}()

if l.opts.cache.Enabled {
var err error
l.channels, err = m.LoadChannels(teamID, l.opts.cache.Retention)
if err == nil {
l.users = <-usersc
return nil
}
}
cc, err := sess.GetChannels(ctx)
if err != nil {
return err
return fmt.Errorf("error getting channels: %w", err)
}
return m.CacheChannels(teamID, cc)
l.channels = cc
l.users = <-usersc
if err := m.CacheChannels(teamID, cc); err != nil {
logger.FromContext(ctx).Printf("warning: failed to cache channels (ignored): %s", err)
}
return nil
}
98 changes: 41 additions & 57 deletions cmd/slackdump/internal/list/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,13 @@ package list

import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"

"github.com/rusq/fsadapter"
"github.com/rusq/slack"
"github.com/rusq/slackdump/v3"
"github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap"
"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
"github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base"
"github.com/rusq/slackdump/v3/internal/cache"
Expand All @@ -21,10 +17,6 @@ import (
"github.com/rusq/slackdump/v3/types"
)

const (
userCacheBase = "users.cache"
)

// CmdList is the list command. The logic is in the subcommands.
var CmdList = &base.Command{
UsageLine: "slackdump list",
Expand Down Expand Up @@ -55,14 +47,26 @@ The caching can be turned off by using flags "-no-user-cache" and
},
}

type lister[T any] interface {
// Type should return the type of the lister.
Type() string
// Retrieve should retrieve the data from the API or cache.
Retrieve(ctx context.Context, sess *slackdump.Session, m *cache.Manager) error
// Data should return the retrieved data.
Data() T
// Users should return the users for the data, or nil, which indicates
// that there are no associated users or that the users are not resolved.
Users() []slack.User
}

// common flags
type listOptions struct {
type commonOpts struct {
listType format.Type
quiet bool // quiet mode: don't print anything on the screen, just save the file
nosave bool // nosave mode: don't save the data to a file, just print it to the screen
}

var commonParams = listOptions{
var commonFlags = commonOpts{
listType: format.CText,
}

Expand All @@ -74,83 +78,63 @@ func init() {

// addCommonFlags adds common flags to the flagset.
func addCommonFlags(fs *flag.FlagSet) {
fs.Var(&commonParams.listType, "format", fmt.Sprintf("listing format, should be one of: %v", format.All()))
fs.BoolVar(&commonParams.quiet, "q", false, "quiet mode: don't print anything on the screen, just save the file")
fs.BoolVar(&commonParams.nosave, "no-json", false, "don't save the data to a file, just print it to the screen")
fs.Var(&commonFlags.listType, "format", fmt.Sprintf("listing format, should be one of: %v", format.All()))
fs.BoolVar(&commonFlags.quiet, "q", false, "quiet mode: don't print anything on the screen, just save the file")
fs.BoolVar(&commonFlags.nosave, "no-json", false, "don't save the data to a file, just print it to the screen")
}

// listFunc is a function that lists something from the Slack API. It should
// return the object from the api, a filename to save the data to and an
// error.
type listFunc func(ctx context.Context, sess *slackdump.Session) (a any, filename string, err error)

// list authenticates and creates a slackdump instance, then calls a listFn.
// listFn must return the object from the api, a JSON filename and an error.
func list(ctx context.Context, listFn listFunc) error {
// TODO fix users saving JSON to a text file within archive
if commonParams.listType == format.CUnknown {
return errors.New("unknown listing format, seek help")
}

// initialize the session.
sess, err := bootstrap.SlackdumpSession(ctx)
func list[T any](ctx context.Context, sess *slackdump.Session, l lister[T], filename string) error {
m, err := cache.NewManager(cfg.CacheDir())
if err != nil {
base.SetExitStatus(base.SInitializationError)
return err
}

data, filename, err := listFn(ctx, sess)
if err != nil {
return err
}
m, err := cache.NewManager(cfg.CacheDir(), cache.WithUserCacheBase(userCacheBase))
if err != nil {
if err := l.Retrieve(ctx, sess, m); err != nil {
return err
}

teamID := sess.Info().TeamID
users, ok := data.(types.Users) // Hax
if !ok && !chanFlags.noResolve {
if cfg.NoUserCache {
users, err = sess.GetUsers(ctx)
} else {
users, err = getCachedUsers(ctx, sess, m, teamID)
}
if err != nil {
if !commonFlags.quiet {
if err := fmtPrint(ctx, os.Stdout, l.Data(), commonFlags.listType, l.Users()); err != nil {
return err
}
}

if !commonParams.nosave {
fsa, err := fsadapter.New(cfg.Output)
if err != nil {
return err
if !commonFlags.nosave {
if filename == "" {
filename = makeFilename(l.Type(), sess.Info().TeamID, extForType(commonFlags.listType))
}
defer fsa.Close()
if err := saveData(ctx, fsa, data, filename, format.CJSON, users); err != nil {
if err := saveData(ctx, l.Data(), filename, commonFlags.listType, l.Users()); err != nil {
return err
}
}
return nil
}

if !commonParams.quiet {
return fmtPrint(ctx, os.Stdout, data, commonParams.listType, users)
func extForType(typ format.Type) string {
switch typ {
case format.CJSON:
return ".json"
case format.CText:
return ".txt"
case format.CCSV:
return ".csv"
default:
return ".json"
}

return nil
}

// saveData saves the given data to the given filename.
func saveData(ctx context.Context, fs fsadapter.FS, data any, filename string, typ format.Type, users []slack.User) error {
func saveData(ctx context.Context, data any, filename string, typ format.Type, users []slack.User) error {
// save to a filesystem.
f, err := fs.Create(filename)
f, err := os.Create(filename)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
defer f.Close()
if err := fmtPrint(ctx, f, data, typ, users); err != nil {
return err
}
logger.FromContext(ctx).Printf("Data saved to: %q\n", filepath.Join(cfg.Output, filename))
logger.FromContext(ctx).Printf("Data saved to: %q\n", filename)

return nil
}
Expand Down
Loading

0 comments on commit cfbe5b5

Please sign in to comment.