Skip to content

Commit

Permalink
onFiles processor function
Browse files Browse the repository at this point in the history
  • Loading branch information
@rusq
rusq committed Apr 13, 2023
1 parent e40f57d commit 2f1cec0
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 23 deletions.
15 changes: 11 additions & 4 deletions cmd/slackdump/internal/export/expproc/conversations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type Conversations struct {
mu sync.RWMutex
lg logger.Interface
onFinalise func(channelID string) error
onFiles func(channelID string, files []slack.File) error
}

// ConvOption is a function that configures the Conversations processor.
Expand Down Expand Up @@ -47,14 +48,14 @@ type channelproc struct {
refcnt int
}

func NewConversation(dir string, opt ...ConvOption) (*Conversations, error) {
func NewConversation(dir string, opts ...ConvOption) (*Conversations, error) {
c := &Conversations{
dir: dir,
lg: logger.Default,
cw: make(map[string]*channelproc),
}
for _, o := range opt {
o(c)
for _, opt := range opts {
opt(c)
}
return c, nil
}
Expand Down Expand Up @@ -159,6 +160,12 @@ func (cv *Conversations) Files(ctx context.Context, channelID string, parent sla
if err != nil {
return err
}
if cv.onFiles != nil {
if err := cv.onFiles(channelID, ff); err != nil {
return err
}
}

return r.Files(ctx, channelID, parent, isThread, ff)
}

Expand Down Expand Up @@ -191,7 +198,7 @@ func (cv *Conversations) finalise(ctx context.Context, channelID string) error {
dlog.Debugf("channel %s: still processing %d ref count", channelID, tc)
return nil
}
trace.Logf(ctx, "ref", "ref count = 0, finalise", channelID)
trace.Logf(ctx, "ref", "= 0, channel %s finalise", channelID)
dlog.Debugf("channel %s: closing channel file", channelID)
r, err := cv.recorder(channelID)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions cmd/slackdump/internal/export/files.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package export
27 changes: 21 additions & 6 deletions cmd/slackdump/internal/export/v3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import (
"os"
"sync"

"github.com/rusq/asyncdl"
"github.com/rusq/dlog"
"github.com/rusq/fsadapter"
"github.com/rusq/slackdump/v2"
"github.com/rusq/slackdump/v2/auth"
"github.com/rusq/slackdump/v2/cmd/slackdump/internal/export/expproc"
"github.com/rusq/slackdump/v2/export"
"github.com/rusq/slackdump/v2/internal/structures"
Expand All @@ -18,6 +20,7 @@ import (

func exportV3(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, list *structures.EntityList, options export.Config) error {
lg := dlog.FromContext(ctx)

tmpdir, err := os.MkdirTemp("", "slackdump-*")
if err != nil {
return err
Expand All @@ -27,6 +30,18 @@ func exportV3(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, li
return fmt.Errorf("failed to create transformer: %w", err)
}
defer tf.Close()

prov, err := auth.FromContext(ctx) // TODO: implicitly pass auth provider
if err != nil {
return err
}
hcl, err := prov.HTTPClient()
if err != nil {
return err
}
dl := asyncdl.New(fsa, asyncdl.WithClient(hcl))
defer dl.Close()

lg.Printf("using %s as the temporary directory", tmpdir)
lg.Print("running export...")
errC := make(chan error, 1)
Expand All @@ -40,10 +55,10 @@ func exportV3(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, li
var generator linkFeederFunc
if list.HasIncludes() {
// inclusive export, processes only included channels.
generator = listChannelGenerator
generator = genListChannel
} else {
// exclusive export (process only excludes, if any)
generator = apiChannelGenerator(tmpdir, s, options.MemberOnly)
generator = genAPIChannel(tmpdir, s, options.MemberOnly)
}

go func() {
Expand Down Expand Up @@ -103,12 +118,12 @@ func exportV3(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, li

type linkFeederFunc func(ctx context.Context, links chan<- string, list *structures.EntityList) error

// listChannelGenerator feeds the channel IDs that it gets from the list to
// genListChannel feeds the channel IDs that it gets from the list to
// the links channel. It does not fetch the channel list from the api, so
// it's blazing fast in comparison to apiChannelFeeder. When needed, get the
// channel information from the conversations chunk files (they contain the
// chunk with channel information).
func listChannelGenerator(ctx context.Context, links chan<- string, list *structures.EntityList) error {
func genListChannel(ctx context.Context, links chan<- string, list *structures.EntityList) error {
for _, ch := range list.Include {
select {
case <-ctx.Done():
Expand All @@ -119,11 +134,11 @@ func listChannelGenerator(ctx context.Context, links chan<- string, list *struct
return nil
}

// apiChannelGenerator feeds the channel IDs that it gets from the API to the
// genAPIChannel feeds the channel IDs that it gets from the API to the
// links channel. It also filters out channels that are excluded in the list.
// It does not account for "included". It ignores the thread links in the
// list. It writes the channels to the tmpdir.
func apiChannelGenerator(tmpdir string, s *slackdump.Stream, memberOnly bool) linkFeederFunc {
func genAPIChannel(tmpdir string, s *slackdump.Stream, memberOnly bool) linkFeederFunc {
return linkFeederFunc(func(ctx context.Context, links chan<- string, list *structures.EntityList) error {
chIdx := list.Index()
chanproc, err := expproc.NewChannels(tmpdir, func(c []slack.Channel) error {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/joho/godotenv v1.5.1
github.com/kr/pty v1.1.8
github.com/playwright-community/playwright-go v0.2000.1
github.com/rusq/asyncdl v0.2.0
github.com/rusq/chttp v1.0.1
github.com/rusq/dlog v1.4.0
github.com/rusq/encio v0.1.0
Expand Down Expand Up @@ -47,7 +48,6 @@ require (
github.com/google/go-cmp v0.5.9 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kyokomi/emoji/v2 v2.2.12 // indirect
github.com/leodido/go-urn v1.2.2 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
Expand Down
15 changes: 3 additions & 12 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBo
github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
Expand All @@ -36,17 +35,13 @@ github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnm
github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 h1:vbix8DDQ/rfatfFr/8cf/sJfIL69i4BcZfjrVOxsMqk=
github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75/go.mod h1:0gZuvTO1ikSA5LtTI6E13LEOdWQNjIo5MTQOvrV0eFg=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU=
github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s=
github.com/go-playground/validator/v10 v10.12.0 h1:E4gtWgxWxp8YSxExrQFv5BpCahla0PVF2oTTEYaWQGI=
github.com/go-playground/validator/v10 v10.12.0/go.mod h1:hCAPuzYvKdP33pxWa+2+6AIKXEKqjIUyqsNCtbsSJrA=
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
Expand All @@ -72,11 +67,8 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kyokomi/emoji/v2 v2.2.8/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
github.com/kyokomi/emoji/v2 v2.2.12 h1:sSVA5nH9ebR3Zji1o31wu3yOwD1zKXQA2z0zUyeit60=
github.com/kyokomi/emoji/v2 v2.2.12/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
Expand Down Expand Up @@ -115,11 +107,10 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rusq/asyncdl v0.2.0 h1:MPTZivq8xfkgkU9jeVNeGXl6uiNUCL9W1ca8Q0YLRiU=
github.com/rusq/asyncdl v0.2.0/go.mod h1:JjwnZNePxhVuJYdv4gzQUYmprKiyO8iaRmMJ3iEIxp0=
github.com/rusq/chttp v1.0.1 h1:j3WE7+jQE9Rgw0E6mGMje8HxMCv09QRkKvR0oZ1R2vY=
github.com/rusq/chttp v1.0.1/go.mod h1:9H/mMp/iUc4xDkSOY0rL6ecxd/YyaW7zE9GhR+mZfRg=
github.com/rusq/dlog v1.3.3 h1:Q9fZW1H/YEnlDg3Ph1k/BRSBfi/q5ezI+8Metws9tTI=
github.com/rusq/dlog v1.3.3/go.mod h1:kjZAEvBu7m3+mnJQKoIeLul1YB3kJq/6lZBdDTZmpzA=
github.com/rusq/dlog v1.4.0 h1:64oHTSzHjzG6TXKvMbPKQzvqADCZRn6XgAWnp7ASr5k=
github.com/rusq/dlog v1.4.0/go.mod h1:kjZAEvBu7m3+mnJQKoIeLul1YB3kJq/6lZBdDTZmpzA=
github.com/rusq/encio v0.1.0 h1:DauNaVtIf79kILExhMGIsE5svYwPnDSksdYP0oVVcr8=
Expand Down Expand Up @@ -199,8 +190,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down

0 comments on commit 2f1cec0

Please sign in to comment.