Skip to content

Commit

Permalink
testing harness and exportV3 test
Browse files Browse the repository at this point in the history
  • Loading branch information
rusq committed Apr 21, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 7e14c68 commit 7ba3d93
Showing 19 changed files with 601 additions and 260 deletions.
15 changes: 15 additions & 0 deletions clienter_mock_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions cmd/slackdump/internal/export/expproc/transform.go
Original file line number Diff line number Diff line change
@@ -22,8 +22,6 @@ import (
"github.com/rusq/slackdump/v2/types"
)

const deletedThread = "0000000000.000000"

// Transform is a transformer that takes the chunks produced by the processor
// and transforms them into a Slack Export format. It is sutable for async
// processing, in which case, OnFinalise function is passed to the processor,
@@ -218,6 +216,7 @@ func (t *Transform) WriteIndex(currentUserID string) error {
func (t *Transform) Close() error {
dlog.Debugln("transform: closing transform")
close(t.ids)
close(t.start)
dlog.Debugln("transform: waiting for workers to finish")
<-t.done
return nil
@@ -309,7 +308,7 @@ func writeMessages(ctx context.Context, fsa fsadapter.FS, pl *chunk.File, ci *sl
// in original Slack Export, thread starting messages have some thread
// statistics, and for this we need to scan the chunk file and get it.
var thread []slack.Message
if m.ThreadTimestamp == m.Timestamp && m.LatestReply != deletedThread {
if m.ThreadTimestamp == m.Timestamp && m.LatestReply != structures.NoRepliesLatestReply {
// get the thread for the initial thread message only.
var err error
thread, err = pl.AllThreadMessages(ci.ID, m.ThreadTimestamp)
2 changes: 1 addition & 1 deletion cmd/slackdump/internal/export/v3.go
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ func (e ExportError) Error() string {
}

func exportV3(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, list *structures.EntityList, options export.Config) error {
lg := options.Logger
lg := dlog.FromContext(ctx)

tmpdir, err := os.MkdirTemp("", "slackdump-*")
if err != nil {
79 changes: 79 additions & 0 deletions cmd/slackdump/internal/export/v3_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package export

import (
"bytes"
"compress/gzip"
"context"
"io"
"net/http"
"os"
"path/filepath"
"testing"

"github.com/rusq/fsadapter"
"github.com/rusq/slackdump/v2"
"github.com/rusq/slackdump/v2/export"
"github.com/rusq/slackdump/v2/internal/chunk/chunktest"
"github.com/rusq/slackdump/v2/internal/structures"
"github.com/slack-go/slack"
)

const (
baseDir = "../../../../"
chunkDir = baseDir + "tmp/2/"
largeFile = chunkDir + "C0BBSGYFN.json.gz"
)

func Test_exportV3(t *testing.T) {
// TODO: this is manual
t.Run("large file", func(t *testing.T) {
srv := chunktest.NewDirServer(chunkDir, "U0BBSGYFN")
defer srv.Close()
cl := slack.New("", slack.OptionAPIURL(srv.URL+"/api/"))

ctx := context.Background()
cl.AuthTestContext(ctx)
prov := &chunktest.TestAuth{
FakeToken: "xoxp-1234567890-1234567890-1234567890-1234567890",
WantHTTPClient: http.DefaultClient,
}
sess, err := slackdump.New(ctx, prov, slackdump.WithSlackClient(cl), slackdump.WithLimits(slackdump.NoLimits))
if err != nil {
t.Fatal(err)
}
output := filepath.Join(baseDir, "output.zip")
fsa, err := fsadapter.New(output)
if err != nil {
t.Fatal(err)
}
defer fsa.Close()

list := &structures.EntityList{Include: []string{"C0BBSGYFN"}}
if err := exportV3(ctx, sess, fsa, list, export.Config{List: list}); err != nil {
t.Fatal(err)
}
})
}

func load(t *testing.T, filename string) io.ReadSeeker {
absPath, err := filepath.Abs(largeFile)
if err != nil {
t.Fatal(err)
}
t.Log("test file", absPath)
f, err := os.Open(absPath)
if err != nil {
t.Fatal(err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
_, err = io.Copy(&buf, gz)
if err != nil {
t.Fatal(err)
}
return bytes.NewReader(buf.Bytes())
}
19 changes: 19 additions & 0 deletions config.go
Original file line number Diff line number Diff line change
@@ -81,6 +81,25 @@ var DefLimits = Limits{
},
}

// NoLimits is setting the limits to high values, effectively disabling them.
var NoLimits = Limits{
Workers: 4,
DownloadRetries: 3, // this shouldn't even happen, as we have no limiter on files download.
Tier2: noTierLimits,
Tier3: noTierLimits,
Request: RequestLimit{
Conversations: 100, // this is the recommended value by Slack. But who listens to them anyway.
Channels: 100, // channels are Tier2 rate limited. Slack is greedy and never returns more than 100 per call.
Replies: 1000,
},
}

var noTierLimits = TierLimits{
Boost: 10_000,
Burst: 10_000,
Retries: 3,
}

var (
cfgValidator *validator.Validate // options validator
// OptErrTranslations is the english translations for the validation
4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -2,8 +2,6 @@ module github.com/rusq/slackdump/v2

go 1.19

replace github.com/rusq/fsadapter => ../fsadapter

require (
github.com/AlecAivazis/survey/v2 v2.3.6
github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403
@@ -21,7 +19,7 @@ require (
github.com/rusq/chttp v1.0.1
github.com/rusq/dlog v1.4.0
github.com/rusq/encio v0.1.0
github.com/rusq/fsadapter v1.0.0
github.com/rusq/fsadapter v1.0.1
github.com/rusq/osenv/v2 v2.0.1
github.com/rusq/tracer v1.0.1
github.com/schollz/progressbar/v3 v3.13.0
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -121,6 +121,8 @@ github.com/rusq/encio v0.1.0 h1:DauNaVtIf79kILExhMGIsE5svYwPnDSksdYP0oVVcr8=
github.com/rusq/encio v0.1.0/go.mod h1:AP3lDpo/BkcHcOMNduBlZdd0sbwhruq6+NZtYm5Mxb0=
github.com/rusq/fsadapter v1.0.0 h1:6IZFkpX/MgAKFMyKPDWAnBz2RxYg8MH4EqV9Xy3m4lQ=
github.com/rusq/fsadapter v1.0.0/go.mod h1:56enDqgnY1Mu+brFsVoA3WW4VbKdtjflR7qz/7Rab1E=
github.com/rusq/fsadapter v1.0.1 h1:ADFlyApviYrBWo6N2xm4fh/gFFTy6XoLBU1m7Tvf05k=
github.com/rusq/fsadapter v1.0.1/go.mod h1:56enDqgnY1Mu+brFsVoA3WW4VbKdtjflR7qz/7Rab1E=
github.com/rusq/osenv/v2 v2.0.1 h1:1LtNt8VNV/W86wb38Hyu5W3Rwqt/F1JNRGE+8GRu09o=
github.com/rusq/osenv/v2 v2.0.1/go.mod h1:+wJBSisjNZpfoD961JzqjaM+PtaqSusO3b4oVJi7TFY=
github.com/rusq/secure v0.0.4 h1:svpiZHfHnx89eEDCCFI9OXG1Y8hL9kUWUG6fJbrWUOI=
50 changes: 50 additions & 0 deletions internal/chunk/chunktest/auth.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package chunktest

import (
"context"
"net/http"

"github.com/rusq/slackdump/v2/auth"
)

// TestAuth to use with the chunktest server.
type TestAuth struct {
FakeToken string
FakeCookies []*http.Cookie
WantValidateError error
WantTestError error
WantHTTPClient *http.Client
WantHTTPClientErr error
}

// SlackToken should return the Slack Token value.
func (a *TestAuth) SlackToken() string {
return a.FakeToken
}

// Cookies should return a set of Slack Session cookies.
func (a *TestAuth) Cookies() []*http.Cookie {
return nil
}

// Type returns the auth type.
func (a *TestAuth) Type() auth.Type {
return auth.Type(255)
}

// Validate should return error, in case the token or cookies cannot be
// retrieved.
func (a *TestAuth) Validate() error {
// chur
return a.WantValidateError
}

// Test tests if credentials are valid.
func (a *TestAuth) Test(ctx context.Context) error {
return a.WantTestError
}

// Client returns an authenticated HTTP client
func (a *TestAuth) HTTPClient() (*http.Client, error) {
return a.WantHTTPClient, a.WantHTTPClientErr
}
90 changes: 90 additions & 0 deletions internal/chunk/chunktest/dirserver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package chunktest

import (
"net/http"
"net/http/httptest"
"sync"

"github.com/rusq/slackdump/v2/internal/chunk"
)

// DirServer is a test server that serves files from a chunk.Directory.
type DirServer struct {
*httptest.Server
cd *chunk.Directory
userID string

mu sync.Mutex
ptrs map[string]map[chunk.GroupID]int
}

func NewDirServer(dir string, currentUserID string) *DirServer {
cd, err := chunk.OpenDir(dir)
if err != nil {
panic(err)
}
ds := &DirServer{
cd: cd,
userID: currentUserID,
ptrs: make(map[string]map[chunk.GroupID]int),
}
ds.init()
return ds
}

func (s *DirServer) init() {
s.Server = httptest.NewServer(s.dirRouter())
}

func (s *DirServer) dirRouter() *http.ServeMux {
mux := http.NewServeMux()
mux.Handle("/api/auth.test", authHandler{s.userID})

mux.Handle("/api/conversations.info", s.chunkWrapper(handleConversationsInfo))
mux.Handle("/api/conversations.history", s.chunkWrapper(handleConversationsHistory))
mux.Handle("/api/conversations.replies", s.chunkWrapper(handleConversationsReplies))
mux.Handle("/api/conversations.list", s.chunkWrapper(handleConversationsList))
mux.Handle("/api/users.list", s.chunkfileWrapper("users", handleUsersList))
return mux
}

func (s *DirServer) chunkWrapper(fn func(p *chunk.Player) http.HandlerFunc) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
channel := r.FormValue("channel")
rs, err := s.cd.Open(channel)
if err != nil {
http.NotFound(w, r)
return
}
p := chunk.NewPlayerFromFile(rs)

s.mu.Lock()
if state, ok := s.ptrs[channel]; ok {
p.SetState(state)
} else {
s.ptrs[channel] = make(map[chunk.GroupID]int)
}
s.mu.Unlock()
fn(p)(w, r)
s.mu.Lock()
s.ptrs[channel] = p.State()
s.mu.Unlock()
})
}

func (s *DirServer) chunkfileWrapper(name string, fn func(p *chunk.Player) http.HandlerFunc) http.Handler {
rs, err := s.cd.Open(name)
if err != nil {
panic(err)
}
p := chunk.NewPlayerFromFile(rs)
s.mu.Lock()
if state, ok := s.ptrs["$"+name]; ok {
p.SetState(state)
} else {
s.ptrs["$"+name] = make(map[chunk.GroupID]int)
}
s.mu.Unlock()

return fn(p)
}
Loading

0 comments on commit 7ba3d93

Please sign in to comment.