Merge branch 'master' into cli-remake

README.rst

@@ -211,7 +211,7 @@ Messages that were conveyed with the donations:
 - 25/01/2022: Stay away from `TheSignChef.com`_, ya hear, they don't pay what
   they owe to their employees.
 
-.. _Application: https://stackoverflow.com/questions/12908881/how-to-copy-cookies-in-google-chrome
+
 .. _`Buy me a cup of tea`: https://www.paypal.com/donate/?hosted_button_id=GUHCLSM7E54ZW
 .. _Telegram: https://t.me/slackdump
 .. _Slack: https://slackdump.herokuapp.com/
@@ -229,13 +229,12 @@ Messages that were conveyed with the donations:
 .. _slack export viewer: https://github.com/hfaran/slack-export-viewer
 .. _releases: https://github.com/rusq/slackdump/releases/
 .. _Slackord2: https://github.com/thomasloupe/Slackord2
-.. _SlackLogViewer: https://github.com/thayakawa-gh/SlackLogViewer/actions/runs/3029568329
+.. _SlackLogViewer: https://github.com/thayakawa-gh/SlackLogViewer/releases
 
 ..
   bulletin board links
 
 .. _`TheSignChef.com`: https://www.glassdoor.com.au/Reviews/TheSignChef-com-Reviews-E793259.htm
-.. _`Get cookies.txt Chrome extension`: https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid
 
 .. |go ref| image:: https://pkg.go.dev/badge/github.com/rusq/slackdump/v2.svg
               :alt: Go Reference

auth/auth.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"strings"
 )
 
 // Type is the auth type.
@@ -45,7 +46,7 @@ func (c simpleProvider) Validate() error {
 	if c.Token == "" {
 		return ErrNoToken
 	}
-	if len(c.Cookie) == 0 {
+	if IsClientToken(c.Token) && len(c.Cookie) == 0 {
 		return ErrNoCookies
 	}
 	return nil
@@ -99,3 +100,7 @@ func Save(w io.Writer, p Provider) error {
 
 	return nil
 }
+
+func IsClientToken(tok string) bool {
+	return strings.HasPrefix(tok, "xoxc-")
+}

auth/auth_test.go

@@ -81,11 +81,18 @@ func TestSave(t *testing.T) {
 			true,
 		},
 		{
-			"cookies missing",
-			args{ValueAuth{simpleProvider{Token: "token_value", Cookie: []http.Cookie{}}}},
+			"cookies missing on client token",
+			args{ValueAuth{simpleProvider{Token: "xoxc-blah", Cookie: []http.Cookie{}}}},
 			"",
 			true,
 		},
+		{
+			"cookies missing on non-client token",
+			args{ValueAuth{simpleProvider{Token: "xoxp-blah", Cookie: []http.Cookie{}}}},
+			`{"Token":"xoxp-blah","Cookie":[]}
+`,
+			false,
+		},
 		{
 			"token and cookie are missing",
 			args{ValueAuth{simpleProvider{}}},

auth/browser/browser.go

@@ -15,6 +15,8 @@ import (
 	"github.com/rusq/slackdump/v2/logger"
 )
 
+const slackDomain = ".slack.com"
+
 // Client is the client for Browser Auth Provider.
 type Client struct {
 	workspace  string
@@ -62,7 +64,7 @@ func (cl *Client) Authenticate(ctx context.Context) (string, []http.Cookie, erro
 		_f = playwright.Float
 	)
 	if err := context.AddCookies(playwright.BrowserContextAddCookiesOptionsCookies{
-		Domain:  _s(".slack.com"),
+		Domain:  _s(slackDomain),
 		Path:    _s("/"),
 		Name:    _s("OptanonAlertBoxClosed"),
 		Value:   _s(time.Now().Add(-10 * time.Minute).Format(time.RFC3339)),
@@ -77,7 +79,7 @@ func (cl *Client) Authenticate(ctx context.Context) (string, []http.Cookie, erro
 	}
 	page.On("close", func() { trace.Log(ctx, "user", "page closed"); close(cl.pageClosed) })
 
-	uri := fmt.Sprintf("https://%s.slack.com", cl.workspace)
+	uri := fmt.Sprintf("https://%s"+slackDomain, cl.workspace)
 	l().Debugf("opening browser URL=%s", uri)
 
 	if _, err := page.Goto(uri); err != nil {
@@ -143,9 +145,13 @@ func extractToken(uri string) (string, error) {
 }
 
 func convertCookies(pwc []playwright.Cookie) []http.Cookie {
-	var ret = make([]http.Cookie, len(pwc))
-	for i, p := range pwc {
-		ret[i] = http.Cookie{
+	var ret = make([]http.Cookie, 0, len(pwc))
+	for _, p := range pwc {
+		if !strings.HasSuffix(p.Domain, slackDomain) {
+			// ignoring filth (thirdparty tracking cookies)
+			continue
+		}
+		ret = append(ret, http.Cookie{
 			Name:     p.Name,
 			Value:    p.Value,
 			Path:     p.Path,
@@ -155,7 +161,7 @@ func convertCookies(pwc []playwright.Cookie) []http.Cookie {
 			Secure:   p.Secure,
 			HttpOnly: p.HttpOnly,
 			SameSite: sameSite(p.SameSite),
-		}
+		})
 	}
 	return ret
 }

auth/value.go

@@ -24,16 +24,19 @@ func NewValueAuth(token string, cookie string) (ValueAuth, error) {
 	if token == "" {
 		return ValueAuth{}, ErrNoToken
 	}
-	if cookie == "" {
-		return ValueAuth{}, ErrNoCookies
-	}
-	return ValueAuth{simpleProvider{
+	c := ValueAuth{simpleProvider{
 		Token: token,
-		Cookie: []http.Cookie{
+	}}
+	if IsClientToken(token) {
+		if len(cookie) == 0 {
+			return ValueAuth{}, ErrNoCookies
+		}
+		c.Cookie = []http.Cookie{
 			makeCookie("d", cookie),
 			makeCookie("d-s", fmt.Sprintf("%d", time.Now().Unix()-10)),
-		},
-	}}, nil
+		}
+	}
+	return c, nil
 }
 
 func (ValueAuth) Type() Type {

doc/login-manual.rst

@@ -16,19 +16,16 @@ TOKEN
    #. In Firefox, under `Tools -> Browser Tools -> Web Developer tools` in the menu bar
    #. In Chrome, click the 'three dots' button to the right of the URL Bar, then select
       'More Tools -> Developer Tools'
-#. Go to the Network tab
-#. In the toolbar, switch to ``Fetch/XHR`` view.
-#. Open any channel or private conversation in Slack.  You'll see a
-   bunch of stuff appearing in Network panel.
-#. In the list of requests, find the one starting with
-   ``channels.prefs.get?``, click it and click on *Headers* tab in the
-   opened pane.
-#. Scroll down, until you see **Form Data**
-#. Grab the **token:** value (it starts with ``xoxc-``), by right
-   clicking the value and choosing "Copy Value".
+#. Switch to the console tab.
+#. Paste the following snippet and press ENTER to execute::
 
-**If you don't see the token value** in Google Chrome - switch to `Payload` tab,
-your token is waiting for you there.
+     JSON.parse(localStorage.localConfig_v2).teams[document.location.pathname.match(/^\/client\/(T[A-Z0-9]+)/)[1]].token
+
+#. Token value is printed right after the executed command (it starts with
+   "``xoxc-``"), save it somewhere for now.
+
+.. NOTE:: if you're having problems running the code snippet above, you can
+          get the token the conventional way, see Troubleshooting_ section below.
 
 COOKIE
 ++++++
@@ -83,6 +80,31 @@ Setting up the application
 
 #. Save the file and close the editor.
 
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+Getting token the hard way
+++++++++++++++++++++++++++
+
+#. Open your browser's *Developer Console*, as described in the TOKEN_ section
+   steps above.
+#. Go to the Network tab
+#. In the toolbar, switch to ``Fetch/XHR`` view.
+#. Open any channel or private conversation in Slack.  You'll see a
+   bunch of stuff appearing in Network panel.
+#. In the list of requests, find the one starting with
+   ``channels.prefs.get?``, click it and click on *Headers* tab in the
+   opened pane.
+#. Scroll down, until you see **Form Data**
+#. Grab the **token:** value (it starts with "``xoxc-``"), by right
+   clicking the value and choosing "Copy Value".
+
+**If you don't see the token value** in Google Chrome - switch to `Payload` tab,
+your token is waiting for you there.
+
+
 [Index_]
 
-.. _Index: README.rst
+.. _Index: README.rst
+.. _Application: https://stackoverflow.com/questions/12908881/how-to-copy-cookies-in-google-chrome
+.. _`Get cookies.txt Chrome extension`: https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid

doc/usage-export.rst

@@ -306,4 +306,4 @@ slack-like GUI.
 .. _Slackord2: https://github.com/thomasloupe/Slackord2
 .. _issue: https://github.com/rusq/slackdump/issues
 .. _SlackLogViewer: https://github.com/thayakawa-gh/SlackLogViewer
-.. _Download SlackLogViewer: https://github.com/thayakawa-gh/SlackLogViewer/actions/runs/3029568329
+.. _Download SlackLogViewer: https://github.com/thayakawa-gh/SlackLogViewer/releases

export/export.go

@@ -189,7 +189,7 @@ func (se *Export) exportConversation(ctx context.Context, userIdx structures.Use
 	ctx, task := trace.NewTask(ctx, "export.conversation")
 	defer task.End()
 
-	messages, err := se.sd.DumpRaw(ctx, ch.ID, se.opts.Oldest, se.opts.Latest, se.dl.ProcessFunc(ch.Name))
+	messages, err := se.sd.DumpRaw(ctx, ch.ID, se.opts.Oldest, se.opts.Latest, se.dl.ProcessFunc(validName(ch)))
 	if err != nil {
 		return fmt.Errorf("failed to dump %q (%s): %w", ch.Name, ch.ID, err)
 	}

export/export_test.go

@@ -194,6 +194,8 @@ func Test_serializeToFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		defer arc.Close()
+
 		r, err := arc.Open("test.json")
 		if err != nil {
 			t.Fatal(err)

export/index.go

@@ -52,7 +52,11 @@ func createIndex(channels []slack.Channel, users types.Users, currentUserID stri
 	}
 
 	var idx = index{
-		Users: users,
+		Users:    users,
+		Channels: []slack.Channel{},
+		Groups:   []slack.Channel{},
+		MPIMs:    []slack.Channel{},
+		DMs:      []DM{},
 	}
 
 	for _, ch := range channels {

fsadapter/zipfs.go

@@ -8,24 +8,26 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 )
 
 var _ FS = &ZIP{}
 
 type ZIP struct {
-	zw *zip.Writer
-	mu sync.Mutex
-	f  *os.File
+	zw   *zip.Writer
+	mu   sync.Mutex
+	f    *os.File
+	seen map[string]bool // seen holds the list of seen directories.
 }
 
 func (z *ZIP) String() string {
 	return fmt.Sprintf("<zip archive: %s>", z.f.Name())
 }
 
 func NewZIP(zw *zip.Writer) *ZIP {
-	return &ZIP{zw: zw}
+	return &ZIP{zw: zw, seen: make(map[string]bool)}
 }
 
 func NewZipFile(filename string) (*ZIP, error) {
@@ -34,11 +36,12 @@ func NewZipFile(filename string) (*ZIP, error) {
 		return nil, err
 	}
 	zw := zip.NewWriter(f)
-	return &ZIP{zw: zw, f: f}, nil
+	return &ZIP{zw: zw, f: f, seen: make(map[string]bool)}, nil
 }
 
 func (*ZIP) normalizePath(p string) string {
-	return path.Join(filepath.SplitList(filepath.Clean(p))...)
+	split := strings.Split(filepath.Clean(p), string(os.PathSeparator))
+	return path.Join(split...)
 }
 
 func (z *ZIP) Create(filename string) (io.WriteCloser, error) {
@@ -55,13 +58,53 @@ func (z *ZIP) Create(filename string) (io.WriteCloser, error) {
 }
 
 func (z *ZIP) create(filename string) (io.Writer, error) {
+	if err := z.ensureDir(filename); err != nil {
+		return nil, err
+	}
 	header := &zip.FileHeader{
 		Name:     filename,
 		Method:   zip.Deflate,
 		Modified: time.Now(),
 	}
 	return z.zw.CreateHeader(header)
+}
 
+func (z *ZIP) ensureDir(filename string) error {
+	if z.seen == nil {
+		z.seen = make(map[string]bool, 0)
+	}
+	var ensureFn = func(dir string) error {
+		if _, seen := z.seen[dir]; seen {
+			return nil
+		}
+		// not seen, create an empty directory.
+		if _, err := z.zw.Create(dir); err != nil {
+			return err
+		}
+		z.seen[dir] = true
+		return nil
+	}
+	dir, _ := path.Split(filename)
+	for _, d := range z.dirpath(dir) {
+		if err := ensureFn(d); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (*ZIP) dirpath(dir string) []string {
+	const sep = "/"
+	if len(dir) == 0 {
+		return nil
+	}
+	var ret []string
+	d := strings.TrimRight(dir, sep)
+	for len(d) > 0 {
+		ret = append([]string{strings.TrimRight(d, sep) + sep}, ret...)
+		d, _ = path.Split(strings.TrimRight(d, sep))
+	}
+	return ret
 }
 
 func (z *ZIP) WriteFile(filename string, data []byte, _ os.FileMode) error {