Skip to content

Commit

Permalink
chunk files
Browse files Browse the repository at this point in the history
  • Loading branch information
rusq committed Mar 25, 2024
1 parent f12b1f8 commit 949ddb6
Show file tree
Hide file tree
Showing 17 changed files with 298 additions and 44 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/rusq/chttp v1.0.2
github.com/rusq/dlog v1.4.0
github.com/rusq/encio v0.1.0
github.com/rusq/fsadapter v1.0.1
github.com/rusq/fsadapter v1.0.2
github.com/rusq/osenv/v2 v2.0.1
github.com/rusq/slack v0.9.6-0.20240211120639-93c163940e55
github.com/rusq/slackauth v0.1.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ github.com/rusq/encio v0.1.0 h1:DauNaVtIf79kILExhMGIsE5svYwPnDSksdYP0oVVcr8=
github.com/rusq/encio v0.1.0/go.mod h1:AP3lDpo/BkcHcOMNduBlZdd0sbwhruq6+NZtYm5Mxb0=
github.com/rusq/fsadapter v1.0.1 h1:ADFlyApviYrBWo6N2xm4fh/gFFTy6XoLBU1m7Tvf05k=
github.com/rusq/fsadapter v1.0.1/go.mod h1:56enDqgnY1Mu+brFsVoA3WW4VbKdtjflR7qz/7Rab1E=
github.com/rusq/fsadapter v1.0.2 h1:T+hG8nvA4WlM5oLRcUMEPEOdmDWEd9wmD2oNpztwz90=
github.com/rusq/fsadapter v1.0.2/go.mod h1:oqHuzWZKhum7JBLCUi/mkyrxngtfemuJB0HgEJ6595A=
github.com/rusq/osenv/v2 v2.0.1 h1:1LtNt8VNV/W86wb38Hyu5W3Rwqt/F1JNRGE+8GRu09o=
github.com/rusq/osenv/v2 v2.0.1/go.mod h1:+wJBSisjNZpfoD961JzqjaM+PtaqSusO3b4oVJi7TFY=
github.com/rusq/secure v0.0.4 h1:svpiZHfHnx89eEDCCFI9OXG1Y8hL9kUWUG6fJbrWUOI=
Expand Down
11 changes: 10 additions & 1 deletion internal/chunk/directory.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ const (
FWorkspace FileID = "workspace"
)

const uploadsDir = "__uploads" // for serving files

// Directory is an abstraction over the directory with chunk files. It
// provides a way to write chunk files and read channels, users and messages
// across many the chunk files. All functions that require a name, except
Expand All @@ -33,7 +35,9 @@ const (
// file with the extension. All files created by this package will be
// compressed with GZIP, unless stated otherwise.
type Directory struct {
dir string // path to a physical directory on the filesystem
// dir is a path to a physical directory on the filesystem with chunks and
// uploads.
dir string
cache dcache

wantCache bool
Expand Down Expand Up @@ -349,3 +353,8 @@ func cachedFromReader(wf osext.ReadSeekCloseNamer, wantCache bool) (*File, error
}
return cf, nil
}

// File returns the file with the given id and name.
func (d *Directory) File(id string, name string) (fs.File, error) {
return os.Open(filepath.Join(d.dir, uploadsDir, id, name))
}
2 changes: 1 addition & 1 deletion internal/chunk/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ func (f *File) ChannelUsers(channelID string) ([]string, error) {
})
}

func (f *File) channelInfo(channelID string, thread bool) (*slack.Channel, error) {
func (f *File) channelInfo(channelID string, _ bool) (*slack.Channel, error) {
chunk, err := f.firstChunkForID(channelInfoID(channelID))
if err != nil {
return nil, err
Expand Down
28 changes: 28 additions & 0 deletions internal/viewer/handlers.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package viewer

import (
"errors"
"io"
"net/http"
"os"
"path/filepath"

"github.com/davecgh/go-spew/spew"
Expand Down Expand Up @@ -123,6 +126,31 @@ func (v *Viewer) threadHandler(w http.ResponseWriter, r *http.Request, id string
}

func (v *Viewer) fileHandler(w http.ResponseWriter, r *http.Request) {
var (
id = r.PathValue("id")
filename = r.PathValue("filename")
)
if id == "" || filename == "" {
http.NotFound(w, r)
return
}
f, err := v.src.File(id, filename)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
http.NotFound(w, r)
return
}
v.lg.Printf("error: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
fi, err := f.Stat()
if err != nil {
v.lg.Printf("error: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.ServeContent(w, r, filename, fi.ModTime(), f.(io.ReadSeeker)) // TODO: hack
}

func (v *Viewer) userHandler(w http.ResponseWriter, r *http.Request) {
Expand Down
24 changes: 24 additions & 0 deletions internal/viewer/middleware.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package viewer

import (
"fmt"
"net/http"
"time"
)

// cacheMware is a middleware that sets cache control headers.
// [Mozilla reference].
//
// [Mozilla reference]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
func cacheMwareFunc(t time.Duration) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
val := "no-cache, no-store, must-revalidate"
if t > 0 {
val = fmt.Sprintf("max-age=%d", int(t.Seconds()))
}
w.Header().Set("Cache-Control", val)
next.ServeHTTP(w, r)
})
}
}
29 changes: 21 additions & 8 deletions internal/viewer/renderer/slack.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,14 @@ func (*Slack) RenderText(ctx context.Context, s string) (v template.HTML) {
}

func (s *Slack) Render(ctx context.Context, m *slack.Message) (v template.HTML) {
var buf strings.Builder

if len(m.Blocks.BlockSet) == 0 {
return s.RenderText(ctx, m.Text)
s.RenderText(ctx, m.Text)
} else {
s.renderBlocks(ctx, &buf, m.Timestamp, m.Blocks.BlockSet)
}

var buf strings.Builder
s.renderBlocks(ctx, &buf, m.Timestamp, m.Blocks.BlockSet)
s.renderFiles(ctx, &buf, m.Timestamp, m.Files)
s.renderAttachments(ctx, &buf, m.Timestamp, m.Attachments)

return template.HTML(buf.String())
Expand Down Expand Up @@ -84,11 +86,22 @@ func (s *Slack) renderBlocks(ctx context.Context, buf *strings.Builder, msgTS st
}

func (s *Slack) renderAttachments(ctx context.Context, buf *strings.Builder, msgTS string, attachments []slack.Attachment) {
attrMsgID := slog.String("message_ts", msgTS)
for _, a := range attachments {
if err := s.tmpl.ExecuteTemplate(buf, "attachment.html", a); err != nil {
slog.ErrorContext(ctx, "error rendering attachment", "error", err, attrMsgID)
}
s.renderAttachment(ctx, buf, msgTS, a)
}
}

func (s *Slack) renderAttachment(ctx context.Context, buf *strings.Builder, msgTS string, a slack.Attachment) {
attrMsgID := slog.String("message_ts", msgTS)
if err := s.tmpl.ExecuteTemplate(buf, "attachment.html", a); err != nil {
slog.ErrorContext(ctx, "error rendering attachment", "error", err, attrMsgID)
}
}

func (s *Slack) renderFiles(ctx context.Context, buf *strings.Builder, msgTS string, files []slack.File) {
attrMsgID := slog.String("message_ts", msgTS)
if err := s.tmpl.ExecuteTemplate(buf, "file.html", files); err != nil {
slog.ErrorContext(ctx, "error rendering files", "error", err, attrMsgID)
}
}

Expand Down
81 changes: 81 additions & 0 deletions internal/viewer/renderer/slack_fixtures_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,3 +188,84 @@ const (
]
}`
)

// Attachments
const (
fxtrAttYoutube = `{
"fallback": "YouTube Video: Microsoft Account Takeover: Combination of Subdomain Takeovers and Open Redirection Vulnerabilities",
"id": 1,
"author_name": "VULLNERABILITY",
"author_link": "https://www.youtube.com/channel/UClWkD38yogV4fRktm6Kb_2w",
"title": "Microsoft Account Takeover: Combination of Subdomain Takeovers and Open Redirection Vulnerabilities",
"title_link": "https://youtu.be/Jg3mkLm2K2g",
"thumb_url": "https://i.ytimg.com/vi/Jg3mkLm2K2g/hqdefault.jpg",
"service_name": "YouTube",
"service_icon": "https://a.slack-edge.com/80588/img/unfurl_icons/youtube.png",
"from_url": "https://youtu.be/Jg3mkLm2K2g",
"original_url": "https://youtu.be/Jg3mkLm2K2g",
"blocks": null
}`
fxtrAttTwitter = `{
"fallback": "\u003chttps://twitter.com/edwardodell|@edwardodell\u003e: NEVER LEAVE, NEVER PAY",
"id": 1,
"author_name": "Edward Odell",
"author_subname": "@edwardodell",
"author_link": "https://twitter.com/edwardodell/status/1591044196705927168",
"author_icon": "https://pbs.twimg.com/profile_images/1590674641458167808/Z4vACFd0_normal.jpg",
"text": "NEVER LEAVE, NEVER PAY",
"image_url": "https://pbs.twimg.com/media/FhSGQsVXwAUpyU5.jpg",
"service_name": "twitter",
"from_url": "https://twitter.com/edwardodell/status/1591044196705927168?s=46\u0026amp;t=w-i11UUFTIWOtvEWpF2hpQ",
"original_url": "https://twitter.com/edwardodell/status/1591044196705927168?s=46\u0026amp;t=w-i11UUFTIWOtvEWpF2hpQ",
"blocks": null,
"footer": "Twitter",
"footer_icon": "https://a.slack-edge.com/80588/img/services/twitter_pixel_snapped_32.png",
"ts": 1668169471
}`
fxtrAttTwitterX = `{
"fallback": "X (formerly Twitter): Elon Musk Junior :flag-ke: (@ElonMursq) on X",
"id": 1,
"title": "Elon Musk Junior :flag-ke: (@ElonMursq) on X",
"title_link": "https://twitter.com/ElonMursq",
"text": "Please help me reconnect with my dad @ElonMusk.\n\nMy BTC wallet address: 1FM6odFCta6gKwo2ib9jJ2JCEJgQixLoc2",
"thumb_url": "https://pbs.twimg.com/profile_images/1767278440892289024/WXKK1Oa-_200x200.jpg",
"service_name": "X (formerly Twitter)",
"service_icon": "http://abs.twimg.com/favicons/twitter.3.ico",
"from_url": "https://twitter.com/ElonMursq",
"original_url": "https://twitter.com/ElonMursq",
"blocks": null
}`
fxtrAttNzHerald = `{
"fallback": "NZ Herald: NZ Herald: Latest NZ news, plus World, Sport, Business and more - NZ Herald",
"id": 3,
"title": "NZ Herald: Latest NZ news, plus World, Sport, Business and more - NZ Herald",
"title_link": "https://www.nzherald.co.nz/",
"text": "Get the latest breaking news, analysis and opinion from NZ and around the world, including politics, business, sport, entertainment, travel and more, with NZ Herald",
"thumb_url": "https://www.nzherald.co.nz/pf/resources/images/fallback-promo-image.png?d=744",
"service_name": "NZ Herald",
"service_icon": "https://www.nzherald.co.nz/pf/resources/images/favicons/apple-touch-icon-57x57-precomposed.png?d=744",
"from_url": "https://www.nzherald.co.nz/",
"original_url": "https://www.nzherald.co.nz/",
"blocks": null
}`
fxtrAttImage = `{
"fallback": "1200x1200px image",
"id": 2,
"image_url": "https://pbs.twimg.com/media/FhTXW4bWAA4jqXk.jpg",
"from_url": "https://twitter.com/rafaelshimunov/status/1591133819918114816?s=46\u0026amp;t=w-i11UUFTIWOtvEWpF2hpQ",
"blocks": null
}`
fxtrAttBBC = `{
"fallback": "BBC News: Elon Musk: Judge blocks 'unfathomable' $56bn Tesla pay deal",
"id": 1,
"title": "Elon Musk: Judge blocks 'unfathomable' $56bn Tesla pay deal",
"title_link": "https://www.bbc.co.uk/news/business-68150306",
"text": "The lawsuit was filed by a shareholder who argued that it was an inappropriate overpayment.",
"image_url": "https://ichef.bbci.co.uk/news/1024/branded_news/F474/production/_132508526_gettyimages-1963458442.jpg",
"service_name": "BBC News",
"service_icon": "https://www.bbc.co.uk/favicon.ico",
"from_url": "https://www.bbc.co.uk/news/business-68150306",
"original_url": "https://www.bbc.co.uk/news/business-68150306",
"blocks": null
}`
)
32 changes: 32 additions & 0 deletions internal/viewer/renderer/slack_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"html/template"
"reflect"
"strings"
"testing"

"github.com/rusq/slack"
Expand Down Expand Up @@ -46,3 +47,34 @@ func TestSlack_Render(t *testing.T) {
})
}
}

func TestSlack_renderAttachment(t *testing.T) {
type fields struct {
tmpl *template.Template
uu map[string]slack.User
cc map[string]slack.Channel
}
type args struct {
ctx context.Context
buf *strings.Builder
msgTS string
a slack.Attachment
}
tests := []struct {
name string
fields fields
args args
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &Slack{
tmpl: tt.fields.tmpl,
uu: tt.fields.uu,
cc: tt.fields.cc,
}
s.renderAttachment(tt.args.ctx, tt.args.buf, tt.args.msgTS, tt.args.a)
})
}
}
6 changes: 6 additions & 0 deletions internal/viewer/source/chunkdir.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package source

import (
"io/fs"

"github.com/rusq/slack"
"github.com/rusq/slackdump/v3/internal/chunk"
)
Expand Down Expand Up @@ -64,3 +66,7 @@ func (c *ChunkDir) Type() string {
func (c *ChunkDir) Users() ([]slack.User, error) {
return c.d.Users()
}

func (c *ChunkDir) File(fileID string, filename string) (f fs.File, err error) {
return c.d.File(fileID, filename)
}
4 changes: 4 additions & 0 deletions internal/viewer/source/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,7 @@ func (d Dump) ChannelInfo(channelID string) (*slack.Channel, error) {
}
return nil, fs.ErrNotExist
}

func (d Dump) File(id string, filename string) (f fs.File, err error) {
panic("not implemented")
}
34 changes: 8 additions & 26 deletions internal/viewer/source/export.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package source

import (
"encoding/json"
"fmt"
"io/fs"
"path"
Expand Down Expand Up @@ -37,7 +36,12 @@ func NewExport(fsys fs.FS, name string) (*Export, error) {
}

func (e *Export) Channels() ([]slack.Channel, error) {
return unmarshal[[]slack.Channel](e.fs, "channels.json")
cc, err := unmarshal[[]slack.Channel](e.fs, "channels.json")
if err != nil {
return nil, err
}
// TODO: check dms.json and groups.json
return cc, nil
}

func (e *Export) Users() ([]slack.User, error) {
Expand Down Expand Up @@ -112,28 +116,6 @@ func (e *Export) ChannelInfo(channelID string) (*slack.Channel, error) {
return nil, fmt.Errorf("%s: %s", "channel not found", channelID)
}

func unmarshalOne[T any](fsys fs.FS, name string) (T, error) {
var v T
f, err := fsys.Open(name)
if err != nil {
return v, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&v); err != nil {
return v, err
}
return v, nil
}

func unmarshal[T ~[]S, S any](fsys fs.FS, name string) (T, error) {
f, err := fsys.Open(name)
if err != nil {
return nil, err
}
defer f.Close()
var v T
if err := json.NewDecoder(f).Decode(&v); err != nil {
return nil, err
}
return v, nil
func (e *Export) File(id string, name string) (fs.File, error) {
panic("not implemented")
}
Loading

0 comments on commit 949ddb6

Please sign in to comment.