Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose Export package and simplify app #96

Merged
merged 3 commits into from
Jul 25, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
isolate dump from app
rusq committed Jul 23, 2022

Verified

This commit was signed with the committer’s verified signature.
DrJosh9000 Josh Deprez
commit 4ca4e6bd43681bda46779156547a965ef8941fe4
82 changes: 15 additions & 67 deletions internal/app/app.go
Original file line number Diff line number Diff line change
@@ -2,13 +2,11 @@ package app

import (
"context"
"html/template"
"runtime/trace"
"time"

"github.com/rusq/slackdump/v2"
"github.com/rusq/slackdump/v2/auth"
"github.com/rusq/slackdump/v2/fsadapter"
"github.com/rusq/slackdump/v2/logger"
)

@@ -18,9 +16,7 @@ const (
)

type App struct {
sd *slackdump.Session
tmpl *template.Template
fs fsadapter.FS
sd *slackdump.Session

prov auth.Provider
cfg Config
@@ -32,40 +28,23 @@ func New(cfg Config, provider auth.Provider) (*App, error) {
if err := cfg.Validate(); err != nil {
return nil, err
}
tmpl, err := cfg.compileTemplates()
if err != nil {
return nil, err
}
fs, err := fsadapter.ForFilename(cfg.Output.Base)
if err != nil {
return nil, err
}
app := &App{cfg: cfg, prov: provider, tmpl: tmpl, fs: fs}
app := &App{cfg: cfg, prov: provider}
return app, nil
}

func (app *App) Run(ctx context.Context) error {
ctx, task := trace.NewTask(ctx, "app.Run")
defer task.End()

if err := app.initSlackdump(ctx); err != nil {
return err
}

start := time.Now()

var err error
switch {
case app.cfg.ListFlags.FlagsPresent():
err = app.runListEntities(ctx)
case app.cfg.ExportName != "":
app.l().Debug("export mode ON")
err = app.runExport(ctx)
default:
if app.cfg.ExportName != "" {
err = app.Export(ctx, app.cfg.ExportName)
} else {
err = app.runDump(ctx)
}

if err != nil {
trace.Log(ctx, "error", err.Error())
return err
}

@@ -75,57 +54,26 @@ func (app *App) Run(ctx context.Context) error {

// Close closes all open handles.
func (app *App) Close() error {
return fsadapter.Close(app.fs)
}

// initSlackdump initialises the slack dumper app.
func (app *App) initSlackdump(ctx context.Context) error {
sd, err := slackdump.NewWithOptions(
ctx,
app.prov,
app.cfg.Options,
)
if err != nil {
return err
}
app.sd = sd
app.sd.SetFS(app.fs)
return nil
}

func (app *App) runListEntities(ctx context.Context) error {
ctx, task := trace.NewTask(ctx, "runListEntities")
defer task.End()

if err := app.listEntities(ctx, app.cfg.Output, app.cfg.ListFlags); err != nil {
return err
}

return nil
}

func (app *App) runExport(ctx context.Context) error {
ctx, task := trace.NewTask(ctx, "runExport")
defer task.End()

if err := app.Export(ctx, app.cfg.ExportName); err != nil {
return err
}

return nil
}

func (app *App) runDump(ctx context.Context) error {
ctx, task := trace.NewTask(ctx, "runDump")
defer task.End()

n, err := app.dump(ctx, app.cfg.Input)
dm, err := newDump(ctx, &app.cfg, app.prov)
if err != nil {
return err
}

app.l().Printf("dumped %d item(s)", n)
return nil
if app.cfg.ListFlags.FlagsPresent() {
err = dm.List(ctx)
} else {
var n int
n, err = dm.Dump(ctx)
app.l().Printf("dumped %d item(s)", n)
}
return err
}

func (app *App) l() logger.Interface {
4 changes: 2 additions & 2 deletions internal/app/config.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,8 @@ const (
filenameTmplName = "fnt"
)

var errSkip = errors.New("skip")

type Config struct {
ListFlags ListFlags

@@ -45,8 +47,6 @@ type Input struct {

var (
ErrInvalidInput = errors.New("no valid input")

errSkip = errors.New("skip")
)

func (in *Input) IsValid() bool {
146 changes: 127 additions & 19 deletions internal/app/dump.go
Original file line number Diff line number Diff line change
@@ -5,13 +5,43 @@ import (
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"os"
"strings"
"time"

"github.com/rusq/slackdump/v2"
"github.com/rusq/slackdump/v2/auth"
"github.com/rusq/slackdump/v2/fsadapter"
"github.com/rusq/slackdump/v2/internal/structures"
"github.com/rusq/slackdump/v2/logger"
"github.com/rusq/slackdump/v2/types"
)

type dump struct {
sess *slackdump.Session
cfg *Config

log logger.Interface
}

func newDump(ctx context.Context, cfg *Config, prov auth.Provider) (*dump, error) {
sess, err := slackdump.NewWithOptions(ctx, prov, cfg.Options)
if err != nil {
return nil, err
}

var l logger.Interface
if cfg.Options.Logger == nil {
l = logger.Default
} else {
l = cfg.Options.Logger
}

return &dump{sess: sess, cfg: cfg, log: l}, nil
}

// dump dumps the input, if dumpfiles is true, it will save the files into a
// respective directory with ID of the channel as the name. If generateText is
// true, it will additionally format the conversation as text file and write it
@@ -27,15 +57,27 @@ import (
// +--<ID>.json - json file with conversation and users
// +--<ID>.txt - formatted conversation in text format, if generateText is true.
//
func (app *App) dump(ctx context.Context, input Input) (int, error) {
if !input.IsValid() {
func (app *dump) Dump(ctx context.Context) (int, error) {
if !app.cfg.Input.IsValid() {
return 0, errors.New("no valid input")
}

fs, err := fsadapter.ForFilename(app.cfg.Output.Base)
if err != nil {
return 0, err
}
defer fsadapter.Close(fs)
app.sess.SetFS(fs)

tmpl, err := app.cfg.compileTemplates()
if err != nil {
return 0, err
}

total := 0
if err := input.producer(func(channelID string) error {
if err := app.dumpOne(ctx, channelID, app.sd.Dump); err != nil {
app.l().Printf("error processing: %q (conversation will be skipped): %s", channelID, err)
if err := app.cfg.Input.producer(func(channelID string) error {
if err := app.dumpOne(ctx, fs, tmpl, channelID, app.sess.Dump); err != nil {
app.log.Printf("error processing: %q (conversation will be skipped): %s", channelID, err)
return errSkip
}
total++
@@ -50,9 +92,9 @@ type dumpFunc func(context.Context, string, time.Time, time.Time, ...slackdump.P

// renderFilename returns the filename that is rendered according to the
// file naming template.
func (app *App) renderFilename(c *types.Conversation) string {
func renderFilename(tmpl *template.Template, c *types.Conversation) string {
var buf strings.Builder
if err := app.tmpl.ExecuteTemplate(&buf, filenameTmplName, c); err != nil {
if err := tmpl.ExecuteTemplate(&buf, filenameTmplName, c); err != nil {
// this should nevar happen
panic(err)
}
@@ -61,32 +103,31 @@ func (app *App) renderFilename(c *types.Conversation) string {

// dumpOneChannel dumps just one channel specified by channelInput. If
// generateText is true, it will also generate a ID.txt text file.
func (app *App) dumpOne(ctx context.Context, channelInput string, fn dumpFunc) error {
func (app *dump) dumpOne(ctx context.Context, fs fsadapter.FS, filetmpl *template.Template, channelInput string, fn dumpFunc) error {
cnv, err := fn(ctx, channelInput, time.Time(app.cfg.Oldest), time.Time(app.cfg.Latest))
if err != nil {
return err
}

return app.writeFiles(app.renderFilename(cnv), cnv)
return app.writeFiles(fs, renderFilename(filetmpl, cnv), cnv)
}

// writeFiles writes the conversation to disk. If text output is set, it will
// also generate a text file having the same name as JSON file.
func (app *App) writeFiles(name string, cnv *types.Conversation) error {
if err := app.writeJSON(name+".json", cnv); err != nil {
func (app *dump) writeFiles(fs fsadapter.FS, name string, cnv *types.Conversation) error {
if err := app.writeJSON(fs, name+".json", cnv); err != nil {
return err
}
if app.cfg.Output.IsText() {
if err := app.writeText(name+".txt", cnv); err != nil {
if err := app.writeText(fs, name+".txt", cnv); err != nil {
return err
}
}
return nil
}

func (app *App) writeJSON(filename string, m any) error {
app.l().Printf("generating %s", filename)
f, err := app.fs.Create(filename)
func (app *dump) writeJSON(fs fsadapter.FS, filename string, m any) error {
f, err := fs.Create(filename)
if err != nil {
return fmt.Errorf("error writing %q: %w", filename, err)
}
@@ -100,13 +141,80 @@ func (app *App) writeJSON(filename string, m any) error {
return nil
}

func (app *App) writeText(filename string, m *types.Conversation) error {
app.l().Printf("generating %s", filename)
f, err := app.fs.Create(filename)
func (app *dump) writeText(fs fsadapter.FS, filename string, m *types.Conversation) error {
app.log.Printf("generating %s", filename)
f, err := fs.Create(filename)
if err != nil {
return fmt.Errorf("error writing %q: %w", filename, err)
}
defer f.Close()

return m.ToText(f, app.sd.UserIndex)
return m.ToText(f, app.sess.UserIndex)
}

// reporter is an interface defining output functions
type reporter interface {
ToText(w io.Writer, ui structures.UserIndex) error
}

// List lists the supported entities, and writes the output to the output
// defined in the app.cfg.
func (app *dump) List(ctx context.Context) error {
f, err := createFile(app.cfg.Output.Filename)
if err != nil {
return err
}
defer f.Close()

app.log.Print("retrieving data...")
rep, err := app.fetchEntity(ctx, app.cfg.ListFlags)
if err != nil {
return err
}

if err := app.formatEntity(f, rep, app.cfg.Output); err != nil {
return err
}
return nil
}

// createFile creates the file, or opens the Stdout, if the filename is "-".
// It will return an error, if things go pear-shaped.
func createFile(filename string) (f io.WriteCloser, err error) {
if filename == "-" {
f = os.Stdout
return
}
return os.Create(filename)
}

// fetchEntity retrieves the data from the API according to the ListFlags.
func (dm *dump) fetchEntity(ctx context.Context, listFlags ListFlags) (rep reporter, err error) {
switch {
case listFlags.Channels:
rep, err = dm.sess.GetChannels(ctx)
if err != nil {
return
}
case listFlags.Users:
rep, err = dm.sess.GetUsers(ctx)
if err != nil {
return
}
default:
err = errors.New("nothing to do")
}
return
}

// formatEntity formats reporter output as defined in the "Output".
func (app *dump) formatEntity(w io.Writer, rep reporter, output Output) error {
switch output.Format {
case OutputTypeText:
return rep.ToText(w, app.sess.UserIndex)
case OutputTypeJSON:
enc := json.NewEncoder(w)
return enc.Encode(rep)
}
return errors.New("invalid Output format")
}
Loading