Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hooks: Move into pkgs/hooks #976

Merged
merged 10 commits into from
Aug 22, 2023
Merged

hooks: Move into pkgs/hooks #976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
bdc00dc
hooks: Move implementations in `pkgs/hooks`
Acconut Jul 3, 2023
802589d
Merge branch 'v2' into v2-hooks-pkg
Acconut Jul 3, 2023
ac45632
misc: Upgrade go.mod to Go 1.20
Acconut Jul 3, 2023
124ce13
hooks: Use built-in function for testing slice elements
Acconut Jul 3, 2023
1562258
Move setup logic into hooks package
Acconut Aug 21, 2023
8919d97
Merge branch 'v2' of github.com:tus/tusd into v2-hooks-pkg
Acconut Aug 21, 2023
5a17ef7
Implement tests
Acconut Aug 21, 2023
ecbc065
Remove duplicate import
Acconut Aug 22, 2023
abdef06
Documentation
Acconut Aug 22, 2023
60b0107
Minor cleanups
Acconut Aug 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cmd/tusd/cli/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import (
"path/filepath"
"strings"

"github.com/tus/tusd/v2/cmd/tusd/cli/hooks"
"github.com/tus/tusd/v2/pkg/hooks"
"golang.org/x/exp/slices"
)

var Flags struct {
Expand Down Expand Up @@ -134,7 +135,7 @@ func SetEnabledHooks() {
for i, h := range slc {
slc[i] = strings.TrimSpace(h)

if !hookTypeInSlice(hooks.HookType(h), hooks.AvailableHooks) {
if !slices.Contains(hooks.AvailableHooks, hooks.HookType(h)) {
stderr.Fatalf("Unknown hook event type in -hooks-enabled-events flag: %s", h)
}

Expand Down
169 changes: 10 additions & 159 deletions cmd/tusd/cli/hooks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,95 +3,25 @@ package cli
import (
"strings"

"github.com/tus/tusd/v2/cmd/tusd/cli/hooks"
"github.com/tus/tusd/v2/pkg/handler"
"github.com/tus/tusd/v2/pkg/hooks"
"github.com/tus/tusd/v2/pkg/hooks/file"
"github.com/tus/tusd/v2/pkg/hooks/grpc"
"github.com/tus/tusd/v2/pkg/hooks/http"
"github.com/tus/tusd/v2/pkg/hooks/plugin"
)

var hookHandler hooks.HookHandler = nil

func hookTypeInSlice(a hooks.HookType, list []hooks.HookType) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}

func preCreateCallback(event handler.HookEvent) (handler.HTTPResponse, handler.FileInfoChanges, error) {
ok, hookRes, err := invokeHookSync(hooks.HookPreCreate, event)
if !ok || err != nil {
return handler.HTTPResponse{}, handler.FileInfoChanges{}, err
}

httpRes := hookRes.HTTPResponse

// If the hook response includes the instruction to reject the upload, reuse the error code
// and message from ErrUploadRejectedByServer, but also include custom HTTP response values.
if hookRes.RejectUpload {
err := handler.ErrUploadRejectedByServer
err.HTTPResponse = err.HTTPResponse.MergeWith(httpRes)

return handler.HTTPResponse{}, handler.FileInfoChanges{}, err
}

// Pass any changes regarding file info from the hook to the handler.
changes := hookRes.ChangeFileInfo
return httpRes, changes, nil
}

func preFinishCallback(event handler.HookEvent) (handler.HTTPResponse, error) {
ok, hookRes, err := invokeHookSync(hooks.HookPreFinish, event)
if !ok || err != nil {
return handler.HTTPResponse{}, err
}

httpRes := hookRes.HTTPResponse
return httpRes, nil
}

func postReceiveCallback(event handler.HookEvent) {
ok, hookRes, _ := invokeHookSync(hooks.HookPostReceive, event)
// invokeHookSync already logs the error, if any occurs. So by checking `ok`, we can ensure
// that the hook finished successfully
if !ok {
return
}

if hookRes.StopUpload {
logEv(stdout, "HookStopUpload", "id", event.Upload.ID)

// TODO: Control response for PATCH request
event.Upload.StopUpload()
}
}

func SetupHookMetrics() {
MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostFinish)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostTerminate)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostReceive)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostCreate)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreCreate)).Add(0)
MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreFinish)).Add(0)
MetricsHookInvocationsTotal.WithLabelValues(string(hooks.HookPostFinish)).Add(0)
MetricsHookInvocationsTotal.WithLabelValues(string(hooks.HookPostTerminate)).Add(0)
MetricsHookInvocationsTotal.WithLabelValues(string(hooks.HookPostReceive)).Add(0)
MetricsHookInvocationsTotal.WithLabelValues(string(hooks.HookPostCreate)).Add(0)
MetricsHookInvocationsTotal.WithLabelValues(string(hooks.HookPreCreate)).Add(0)
MetricsHookInvocationsTotal.WithLabelValues(string(hooks.HookPreFinish)).Add(0)
}

func SetupPreHooks(config *handler.Config) error {
func getHookHandler(config *handler.Config) hooks.HookHandler {
if Flags.FileHooksDir != "" {
stdout.Printf("Using '%s' for hooks", Flags.FileHooksDir)

hookHandler = &hooks.FileHook{
return &file.FileHook{
Directory: Flags.FileHooksDir,
}
} else if Flags.HttpHooksEndpoint != "" {
stdout.Printf("Using '%s' as the endpoint for hooks", Flags.HttpHooksEndpoint)

hookHandler = &hooks.HttpHook{
return &http.HttpHook{
Endpoint: Flags.HttpHooksEndpoint,
MaxRetries: Flags.HttpHooksRetry,
Backoff: Flags.HttpHooksBackoff,
Expand All @@ -100,97 +30,18 @@ func SetupPreHooks(config *handler.Config) error {
} else if Flags.GrpcHooksEndpoint != "" {
stdout.Printf("Using '%s' as the endpoint for gRPC hooks", Flags.GrpcHooksEndpoint)

hookHandler = &hooks.GrpcHook{
return &grpc.GrpcHook{
Endpoint: Flags.GrpcHooksEndpoint,
MaxRetries: Flags.GrpcHooksRetry,
Backoff: Flags.GrpcHooksBackoff,
}
} else if Flags.PluginHookPath != "" {
stdout.Printf("Using '%s' to load plugin for hooks", Flags.PluginHookPath)

hookHandler = &hooks.PluginHook{
return &plugin.PluginHook{
Path: Flags.PluginHookPath,
}
} else {
return nil
}

var enabledHooksString []string
for _, h := range Flags.EnabledHooks {
enabledHooksString = append(enabledHooksString, string(h))
}

stdout.Printf("Enabled hook events: %s", strings.Join(enabledHooksString, ", "))

if err := hookHandler.Setup(); err != nil {
return err
}

config.PreUploadCreateCallback = preCreateCallback
config.PreFinishResponseCallback = preFinishCallback

return nil
}

func SetupPostHooks(handler *handler.Handler) {
go func() {
for {
select {
case event := <-handler.CompleteUploads:
invokeHookAsync(hooks.HookPostFinish, event)
case event := <-handler.TerminatedUploads:
invokeHookAsync(hooks.HookPostTerminate, event)
case event := <-handler.CreatedUploads:
invokeHookAsync(hooks.HookPostCreate, event)
case event := <-handler.UploadProgress:
go postReceiveCallback(event)
}
}
}()
}

func invokeHookAsync(typ hooks.HookType, event handler.HookEvent) {
go func() {
// Error handling is taken care by the function.
_, _, _ = invokeHookSync(typ, event)
}()
}

// invokeHookSync executes a hook of the given type with the given event data. If
// the hook was not executed properly (e.g. an error occurred or not handler is installed),
// `ok` will be false and `res` is not filled. `err` can contain the underlying error.
// If `ok` is true, `res` contains the response as retrieved from the hook.
// Therefore, a caller should always check `ok` and `err` before assuming that the
// hook completed successfully.
func invokeHookSync(typ hooks.HookType, event handler.HookEvent) (ok bool, res hooks.HookResponse, err error) {
// Stop, if no hook handler is installed or this hook event is not enabled
if hookHandler == nil || !hookTypeInSlice(typ, Flags.EnabledHooks) {
return false, hooks.HookResponse{}, nil
}

MetricsHookInvocationsTotal.WithLabelValues(string(typ)).Add(1)

id := event.Upload.ID

if Flags.VerboseOutput {
logEv(stdout, "HookInvocationStart", "type", string(typ), "id", id)
}

res, err = hookHandler.InvokeHook(hooks.HookRequest{
Type: typ,
Event: event,
})
if err != nil {
// If an error occurs during the hook execution, we log and track the error, but do not
// return a hook response.
logEv(stderr, "HookInvocationError", "type", string(typ), "id", id, "error", err.Error())
MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1)
return false, hooks.HookResponse{}, err
}

if Flags.VerboseOutput {
logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", id)
}

return true, res, nil
}
76 changes: 0 additions & 76 deletions cmd/tusd/cli/hooks/hooks.go
View file
Open in desktop

This file was deleted.

21 changes: 3 additions & 18 deletions cmd/tusd/cli/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"net/http"

"github.com/tus/tusd/v2/pkg/handler"
"github.com/tus/tusd/v2/pkg/hooks"
"github.com/tus/tusd/v2/pkg/prometheuscollector"

"github.com/prometheus/client_golang/prometheus"
Expand All @@ -15,26 +16,10 @@ var MetricsOpenConnections = prometheus.NewGauge(prometheus.GaugeOpts{
Help: "Current number of open connections.",
})

var MetricsHookErrorsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "tusd_hook_errors_total",
Help: "Total number of execution errors per hook type.",
},
[]string{"hooktype"},
)

var MetricsHookInvocationsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "tusd_hook_invocations_total",
Help: "Total number of invocations per hook type.",
},
[]string{"hooktype"},
)

func SetupMetrics(mux *http.ServeMux, handler *handler.Handler) {
prometheus.MustRegister(MetricsOpenConnections)
prometheus.MustRegister(MetricsHookErrorsTotal)
prometheus.MustRegister(MetricsHookInvocationsTotal)
prometheus.MustRegister(hooks.MetricsHookErrorsTotal)
prometheus.MustRegister(hooks.MetricsHookInvocationsTotal)
prometheus.MustRegister(prometheuscollector.New(handler.Metrics))

stdout.Printf("Using %s as the metrics path.\n", Flags.MetricsPath)
Expand Down
Loading