From 30b7330f30c18261a20325c6f3bc2ff1f80f20c2 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Wed, 18 May 2022 10:47:36 +0200 Subject: [PATCH] cmd,common,server: Add LP_EXTEND_TIMEOUTS environment variable --- CHANGELOG_PENDING.md | 1 + cmd/livepeer/livepeer.go | 15 +++++++++++++++ common/util.go | 11 ++++++++++- server/segment_rpc.go | 11 +++-------- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index ca86e6e078..54dd33482b 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -10,6 +10,7 @@ - \#2383 Add E2E Tests for checking Livepeer on-chain interactions (@leszko) #### Broadcaster +- \#2392 Add LP_EXTEND_TIMEOUTS env variable to extend timeouts for Stream Tester (@leszko) #### Orchestrator diff --git a/cmd/livepeer/livepeer.go b/cmd/livepeer/livepeer.go index ada1aa465c..fe17dd9a65 100644 --- a/cmd/livepeer/livepeer.go +++ b/cmd/livepeer/livepeer.go @@ -8,9 +8,11 @@ import ( "flag" "fmt" "github.com/livepeer/go-livepeer/cmd/livepeer/starter" + "github.com/livepeer/go-livepeer/common" "os" "os/signal" "runtime" + "strconv" "time" "github.com/livepeer/livepeer-data/pkg/mistconnector" @@ -48,6 +50,7 @@ func main() { } vFlag.Value.Set(*verbosity) + extendTimeouts() cfg = updateNilsForUnsetFlags(cfg) @@ -198,3 +201,15 @@ func updateNilsForUnsetFlags(cfg starter.LivepeerConfig) starter.LivepeerConfig return res } + +// extendTimeouts extends transcoding timeouts for the testing purpose. +// This functionality is intended for Stream Tester to avoid timing out while measuring orchestrator performance. +func extendTimeouts() { + if boolVal, _ := strconv.ParseBool(os.Getenv("LP_EXTEND_TIMEOUTS")); boolVal { + // Make all timeouts 8s for the common segment durations + common.SegUploadTimeoutMultiplier = 4.0 + common.SegmentUploadTimeout = 8 * time.Second + common.HTTPDialTimeout = 8 * time.Second + common.SegHttpPushTimeoutMultiplier = 4.0 + } +} diff --git a/common/util.go b/common/util.go index fe7981ef91..a00d51853d 100644 --- a/common/util.go +++ b/common/util.go @@ -26,10 +26,19 @@ import ( "google.golang.org/grpc/peer" ) +// HTTPDialTimeout timeout used to establish an HTTP connection between nodes +var HTTPDialTimeout = 2 * time.Second + // HTTPTimeout timeout used in HTTP connections between nodes var HTTPTimeout = 8 * time.Second -// SegmentUploadTimeout timeout used in HTTP connections for uploading the segment +// SegHttpPushTimeoutMultiplier used in the HTTP connection for pushing the segment +var SegHttpPushTimeoutMultiplier = 4.0 + +// SegUploadTimeoutMultiplier used in HTTP connection for uploading the segment +var SegUploadTimeoutMultiplier = 0.5 + +// SegmentUploadTimeout timeout used in HTTP connections for uploading the segment duration is not defined var SegmentUploadTimeout = 2 * time.Second // Max Segment Duration diff --git a/server/segment_rpc.go b/server/segment_rpc.go index b4b06857a4..0ac6350a5d 100644 --- a/server/segment_rpc.go +++ b/server/segment_rpc.go @@ -37,9 +37,6 @@ const segmentHeader = "Livepeer-Segment" const pixelEstimateMultiplier = 1.02 -const segUploadTimeoutMultiplier = 0.5 -const segHttpPushTimeoutMultiplier = 4.0 - var errSegEncoding = errors.New("ErrorSegEncoding") var errSegSig = errors.New("ErrSegSig") var errFormat = errors.New("unrecognized profile output format") @@ -48,14 +45,12 @@ var errEncoder = errors.New("unrecognized video codec") var errDuration = errors.New("invalid duration") var errCapCompat = errors.New("incompatible capabilities") -var dialTimeout = 2 * time.Second - var tlsConfig = &tls.Config{InsecureSkipVerify: true} var httpClient = &http.Client{ Transport: &http.Transport{ TLSClientConfig: tlsConfig, DialTLSContext: func(ctx context.Context, network, addr string) (gonet.Conn, error) { - cctx, cancel := context.WithTimeout(ctx, dialTimeout) + cctx, cancel := context.WithTimeout(ctx, common.HTTPDialTimeout) defer cancel() tlsDialer := &tls.Dialer{Config: tlsConfig} @@ -472,12 +467,12 @@ func SubmitSegment(ctx context.Context, sess *BroadcastSession, seg *stream.HLSS // timeout for the whole HTTP call: segment upload, transcoding, reading response httpTimeout := common.HTTPTimeout // set a minimum timeout to accommodate transport / processing overhead - paddedDur := segHttpPushTimeoutMultiplier * seg.Duration + paddedDur := common.SegHttpPushTimeoutMultiplier * seg.Duration if paddedDur > httpTimeout.Seconds() { httpTimeout = time.Duration(paddedDur * float64(time.Second)) } // timeout for the segment upload, until HTTP returns OK 200 - uploadTimeout := time.Duration(segUploadTimeoutMultiplier * seg.Duration * float64(time.Second)) + uploadTimeout := time.Duration(common.SegUploadTimeoutMultiplier * seg.Duration * float64(time.Second)) if uploadTimeout <= 0 { uploadTimeout = common.SegmentUploadTimeout }