Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cmd,common,server: Add LP_EXTEND_TIMEOUTS environment variable #2392

Merged
merged 1 commit into from
May 19, 2022
Merged

cmd,common,server: Add LP_EXTEND_TIMEOUTS environment variable #2392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG_PENDING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
- \#2383 Add E2E Tests for checking Livepeer on-chain interactions (@leszko)

#### Broadcaster
- \#2392 Add LP_EXTEND_TIMEOUTS env variable to extend timeouts for Stream Tester (@leszko)

#### Orchestrator

Expand Down
15 changes: 15 additions & 0 deletions cmd/livepeer/livepeer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ import (
"flag"
"fmt"
"github.com/livepeer/go-livepeer/cmd/livepeer/starter"
"github.com/livepeer/go-livepeer/common"
"os"
"os/signal"
"runtime"
"strconv"
"time"

"github.com/livepeer/livepeer-data/pkg/mistconnector"
Expand Down Expand Up @@ -48,6 +50,7 @@ func main() {
}

vFlag.Value.Set(*verbosity)
extendTimeouts()

cfg = updateNilsForUnsetFlags(cfg)

Expand Down Expand Up @@ -198,3 +201,15 @@ func updateNilsForUnsetFlags(cfg starter.LivepeerConfig) starter.LivepeerConfig

return res
}

// extendTimeouts extends transcoding timeouts for the testing purpose.
// This functionality is intended for Stream Tester to avoid timing out while measuring orchestrator performance.
func extendTimeouts() {
if boolVal, _ := strconv.ParseBool(os.Getenv("LP_EXTEND_TIMEOUTS")); boolVal {
// Make all timeouts 8s for the common segment durations
common.SegUploadTimeoutMultiplier = 4.0
common.SegmentUploadTimeout = 8 * time.Second
common.HTTPDialTimeout = 8 * time.Second
common.SegHttpPushTimeoutMultiplier = 4.0
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this value be any different than the default?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's the same, but anyway I think it's good to set it up here, to make sure that it's still ok when the default changes.

}
}
11 changes: 10 additions & 1 deletion common/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,19 @@ import (
"google.golang.org/grpc/peer"
)

// HTTPDialTimeout timeout used to establish an HTTP connection between nodes
var HTTPDialTimeout = 2 * time.Second

// HTTPTimeout timeout used in HTTP connections between nodes
var HTTPTimeout = 8 * time.Second

// SegmentUploadTimeout timeout used in HTTP connections for uploading the segment
// SegHttpPushTimeoutMultiplier used in the HTTP connection for pushing the segment
var SegHttpPushTimeoutMultiplier = 4.0

// SegUploadTimeoutMultiplier used in HTTP connection for uploading the segment
var SegUploadTimeoutMultiplier = 0.5

// SegmentUploadTimeout timeout used in HTTP connections for uploading the segment duration is not defined
var SegmentUploadTimeout = 2 * time.Second

// Max Segment Duration
Expand Down
11 changes: 3 additions & 8 deletions server/segment_rpc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,6 @@ const segmentHeader = "Livepeer-Segment"

const pixelEstimateMultiplier = 1.02

const segUploadTimeoutMultiplier = 0.5
const segHttpPushTimeoutMultiplier = 4.0

var errSegEncoding = errors.New("ErrorSegEncoding")
var errSegSig = errors.New("ErrSegSig")
var errFormat = errors.New("unrecognized profile output format")
Expand All @@ -48,14 +45,12 @@ var errEncoder = errors.New("unrecognized video codec")
var errDuration = errors.New("invalid duration")
var errCapCompat = errors.New("incompatible capabilities")

var dialTimeout = 2 * time.Second

var tlsConfig = &tls.Config{InsecureSkipVerify: true}
var httpClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsConfig,
DialTLSContext: func(ctx context.Context, network, addr string) (gonet.Conn, error) {
cctx, cancel := context.WithTimeout(ctx, dialTimeout)
cctx, cancel := context.WithTimeout(ctx, common.HTTPDialTimeout)
defer cancel()

tlsDialer := &tls.Dialer{Config: tlsConfig}
Expand Down Expand Up @@ -472,12 +467,12 @@ func SubmitSegment(ctx context.Context, sess *BroadcastSession, seg *stream.HLSS
// timeout for the whole HTTP call: segment upload, transcoding, reading response
httpTimeout := common.HTTPTimeout
// set a minimum timeout to accommodate transport / processing overhead
paddedDur := segHttpPushTimeoutMultiplier * seg.Duration
paddedDur := common.SegHttpPushTimeoutMultiplier * seg.Duration
if paddedDur > httpTimeout.Seconds() {
httpTimeout = time.Duration(paddedDur * float64(time.Second))
}
// timeout for the segment upload, until HTTP returns OK 200
uploadTimeout := time.Duration(segUploadTimeoutMultiplier * seg.Duration * float64(time.Second))
uploadTimeout := time.Duration(common.SegUploadTimeoutMultiplier * seg.Duration * float64(time.Second))
if uploadTimeout <= 0 {
uploadTimeout = common.SegmentUploadTimeout
}
Expand Down