diff --git a/TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs
index fee8dfcc..07945f8e 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs
@@ -8,7 +8,7 @@ internal abstract class TwitchDownloaderArgs
[Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")]
public bool? ShowBanner { get; set; }
- [Option("log-level", Default = Models.LogLevel.Status | LogLevel.Info | LogLevel.Warning | LogLevel.Error, HelpText = "Sets the log level flags. Applicable values are: None, Status, Verbose, Info, Warning, Error, Ffmpeg.")]
+ [Option("log-level", Default = LogLevel.Status | LogLevel.Info | LogLevel.Warning | LogLevel.Error, HelpText = "Sets the log level flags. Applicable values are: None, Status, Verbose, Info, Warning, Error, Ffmpeg.")]
public LogLevel LogLevel { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/Interfaces/ITaskLogger.cs b/TwitchDownloaderCore/Interfaces/ITaskLogger.cs
index facfcc20..7f2d5c09 100644
--- a/TwitchDownloaderCore/Interfaces/ITaskLogger.cs
+++ b/TwitchDownloaderCore/Interfaces/ITaskLogger.cs
@@ -4,7 +4,6 @@ namespace TwitchDownloaderCore.Interfaces
{
public interface ITaskLogger
{
- // TODO: Add DefaultInterpolatedStringHandler overloads once log levels are implemented for zero-alloc logging
void LogVerbose(string logMessage);
void LogVerbose(DefaultInterpolatedStringHandler logMessage);
void LogInfo(string logMessage);
diff --git a/TwitchDownloaderCore/Tools/DownloadTools.cs b/TwitchDownloaderCore/Tools/DownloadTools.cs
new file mode 100644
index 00000000..90ed883d
--- /dev/null
+++ b/TwitchDownloaderCore/Tools/DownloadTools.cs
@@ -0,0 +1,94 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using TwitchDownloaderCore.Interfaces;
+
+namespace TwitchDownloaderCore.Tools
+{
+ public static class DownloadTools
+ {
+ ///
+ /// Downloads the requested to the without storing it in memory.
+ ///
+ /// The to perform the download operation.
+ /// The url of the file to download.
+ /// The path to the file where download will be saved.
+ /// The maximum download speed in kibibytes per second, or -1 for no maximum.
+ /// Logger.
+ /// A containing a to cancel the operation.
+ /// The may be canceled by this method.
+ public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
+ {
+ var request = new HttpRequestMessage(HttpMethod.Get, url);
+
+ var cancellationToken = cancellationTokenSource?.Token ?? CancellationToken.None;
+
+ using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
+ response.EnsureSuccessStatusCode();
+
+ // Why are we setting a CTS CancelAfter timer? See lay295#265
+ const int SIXTY_SECONDS = 60;
+ if (throttleKib == -1 || !response.Content.Headers.ContentLength.HasValue)
+ {
+ cancellationTokenSource?.CancelAfter(TimeSpan.FromSeconds(SIXTY_SECONDS));
+ }
+ else
+ {
+ const double ONE_KIBIBYTE = 1024d;
+ cancellationTokenSource?.CancelAfter(TimeSpan.FromSeconds(Math.Max(
+ SIXTY_SECONDS,
+ response.Content.Headers.ContentLength!.Value / ONE_KIBIBYTE / throttleKib * 8 // Allow up to 8x the shortest download time given the thread bandwidth
+ )));
+ }
+
+ switch (throttleKib)
+ {
+ case -1:
+ {
+ await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
+ await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+ break;
+ }
+ default:
+ {
+ try
+ {
+ await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
+ await using var throttledStream = new ThrottledStream(contentStream, throttleKib);
+ await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
+ await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+ }
+ catch (IOException ex) when (ex.Message.Contains("EOF"))
+ {
+ // If we get an exception for EOF, it may be related to the throttler. Try again without it.
+ logger.LogVerbose($"Unexpected EOF, retrying without bandwidth throttle. Message: {ex.Message}.");
+ await Task.Delay(2_000, cancellationToken);
+ goto case -1;
+ }
+ break;
+ }
+ }
+
+ // Reset the cts timer so it can be reused for the next download on this thread.
+ // Is there a friendlier way to do this? Yes. Does it involve creating and destroying 4,000 CancellationTokenSources that are almost never cancelled? Also Yes.
+ cancellationTokenSource?.CancelAfter(TimeSpan.FromMilliseconds(uint.MaxValue - 1));
+ }
+
+
+ ///
+ /// Some old twitch VODs have files with a query string at the end such as 1.ts?offset=blah which isn't a valid filename
+ ///
+ public static string RemoveQueryString(string inputString)
+ {
+ var queryIndex = inputString.IndexOf('?');
+ if (queryIndex == -1)
+ {
+ return inputString;
+ }
+
+ return inputString[..queryIndex];
+ }
+ }
+}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/Tools/VideoDownloadThread.cs b/TwitchDownloaderCore/Tools/VideoDownloadThread.cs
new file mode 100644
index 00000000..e4162096
--- /dev/null
+++ b/TwitchDownloaderCore/Tools/VideoDownloadThread.cs
@@ -0,0 +1,153 @@
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using TwitchDownloaderCore.Interfaces;
+
+namespace TwitchDownloaderCore.Tools
+{
+ internal sealed record VideoDownloadThread
+ {
+ private readonly ConcurrentQueue _videoPartsQueue;
+ private readonly HttpClient _client;
+ private readonly Uri _baseUrl;
+ private readonly string _cacheFolder;
+ private readonly DateTimeOffset _vodAirDate;
+ private TimeSpan VodAge => DateTimeOffset.UtcNow - _vodAirDate;
+ private readonly int _throttleKib;
+ private readonly ITaskLogger _logger;
+ private readonly CancellationToken _cancellationToken;
+ public Task ThreadTask { get; private set; }
+
+ public VideoDownloadThread(ConcurrentQueue videoPartsQueue, HttpClient httpClient, Uri baseUrl, string cacheFolder, DateTimeOffset vodAirDate, int throttleKib, ITaskLogger logger, CancellationToken cancellationToken)
+ {
+ _videoPartsQueue = videoPartsQueue;
+ _client = httpClient;
+ _baseUrl = baseUrl;
+ _cacheFolder = cacheFolder;
+ _vodAirDate = vodAirDate;
+ _throttleKib = throttleKib;
+ _logger = logger;
+ _cancellationToken = cancellationToken;
+ StartDownload();
+ }
+
+ public void StartDownload()
+ {
+ if (ThreadTask is { Status: TaskStatus.Created or TaskStatus.WaitingForActivation or TaskStatus.WaitingToRun or TaskStatus.Running })
+ {
+ throw new InvalidOperationException($"Tried to start a thread that was already running or waiting to run ({ThreadTask.Status}).");
+ }
+
+ ThreadTask = Task.Factory.StartNew(
+ ExecuteDownloadThread,
+ _cancellationToken,
+ TaskCreationOptions.LongRunning,
+ TaskScheduler.Current);
+ }
+
+ private void ExecuteDownloadThread()
+ {
+ using var cts = new CancellationTokenSource();
+ _cancellationToken.Register(PropagateCancel, cts);
+
+ while (!_videoPartsQueue.IsEmpty)
+ {
+ _cancellationToken.ThrowIfCancellationRequested();
+
+ string videoPart = null;
+ try
+ {
+ if (_videoPartsQueue.TryDequeue(out videoPart))
+ {
+ DownloadVideoPartAsync(videoPart, cts).GetAwaiter().GetResult();
+ }
+ }
+ catch
+ {
+ if (videoPart != null && !_cancellationToken.IsCancellationRequested)
+ {
+ // Requeue the video part now instead of deferring to the verifier since we already know it's bad
+ _videoPartsQueue.Enqueue(videoPart);
+ }
+
+ throw;
+ }
+
+ const int A_PRIME_NUMBER = 71;
+ Thread.Sleep(A_PRIME_NUMBER);
+ }
+ }
+
+ private static void PropagateCancel(object tokenSourceToCancel)
+ {
+ try
+ {
+ (tokenSourceToCancel as CancellationTokenSource)?.Cancel();
+ }
+ catch (ObjectDisposedException) { }
+ }
+
+ /// The may be canceled by this method.
+ private async Task DownloadVideoPartAsync(string videoPartName, CancellationTokenSource cancellationTokenSource)
+ {
+ var tryUnmute = VodAge < TimeSpan.FromHours(24);
+ var errorCount = 0;
+ var timeoutCount = 0;
+ while (true)
+ {
+ cancellationTokenSource.Token.ThrowIfCancellationRequested();
+
+ try
+ {
+ var partFile = Path.Combine(_cacheFolder, DownloadTools.RemoveQueryString(videoPartName));
+ if (tryUnmute && videoPartName.Contains("-muted"))
+ {
+ var unmutedPartName = videoPartName.Replace("-muted", "");
+ await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, unmutedPartName), partFile, _throttleKib, _logger, cancellationTokenSource);
+ }
+ else
+ {
+ await DownloadTools.DownloadFileAsync(_client, new Uri(_baseUrl, videoPartName), partFile, _throttleKib, _logger, cancellationTokenSource);
+ }
+
+ return;
+ }
+ catch (HttpRequestException ex) when (tryUnmute && ex.StatusCode is HttpStatusCode.Forbidden)
+ {
+ _logger.LogVerbose($"Received {ex.StatusCode}: {ex.StatusCode} when trying to unmute {videoPartName}. Disabling {nameof(tryUnmute)}.");
+ tryUnmute = false;
+
+ await Task.Delay(100, cancellationTokenSource.Token);
+ }
+ catch (HttpRequestException ex)
+ {
+ const int MAX_RETRIES = 10;
+
+ _logger.LogVerbose($"Received {(int)(ex.StatusCode ?? 0)}: {ex.StatusCode} for {videoPartName}. {MAX_RETRIES - (errorCount + 1)} retries left.");
+ if (++errorCount > MAX_RETRIES)
+ {
+ throw new HttpRequestException($"Video part {videoPartName} failed after {MAX_RETRIES} retries");
+ }
+
+ await Task.Delay(1_000 * errorCount, cancellationTokenSource.Token);
+ }
+ catch (TaskCanceledException ex) when (ex.Message.Contains("HttpClient.Timeout"))
+ {
+ const int MAX_RETRIES = 3;
+
+ _logger.LogVerbose($"{videoPartName} timed out. {MAX_RETRIES - (timeoutCount + 1)} retries left.");
+ if (++timeoutCount > MAX_RETRIES)
+ {
+ throw new HttpRequestException($"Video part {videoPartName} timed out {MAX_RETRIES} times");
+ }
+
+ await Task.Delay(5_000 * timeoutCount, cancellationTokenSource.Token);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs
index bcceab4f..47a9983f 100644
--- a/TwitchDownloaderCore/VideoDownloader.cs
+++ b/TwitchDownloaderCore/VideoDownloader.cs
@@ -67,7 +67,7 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
var videoLength = TimeSpan.FromSeconds(videoInfoResponse.data.video.lengthSeconds);
CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength);
- var (playlist, videoListCrop, vodAge) = await GetVideoPlaylist(playlistUrl, cancellationToken);
+ var (playlist, videoListCrop, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken);
if (Directory.Exists(downloadFolder))
Directory.Delete(downloadFolder, true);
@@ -75,11 +75,11 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
_progress.SetTemplateStatus("Downloading {0}% [2/5]", 0);
- await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, vodAge, cancellationToken);
+ await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken);
_progress.SetTemplateStatus("Verifying Parts {0}% [3/5]", 0);
- await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, vodAge, cancellationToken);
+ await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken);
_progress.SetTemplateStatus("Combining Parts {0}% [4/5]", 0);
@@ -165,79 +165,23 @@ private void CheckAvailableStorageSpace(int bandwidth, TimeSpan videoLength)
}
}
- private async Task DownloadVideoPartsAsync(IEnumerable playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken)
+ private async Task DownloadVideoPartsAsync(IEnumerable playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, DateTimeOffset vodAirDate, CancellationToken cancellationToken)
{
var partCount = videoListCrop.End.Value - videoListCrop.Start.Value;
var videoPartsQueue = new ConcurrentQueue(playlist.Take(videoListCrop).Select(x => x.Path));
- var downloadTasks = new Task[downloadOptions.DownloadThreads];
+ var downloadThreads = new VideoDownloadThread[downloadOptions.DownloadThreads];
for (var i = 0; i < downloadOptions.DownloadThreads; i++)
{
- downloadTasks[i] = StartNewDownloadThread(videoPartsQueue, baseUrl, downloadFolder, vodAge, cancellationToken);
+ downloadThreads[i] = new VideoDownloadThread(videoPartsQueue, _httpClient, baseUrl, downloadFolder, vodAirDate, downloadOptions.ThrottleKib, _progress, cancellationToken);
}
- var downloadExceptions = await WaitForDownloadThreads(downloadTasks, videoPartsQueue, baseUrl, downloadFolder, vodAge, partCount, cancellationToken);
+ var downloadExceptions = await WaitForDownloadThreads(downloadThreads, videoPartsQueue, partCount, cancellationToken);
LogDownloadThreadExceptions(downloadExceptions);
}
- private Task StartNewDownloadThread(ConcurrentQueue videoPartsQueue, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken)
- {
- return Task.Factory.StartNew(
- ExecuteDownloadThread,
- new Tuple, HttpClient, Uri, string, double, int, CancellationToken>(
- videoPartsQueue, _httpClient, baseUrl, downloadFolder, vodAge, downloadOptions.ThrottleKib, cancellationToken),
- cancellationToken,
- TaskCreationOptions.LongRunning,
- TaskScheduler.Current);
-
- static void ExecuteDownloadThread(object state)
- {
- var (partQueue, httpClient, rootUrl, cacheFolder, videoAge, throttleKib, cancelToken) =
- (Tuple, HttpClient, Uri, string, double, int, CancellationToken>)state;
-
- using var cts = new CancellationTokenSource();
- cancelToken.Register(PropagateCancel, cts);
-
- while (!partQueue.IsEmpty)
- {
- cancelToken.ThrowIfCancellationRequested();
-
- string videoPart = null;
- try
- {
- if (partQueue.TryDequeue(out videoPart))
- {
- DownloadVideoPartAsync(httpClient, rootUrl, videoPart, cacheFolder, videoAge, throttleKib, cts).GetAwaiter().GetResult();
- }
- }
- catch
- {
- if (videoPart != null && !cancelToken.IsCancellationRequested)
- {
- // Requeue the video part now instead of deferring to the verifier since we already know it's bad
- partQueue.Enqueue(videoPart);
- }
-
- throw;
- }
-
- const int A_PRIME_NUMBER = 71;
- Thread.Sleep(A_PRIME_NUMBER);
- }
- }
-
- static void PropagateCancel(object tokenSourceToCancel)
- {
- try
- {
- ((CancellationTokenSource)tokenSourceToCancel)?.Cancel();
- }
- catch (ObjectDisposedException) { }
- }
- }
-
- private async Task> WaitForDownloadThreads(Task[] tasks, ConcurrentQueue videoPartsQueue, Uri baseUrl, string downloadFolder, double vodAge, int partCount, CancellationToken cancellationToken)
+ private async Task> WaitForDownloadThreads(VideoDownloadThread[] downloadThreads, ConcurrentQueue videoPartsQueue, int partCount, CancellationToken cancellationToken)
{
var allThreadsExited = false;
var previousDoneCount = 0;
@@ -254,9 +198,9 @@ private async Task> WaitForDownloadThreads(Task[]
}
allThreadsExited = true;
- for (var t = 0; t < tasks.Length; t++)
+ foreach (var thread in downloadThreads)
{
- var task = tasks[t];
+ var task = thread.ThreadTask;
if (task.IsFaulted)
{
@@ -264,7 +208,7 @@ private async Task> WaitForDownloadThreads(Task[]
if (restartedThreads <= maxRestartedThreads)
{
- tasks[t] = StartNewDownloadThread(videoPartsQueue, baseUrl, downloadFolder, vodAge, cancellationToken);
+ thread.StartDownload();
restartedThreads++;
}
}
@@ -325,7 +269,7 @@ private void LogDownloadThreadExceptions(IReadOnlyCollection download
_progress.LogInfo(sb.ToString());
}
- private async Task VerifyDownloadedParts(ICollection playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, double vodAge, CancellationToken cancellationToken)
+ private async Task VerifyDownloadedParts(ICollection playlist, Range videoListCrop, Uri baseUrl, string downloadFolder, DateTimeOffset vodAirDate, CancellationToken cancellationToken)
{
var failedParts = new List();
var partCount = videoListCrop.End.Value - videoListCrop.Start.Value;
@@ -333,7 +277,7 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang
foreach (var part in playlist.Take(videoListCrop))
{
- var filePath = Path.Combine(downloadFolder, RemoveQueryString(part.Path));
+ var filePath = Path.Combine(downloadFolder, DownloadTools.RemoveQueryString(part.Path));
if (!VerifyVideoPart(filePath))
{
failedParts.Add(part);
@@ -362,7 +306,7 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang
}
_progress.LogInfo($"The following parts will be redownloaded: {string.Join(", ", failedParts)}");
- await DownloadVideoPartsAsync(failedParts, videoListCrop, baseUrl, downloadFolder, vodAge, cancellationToken);
+ await DownloadVideoPartsAsync(failedParts, videoListCrop, baseUrl, downloadFolder, vodAirDate, cancellationToken);
}
}
@@ -450,71 +394,21 @@ private void HandleFfmpegOutput(string output, Regex encodingTimeRegex, TimeSpan
_progress.ReportProgress(Math.Clamp(percent, 0, 100));
}
- /// The may be canceled by this method.
- private static async Task DownloadVideoPartAsync(HttpClient httpClient, Uri baseUrl, string videoPartName, string downloadFolder, double vodAge, int throttleKib, CancellationTokenSource cancellationTokenSource)
- {
- bool tryUnmute = vodAge < 24;
- int errorCount = 0;
- int timeoutCount = 0;
- while (true)
- {
- cancellationTokenSource.Token.ThrowIfCancellationRequested();
-
- try
- {
- if (tryUnmute && videoPartName.Contains("-muted"))
- {
- await DownloadFileAsync(httpClient, new Uri(baseUrl, videoPartName.Replace("-muted", "")), Path.Combine(downloadFolder, RemoveQueryString(videoPartName)), throttleKib, cancellationTokenSource);
- }
- else
- {
- await DownloadFileAsync(httpClient, new Uri(baseUrl, videoPartName), Path.Combine(downloadFolder, RemoveQueryString(videoPartName)), throttleKib, cancellationTokenSource);
- }
-
- return;
- }
- catch (HttpRequestException ex) when (tryUnmute && ex.StatusCode is HttpStatusCode.Forbidden)
- {
- tryUnmute = false;
- }
- catch (HttpRequestException)
- {
- const int MAX_RETRIES = 10;
- if (++errorCount > MAX_RETRIES)
- {
- throw new HttpRequestException($"Video part {videoPartName} failed after {MAX_RETRIES} retries");
- }
-
- await Task.Delay(1_000 * errorCount, cancellationTokenSource.Token);
- }
- catch (TaskCanceledException ex) when (ex.Message.Contains("HttpClient.Timeout"))
- {
- const int MAX_RETRIES = 3;
- if (++timeoutCount > MAX_RETRIES)
- {
- throw new HttpRequestException($"Video part {videoPartName} timed out {MAX_RETRIES} times");
- }
-
- await Task.Delay(5_000 * timeoutCount, cancellationTokenSource.Token);
- }
- }
- }
-
- private async Task<(M3U8 playlist, Range cropRange, double vodAge)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken)
+ private async Task<(M3U8 playlist, Range cropRange, DateTimeOffset airDate)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken)
{
var playlistString = await _httpClient.GetStringAsync(playlistUrl, cancellationToken);
var playlist = M3U8.Parse(playlistString);
- double vodAge = 25;
+ var airDate = DateTimeOffset.UtcNow.AddHours(-25);
var airDateKvp = playlist.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:");
- if (DateTimeOffset.TryParse(airDateKvp.Value, out var airDate))
+ if (DateTimeOffset.TryParse(airDateKvp.Value, out var vodAirDate))
{
- vodAge = (DateTimeOffset.UtcNow - airDate).TotalHours;
+ airDate = vodAirDate;
}
var videoListCrop = GetStreamListCrop(playlist.Streams, downloadOptions);
- return (playlist, videoListCrop, vodAge);
+ return (playlist, videoListCrop, airDate);
}
private static Range GetStreamListCrop(IList streamList, VideoDownloadOptions downloadOptions)
@@ -572,72 +466,6 @@ private static Range GetStreamListCrop(IList streamList, VideoDownl
return m3u8.GetStreamOfQuality(downloadOptions.Quality);
}
- ///
- /// Downloads the requested to the without storing it in memory.
- ///
- /// The to perform the download operation.
- /// The url of the file to download.
- /// The path to the file where download will be saved.
- /// The maximum download speed in kibibytes per second, or -1 for no maximum.
- /// A containing a to cancel the operation.
- /// The may be canceled by this method.
- private static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, CancellationTokenSource cancellationTokenSource = null)
- {
- var request = new HttpRequestMessage(HttpMethod.Get, url);
-
- var cancellationToken = cancellationTokenSource?.Token ?? CancellationToken.None;
-
- using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
- response.EnsureSuccessStatusCode();
-
- // Why are we setting a CTS CancelAfter timer? See lay295#265
- const int SIXTY_SECONDS = 60;
- if (throttleKib == -1 || !response.Content.Headers.ContentLength.HasValue)
- {
- cancellationTokenSource?.CancelAfter(TimeSpan.FromSeconds(SIXTY_SECONDS));
- }
- else
- {
- const double ONE_KIBIBYTE = 1024d;
- cancellationTokenSource?.CancelAfter(TimeSpan.FromSeconds(Math.Max(
- SIXTY_SECONDS,
- response.Content.Headers.ContentLength!.Value / ONE_KIBIBYTE / throttleKib * 8 // Allow up to 8x the shortest download time given the thread bandwidth
- )));
- }
-
- switch (throttleKib)
- {
- case -1:
- {
- await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
- await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
- break;
- }
- default:
- {
- try
- {
- await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
- await using var throttledStream = new ThrottledStream(contentStream, throttleKib);
- await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
- await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
- }
- catch (IOException e) when (e.Message.Contains("EOF"))
- {
- // If we get an exception for EOF, it may be related to the throttler. Try again without it.
- // TODO: Log this somehow
- await Task.Delay(2_000, cancellationToken);
- goto case -1;
- }
- break;
- }
- }
-
- // Reset the cts timer so it can be reused for the next download on this thread.
- // Is there a friendlier way to do this? Yes. Does it involve creating and destroying 4,000 CancellationTokenSources that are almost never cancelled? Also Yes.
- cancellationTokenSource?.CancelAfter(TimeSpan.FromMilliseconds(uint.MaxValue - 1));
- }
-
private async Task CombineVideoParts(string downloadFolder, IEnumerable playlist, Range videoListCrop, CancellationToken cancellationToken)
{
DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder);
@@ -651,7 +479,7 @@ private async Task CombineVideoParts(string downloadFolder, IEnumerable