From 058a052774ceca70b06078b8eb2e6847907f8102 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 29 Jun 2024 13:14:57 -0400 Subject: [PATCH 1/5] Make video finalization async --- TwitchDownloaderCore/VideoDownloader.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 27b7e24e..7cf6efb2 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -128,7 +128,7 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d var ffmpegRetries = 0; do { - ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, startOffset, endDuration, videoLength), cancellationToken); + ffmpegExitCode = await RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, startOffset, endDuration, videoLength, cancellationToken); if (ffmpegExitCode != 0) { _progress.LogError($"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds..."); @@ -326,7 +326,7 @@ private async Task VerifyDownloadedParts(ICollection playlist, Rang } } - private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, decimal endDuration, TimeSpan videoLength) + private async Task RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, decimal endDuration, TimeSpan videoLength, CancellationToken cancellationToken) { using var process = new Process { @@ -391,14 +391,14 @@ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string co process.Start(); process.BeginErrorReadLine(); - using var logWriter = File.AppendText(Path.Combine(tempFolder, "ffmpegLog.txt")); + await using var logWriter = File.AppendText(Path.Combine(tempFolder, "ffmpegLog.txt")); logWriter.AutoFlush = true; do // We cannot handle logging inside the ErrorDataReceived lambda because more than 1 can come in at once and cause a race condition. lay295#598 { - Thread.Sleep(100); + await Task.Delay(200, cancellationToken); while (!logQueue.IsEmpty && logQueue.TryDequeue(out var logMessage)) { - logWriter.WriteLine(logMessage); + await logWriter.WriteLineAsync(logMessage); } } while (!process.HasExited || !logQueue.IsEmpty); From 5074e8251c6d98b55c2d190b7b8738d2d1bd67b1 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 29 Jun 2024 13:15:29 -0400 Subject: [PATCH 2/5] Add length check to clip downloader --- TwitchDownloaderCore/ClipDownloader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index 4ea6393d..6fe2de22 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -98,7 +98,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF await EncodeClipWithMetadata(tempFile, outputFileInfo.FullName, clipInfo.data.clip, clipChapter, cancellationToken); outputFileInfo.Refresh(); - if (!outputFileInfo.Exists) + if (!outputFileInfo.Exists || outputFileInfo.Length == 0) { File.Move(tempFile, outputFileInfo.FullName); _progress.LogError("Unable to serialize metadata. The download has been completed without custom metadata."); From fceddcdb5414cef25a0bdba1903dad987ed0e0c2 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 29 Jun 2024 13:15:55 -0400 Subject: [PATCH 3/5] Remove resolved TODO --- TwitchDownloaderCore/VideoDownloader.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 7cf6efb2..2c97962d 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -357,7 +357,6 @@ private async Task RunFfmpegVideoCopy(string tempFolder, FileInfo outputFil outputFile.FullName }; - // TODO: Make this optional - "Safe" and "Exact" trimming methods if (endDuration > 0) { args.Insert(0, "-t"); From 3e468b5cb584a53c2fcd30cbd4029fd9cbdfdc7e Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 29 Jun 2024 13:19:46 -0400 Subject: [PATCH 4/5] Log clean up failures --- TwitchDownloaderCore/ChatDownloader.cs | 5 ++++- TwitchDownloaderCore/ChatRenderer.cs | 10 ++++++++-- TwitchDownloaderCore/ChatUpdater.cs | 5 ++++- TwitchDownloaderCore/ClipDownloader.cs | 5 ++++- TwitchDownloaderCore/TsMerger.cs | 8 ++++++-- TwitchDownloaderCore/VideoDownloader.cs | 5 ++++- 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs index d560654a..02d6b8d2 100644 --- a/TwitchDownloaderCore/ChatDownloader.cs +++ b/TwitchDownloaderCore/ChatDownloader.cs @@ -273,7 +273,10 @@ public async Task DownloadAsync(CancellationToken cancellationToken) await outputFs.DisposeAsync(); outputFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); + } } throw; diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index 5ffabb11..ddf1cd91 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -95,7 +95,10 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) await outputFs.DisposeAsync(); outputFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); + } } if (maskFileInfo is not null) @@ -108,7 +111,10 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) await maskFs.DisposeAsync(); maskFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty mask file: {e.Message}"); + } } } diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 8cf753e8..259f6c3e 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -54,7 +54,10 @@ public async Task UpdateAsync(CancellationToken cancellationToken) await outputFs.DisposeAsync(); outputFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); + } } throw; diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index 6fe2de22..23120414 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -53,7 +53,10 @@ public async Task DownloadAsync(CancellationToken cancellationToken) await outputFs.DisposeAsync(); outputFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); + } } throw; diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs index e913f1bc..ef3243d2 100644 --- a/TwitchDownloaderCore/TsMerger.cs +++ b/TwitchDownloaderCore/TsMerger.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -48,7 +49,10 @@ public async Task MergeAsync(CancellationToken cancellationToken) await outputFs.DisposeAsync(); outputFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); + } } throw; diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 2c97962d..d5d688ed 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -61,7 +61,10 @@ public async Task DownloadAsync(CancellationToken cancellationToken) await outputFs.DisposeAsync(); outputFileInfo.Delete(); } - catch { } + catch (Exception e) + { + _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); + } } throw; From 289d957eeb17c7bff92a0ca830cc40972797f289 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 29 Jun 2024 13:26:28 -0400 Subject: [PATCH 5/5] DRY --- TwitchDownloaderCore/ChatDownloader.cs | 14 +---------- TwitchDownloaderCore/ChatRenderer.cs | 32 ++----------------------- TwitchDownloaderCore/ChatUpdater.cs | 14 +---------- TwitchDownloaderCore/ClipDownloader.cs | 14 +---------- TwitchDownloaderCore/TsMerger.cs | 14 +---------- TwitchDownloaderCore/TwitchHelper.cs | 31 ++++++++++++++++++++++++ TwitchDownloaderCore/VideoDownloader.cs | 14 +---------- 7 files changed, 38 insertions(+), 95 deletions(-) diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs index 02d6b8d2..3a0c5e8b 100644 --- a/TwitchDownloaderCore/ChatDownloader.cs +++ b/TwitchDownloaderCore/ChatDownloader.cs @@ -265,19 +265,7 @@ public async Task DownloadAsync(CancellationToken cancellationToken) { await Task.Delay(100, CancellationToken.None); - outputFileInfo.Refresh(); - if (outputFileInfo.Exists && outputFileInfo.Length == 0) - { - try - { - await outputFs.DisposeAsync(); - outputFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); - } - } + TwitchHelper.CleanUpClaimedFile(outputFileInfo, outputFs, _progress); throw; } diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index ddf1cd91..e34dd419 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -87,36 +87,8 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) { await Task.Delay(100, CancellationToken.None); - outputFileInfo.Refresh(); - if (outputFileInfo.Exists && outputFileInfo.Length == 0) - { - try - { - await outputFs.DisposeAsync(); - outputFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); - } - } - - if (maskFileInfo is not null) - { - maskFileInfo.Refresh(); - if (maskFileInfo.Exists && maskFileInfo.Length == 0) - { - try - { - await maskFs.DisposeAsync(); - maskFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty mask file: {e.Message}"); - } - } - } + TwitchHelper.CleanUpClaimedFile(outputFileInfo, outputFs, _progress); + TwitchHelper.CleanUpClaimedFile(maskFileInfo, maskFs, _progress); throw; } diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 259f6c3e..cf3c7dc3 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -46,19 +46,7 @@ public async Task UpdateAsync(CancellationToken cancellationToken) { await Task.Delay(100, CancellationToken.None); - outputFileInfo.Refresh(); - if (outputFileInfo.Exists && outputFileInfo.Length == 0) - { - try - { - await outputFs.DisposeAsync(); - outputFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); - } - } + TwitchHelper.CleanUpClaimedFile(outputFileInfo, outputFs, _progress); throw; } diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index 23120414..12151782 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -45,19 +45,7 @@ public async Task DownloadAsync(CancellationToken cancellationToken) { await Task.Delay(100, CancellationToken.None); - outputFileInfo.Refresh(); - if (outputFileInfo.Exists && outputFileInfo.Length == 0) - { - try - { - await outputFs.DisposeAsync(); - outputFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); - } - } + TwitchHelper.CleanUpClaimedFile(outputFileInfo, outputFs, _progress); throw; } diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs index ef3243d2..c10030c2 100644 --- a/TwitchDownloaderCore/TsMerger.cs +++ b/TwitchDownloaderCore/TsMerger.cs @@ -41,19 +41,7 @@ public async Task MergeAsync(CancellationToken cancellationToken) { await Task.Delay(100, CancellationToken.None); - outputFileInfo.Refresh(); - if (outputFileInfo.Exists && outputFileInfo.Length == 0) - { - try - { - await outputFs.DisposeAsync(); - outputFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); - } - } + TwitchHelper.CleanUpClaimedFile(outputFileInfo, outputFs, _progress); throw; } diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs index 9c78ceaa..cd16d92d 100644 --- a/TwitchDownloaderCore/TwitchHelper.cs +++ b/TwitchDownloaderCore/TwitchHelper.cs @@ -1,6 +1,7 @@ using SkiaSharp; using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Compression; using System.Linq; @@ -899,6 +900,36 @@ public static FileInfo ClaimFile(string path, Func fileAlrea return fileInfo; } + public static void CleanUpClaimedFile([AllowNull] FileInfo fileInfo, [AllowNull] FileStream fileStream, ITaskLogger logger) + { + if (fileInfo is null) + { + return; + } + + fileInfo.Refresh(); + if (fileInfo.Exists && fileInfo.Length == 0) + { + try + { + fileStream?.Dispose(); + } + catch + { + // Ignored + } + + try + { + fileInfo.Delete(); + } + catch (Exception e) + { + logger.LogWarning($"Failed to clean up {fileInfo.FullName}: {e.Message}"); + } + } + } + public static DirectoryInfo CreateDirectory(string path) { DirectoryInfo directoryInfo = Directory.CreateDirectory(path); diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index d5d688ed..a026a8cb 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -53,19 +53,7 @@ public async Task DownloadAsync(CancellationToken cancellationToken) { await Task.Delay(100, CancellationToken.None); - outputFileInfo.Refresh(); - if (outputFileInfo.Exists && outputFileInfo.Length == 0) - { - try - { - await outputFs.DisposeAsync(); - outputFileInfo.Delete(); - } - catch (Exception e) - { - _progress.LogWarning($"Failed to clean up empty output file: {e.Message}"); - } - } + TwitchHelper.CleanUpClaimedFile(outputFileInfo, outputFs, _progress); throw; }