From 57aafe63999cef62ae8088d52a17cb5690c07e22 Mon Sep 17 00:00:00 2001 From: "CATERPILLAR\\Braxen" Date: Mon, 6 Nov 2023 14:31:40 +0100 Subject: [PATCH] fix vod splitting function related to #472 --- client-vue/src/components/vod/ExportModal.vue | 2 +- common/Replacements.ts | 10 +- common/ReplacementsConsts.ts | 15 +- common/ServerConfig.ts | 27 ++- server/src/Controllers/Vod.ts | 1 + .../src/Core/Providers/Base/BaseAutomator.ts | 14 ++ server/src/Core/Providers/Base/BaseVOD.ts | 4 + server/src/Core/Providers/Twitch/TwitchVOD.ts | 180 +++++++++++++++--- server/src/Helpers/Execute.ts | 25 ++- server/tests/automator.test.ts | 2 +- 10 files changed, 239 insertions(+), 41 deletions(-) diff --git a/client-vue/src/components/vod/ExportModal.vue b/client-vue/src/components/vod/ExportModal.vue index 2ca25517..5aad09c7 100644 --- a/client-vue/src/components/vod/ExportModal.vue +++ b/client-vue/src/components/vod/ExportModal.vue @@ -248,7 +248,7 @@ const { t } = useI18n(); const exportVodSettings = ref({ // exporter: "file", - title_template: "[{login}] {title} ({date})", + title_template: "[{internalName}] {title} ({date})", directory: "", host: "", username: "", diff --git a/common/Replacements.ts b/common/Replacements.ts index 417e2cfd..08e51300 100644 --- a/common/Replacements.ts +++ b/common/Replacements.ts @@ -1,5 +1,5 @@ export interface VodBasenameTemplate extends Record { - login: string; + // login: string; internalName: string; displayName: string; date: string; @@ -17,6 +17,14 @@ export interface VodBasenameTemplate extends Record { absolute_episode: string; title: string; game_name: string; + game_id: string; +} + +export interface VodBasenameWithChapterTemplate extends VodBasenameTemplate { + chapter_number: string; + chapter_title: string; + chapter_game_name: string; + chapter_game_id: string; } export interface ExporterFilenameTemplate extends Record { diff --git a/common/ReplacementsConsts.ts b/common/ReplacementsConsts.ts index c5bd8c88..799d6c27 100644 --- a/common/ReplacementsConsts.ts +++ b/common/ReplacementsConsts.ts @@ -1,7 +1,7 @@ -import type { ClipBasenameTemplate, ExporterFilenameTemplate, TemplateField, VodBasenameTemplate } from "./Replacements"; +import type { ClipBasenameTemplate, ExporterFilenameTemplate, TemplateField, VodBasenameTemplate, VodBasenameWithChapterTemplate } from "./Replacements"; export const VodBasenameFields: Record = { - login: { display: "MooseStreamer", deprecated: true }, + // login: { display: "MooseStreamer", deprecated: true }, internalName: { display: "moosestreamer" }, displayName: { display: "MooseStreamer", }, date: { display: "2022-12-31T12_05_04Z" }, @@ -15,11 +15,22 @@ export const VodBasenameFields: Record id: { display: "123456789" }, season: { display: "202212" }, absolute_season: { display: "5" }, + absolute_episode: { display: "3" }, episode: { display: "3" }, title: { display: "Moose crosses river HOT NEW CONTENT COME LOOK" }, game_name: { display: "Moose Simulator 2022" }, + game_id: { display: "123456789" }, }; +export const VodBasenameChapterFields: Record = { + ...VodBasenameFields, // extends VodBasenameTemplate + chapter_number: { display: "1" }, + chapter_title: { display: "Moose crosses river" }, + chapter_game_name: { display: "Moose Simulator 2022" }, + chapter_game_id: { display: "123456789" }, +}; + + export const ClipBasenameFields: Record = { id: { display: "MinimalMooseOtterCatcher1234" }, quality: { display: "720p" }, diff --git a/common/ServerConfig.ts b/common/ServerConfig.ts index 70e1f7ef..1531bb03 100644 --- a/common/ServerConfig.ts +++ b/common/ServerConfig.ts @@ -2,6 +2,7 @@ import type { SettingField } from "./Config"; import { ClipBasenameFields, ExporterFilenameFields, + VodBasenameChapterFields, VodBasenameFields, } from "./ReplacementsConsts"; import { YouTubeCategories } from "./YouTube"; @@ -667,7 +668,7 @@ export const settingsFields: Record< group: "Video", text: "Vod filename", type: "template", - default: "{login}_{date}_{id}", + default: "{internalName}_{date}_{id}", help: "Vod filename.", replacements: VodBasenameFields, context: "{template}.json, {template}.mp4", @@ -677,10 +678,30 @@ export const settingsFields: Record< group: "Video", text: "Vod folder name", type: "template", - default: "{login}_{date}_{id}", + default: "{internalName}_{date}_{id}", help: "Vod folder filename.", replacements: VodBasenameFields, - context: "/vods/{login}/{template}/", + context: "/vods/{internalName}/{template}/", + }, + + "template.vodsplit.folder": { + group: "Video", + text: "Vodsplit folder name", + type: "template", + default: "{internalName}_{date}_{id}", + help: "Vodsplit folder filename. If same as Vod folder name, it will be placed in the same folder.", + replacements: VodBasenameChapterFields, + context: "/vods/{internalName}/{template}/", + }, + + "template.vodsplit.filename": { + group: "Video", + text: "Vodsplit filename", + type: "template", + default: "{internalName}_{date}_{id}-{chapter_number}._{chapter_title}_({game_name})", + help: "Vodsplit filename.", + replacements: VodBasenameChapterFields, + context: "{template}.mp4", }, min_chapter_duration: { diff --git a/server/src/Controllers/Vod.ts b/server/src/Controllers/Vod.ts index 3e750eb1..a8c38ce9 100644 --- a/server/src/Controllers/Vod.ts +++ b/server/src/Controllers/Vod.ts @@ -940,6 +940,7 @@ export async function RenameVod( : "", title: vod.stream_title || "", game_name: vod.game_name || "", + game_id: vod.game_id || "", }; const basename = sanitize( diff --git a/server/src/Core/Providers/Base/BaseAutomator.ts b/server/src/Core/Providers/Base/BaseAutomator.ts index bc4bc7d4..8322abdd 100644 --- a/server/src/Core/Providers/Base/BaseAutomator.ts +++ b/server/src/Core/Providers/Base/BaseAutomator.ts @@ -125,6 +125,7 @@ export class BaseAutomator { : "", title: this.getTitle(), game_name: this.getGameName(), + game_id: this.getGameID(), }; return sanitize( @@ -175,6 +176,7 @@ export class BaseAutomator { : "", title: this.getTitle(), game_name: this.getGameName(), + game_id: this.getGameID(), }; return sanitize( @@ -244,6 +246,18 @@ export class BaseAutomator { return ""; } + public getGameID(): string { + if (KeyValue.getInstance().has(`${this.getLogin()}.chapterdata`)) { + const data = KeyValue.getInstance().getObject( + `${this.getLogin()}.chapterdata` + ); + if (data && data.game_id) { + return data.game_id || ""; + } + } + return ""; + } + public streamURL(): string { // return `twitch.tv/${this.broadcaster_user_login}`; if (!this.channel) { diff --git a/server/src/Core/Providers/Base/BaseVOD.ts b/server/src/Core/Providers/Base/BaseVOD.ts index f38da8db..f7ccbdf3 100644 --- a/server/src/Core/Providers/Base/BaseVOD.ts +++ b/server/src/Core/Providers/Base/BaseVOD.ts @@ -2703,6 +2703,10 @@ export class BaseVOD { return ""; // base vod does not have game_name } + public get game_id(): string { + return ""; // base vod does not have game_id + } + public calculateBookmarks(): boolean { if (!this.bookmarks || this.bookmarks.length == 0) return false; if (!this.started_at) return false; diff --git a/server/src/Core/Providers/Twitch/TwitchVOD.ts b/server/src/Core/Providers/Twitch/TwitchVOD.ts index 4f6bdd49..627dc3ef 100644 --- a/server/src/Core/Providers/Twitch/TwitchVOD.ts +++ b/server/src/Core/Providers/Twitch/TwitchVOD.ts @@ -1,5 +1,11 @@ import { progressOutput } from "@/Helpers/Console"; -import { execAdvanced, execSimple, startJob } from "@/Helpers/Execute"; +import { + execAdvanced, + execSimple, + isExecReturn, + startJob, +} from "@/Helpers/Execute"; +import { validateAbsolutePath, validateFilename } from "@/Helpers/Filesystem"; import { formatDuration, formatSubtitleDuration } from "@/Helpers/Format"; import { xClearInterval, xInterval, xTimeout } from "@/Helpers/Timeout"; import { isTwitchVOD } from "@/Helpers/Types"; @@ -10,8 +16,10 @@ import type { VideoQuality } from "@common/Config"; import type { Providers } from "@common/Defs"; import { JobStatus, MuteStatus } from "@common/Defs"; import type { AudioStream, FFProbe, VideoStream } from "@common/FFProbe"; +import { formatString } from "@common/Format"; import type { VideoMetadata } from "@common/MediaInfo"; import type { ProxyVideo } from "@common/Proxies/Video"; +import type { VodBasenameWithChapterTemplate } from "@common/Replacements"; import type { Clip, ClipsResponse } from "@common/TwitchAPI/Clips"; import type { GqlVideoChapterResponse, @@ -30,6 +38,7 @@ import { format, parseJSON } from "date-fns"; import { encode as htmlentities } from "html-entities"; import fs from "node:fs"; import path from "node:path"; +import sanitize from "sanitize-filename"; import { TwitchHelper } from "../../../Providers/Twitch"; import type { TwitchVODChapterJSON, @@ -365,6 +374,11 @@ export class TwitchVOD extends BaseVOD { return this.chapters[0].game_name; } + public get game_id(): string { + if (!this.chapters || this.chapters.length == 0) return ""; + return this.chapters[0].game_id?.toString() || ""; + } + /** * Returns an array of filenames associated with this Twitch VOD, including the JSON metadata file, * chat logs, video files, and other related files. If the VOD has been segmented, the filenames of @@ -554,6 +568,14 @@ export class TwitchVOD extends BaseVOD { return false; } + if (!this.created_at) { + throw new Error( + "TwitchVOD.splitSegmentVideoByChapters: created_at is not set" + ); + } + + const video_input = path.join(this.directory, this.segments_raw[0]); // TODO: burned video etc + const bin = Helper.path_ffmpeg(); if (!bin) { @@ -580,21 +602,32 @@ export class TwitchVOD extends BaseVOD { return false; } - if (!chapter.offset || !chapter.duration) { + if (chapter.offset == undefined) { + log( + LOGLEVEL.ERROR, + "vod.splitSegmentByChapters", + `Chapter ${chapter.title} has no offset for ${this.basename}` + ); + throw new Error( + `Chapter ${chapter.title} has no offset for ${this.basename}` + ); + } + + if (chapter.duration == undefined) { log( LOGLEVEL.ERROR, "vod.splitSegmentByChapters", - `Chapter ${chapter.title} has no offset or duration for ${this.basename}` + `Chapter ${chapter.title} has no duration for ${this.basename}` ); throw new Error( - `Chapter ${chapter.title} has no offset or duration for ${this.basename}` + `Chapter ${chapter.title} has no duration for ${this.basename}` ); } if ( chapter.offset < 0 || chapter.duration < 0 || - chapter.offset + chapter.duration > this.duration + chapter.offset > this.duration ) { log( LOGLEVEL.ERROR, @@ -606,37 +639,128 @@ export class TwitchVOD extends BaseVOD { ); } + if ( + chapter.offset + chapter.duration > this.duration && + chapter_index != this.chapters.length + ) { + log( + LOGLEVEL.ERROR, + "vod.splitSegmentByChapters", + `Chapter ${chapter.title} has invalid duration for ${this.basename}` + ); + throw new Error( + `Chapter ${chapter.title} has invalid duration for ${this.basename}` + ); + } + const chapter_start = chapter.offset || 0; const chapter_end = chapter.offset + chapter.duration; - const chapter_title = - `${chapter_index}. ${chapter.title} (${chapter.game_name})`.replace( - /[^a-z0-9]/gi, - "_" + let basepath = this.directory; + + const channel_basepath = this.getChannel().getFolder(); + + const chapter_template_variables: VodBasenameWithChapterTemplate = { + // login: this.getChannel().login, + internalName: this.getChannel().internalName, + displayName: this.getChannel().displayName, + date: format(this.created_at, "yyyy-MM-dd"), // TODO: check format + year: format(this.created_at, "yyyy"), + year_short: format(this.created_at, "yy"), + month: format(this.created_at, "MM"), + day: format(this.created_at, "dd"), + hour: format(this.created_at, "HH"), + minute: format(this.created_at, "mm"), + second: format(this.created_at, "ss"), + id: this.twitch_vod_id || "", + season: this.stream_season || "", + absolute_season: this.stream_absolute_season?.toString() || "", + episode: this.stream_number?.toString() || "", + absolute_episode: this.stream_absolute_number?.toString() || "", + title: this.twitch_vod_title || "", + game_name: this.game_name, + game_id: this.current_game?.id || "", + chapter_number: chapter_index.toString(), + chapter_title: chapter.title, + chapter_game_id: chapter.game_id?.toString() || "", + chapter_game_name: chapter.game_name, + }; + + if (Config.getInstance().cfg("vod_folders")) { + basepath = path.join( + channel_basepath, + formatString( + Config.getInstance().cfg( + "template.vodsplit.folder" + ), + chapter_template_variables + ) ); - const chapter_filename = `${this.basename}.${chapter_title}.mp4`; + if (!validateAbsolutePath(basepath)) { + throw new Error( + `Invalid basepath for ${this.basename}: ${basepath}` + ); + } + } - const chapter_path = path.join(this.directory, chapter_filename); + if (!fs.existsSync(basepath)) { + fs.mkdirSync(basepath, { recursive: true }); + } - const job = await execAdvanced( - bin, - [ - // "-y", - "-i", - this.realpath( - path.join(this.directory, this.segments_raw[0]) + let chapter_filename = + formatString( + Config.getInstance().cfg( + "template.vodsplit.filename" ), - "-ss", - formatSubtitleDuration(chapter_start), - "-to", - formatSubtitleDuration(chapter_end), - "-c", - "copy", - chapter_path, - ], - `chapter_split_${this.basename}_${chapter_index}` - ); + chapter_template_variables + ) + + "." + + Config.getInstance().cfg("vod_container", "mp4"); + + // quickly replace invalid characters + chapter_filename = sanitize(chapter_filename); + + if (!validateFilename(chapter_filename)) { + throw new Error( + `Invalid filename for ${this.basename}: ${chapter_filename}` + ); + } + + const chapter_path = path.join(basepath, chapter_filename); + + const args: string[] = []; + + // args.push(// "-y",); + args.push( + "-i", + // this.realpath(path.join(this.directory, this.segments_raw[0])) + this.realpath(video_input) + ); + args.push("-ss", formatSubtitleDuration(chapter_start)); + // final chapter doesn't need to end at a specific time + if (chapter_index == this.chapters.length) { + args.push("-to", formatSubtitleDuration(this.duration)); + } + args.push("-c", "copy"); + args.push(chapter_path); + + let job; + + try { + job = await execAdvanced( + bin, + args, + `chapter_split_${this.basename}_${chapter_index}` + ); + } catch (error) { + if (isExecReturn(error)) { + throw new Error( + `Chapter ${chapter.title} failed to split for ${this.basename}: ${error.stderr}` + ); + } + throw error; + } if ( !fs.existsSync(chapter_path) || diff --git a/server/src/Helpers/Execute.ts b/server/src/Helpers/Execute.ts index 9cf5a2e3..2091391d 100644 --- a/server/src/Helpers/Execute.ts +++ b/server/src/Helpers/Execute.ts @@ -1,11 +1,11 @@ +import { Config } from "@/Core/Config"; +import { Job } from "@/Core/Job"; +import { LOGLEVEL, log } from "@/Core/Log"; +import chalk from "chalk"; import type { ChildProcessWithoutNullStreams } from "node:child_process"; import { spawn } from "node:child_process"; -import type { ExecReturn } from "../Providers/Twitch"; -import { LOGLEVEL, log } from "@/Core/Log"; import type { Stream } from "node:stream"; -import { Config } from "@/Core/Config"; -import chalk from "chalk"; -import { Job } from "@/Core/Job"; +import type { ExecReturn } from "../Providers/Twitch"; interface RunningProcess { internal_pid: number; @@ -106,6 +106,21 @@ export function execSimple( }); } +export function isExecReturn( + execReturn: ExecReturn | unknown +): execReturn is ExecReturn { + return ( + typeof execReturn === "object" && + execReturn !== null && + "code" in execReturn && + "stdout" in execReturn && + "stderr" in execReturn && + "bin" in execReturn && + "args" in execReturn && + "what" in execReturn + ); +} + /** * Execute a command, make a job, and when it's done, return the output * diff --git a/server/tests/automator.test.ts b/server/tests/automator.test.ts index 79088ba3..46b7d050 100644 --- a/server/tests/automator.test.ts +++ b/server/tests/automator.test.ts @@ -38,7 +38,7 @@ describe("Automator", () => { Config.getInstance().setConfig( "filename_vod", - "{login}_{year}_{month}_{day}" + "{internalName}_{year}_{month}_{day}" ); expect(TA.vodBasenameTemplate()).toBe("test_2022_09_02");