From fbcd10f80ba0117405e1329fd96c445c076e9f67 Mon Sep 17 00:00:00 2001 From: "CATERPILLAR\\Braxen" Date: Wed, 15 Nov 2023 11:56:28 +0100 Subject: [PATCH 1/8] Update config.test.ts --- server/tests/config.test.ts | 60 ++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/server/tests/config.test.ts b/server/tests/config.test.ts index 5de25a38..dc94ab47 100644 --- a/server/tests/config.test.ts +++ b/server/tests/config.test.ts @@ -1,18 +1,32 @@ import { Config } from "../src/Core/Config"; -import { log, LOGLEVEL } from "../src/Core/Log"; import "./environment"; describe("Config", () => { - it("external url validation", () => { - expect(() => Config.validateExternalURLRules("http://example.com")).toThrow(); - expect(() => Config.validateExternalURLRules("http://example.com:1234")).toThrow(); - expect(() => Config.validateExternalURLRules("http://example.com:80")).toThrow(); - expect(Config.validateExternalURLRules("https://example.com:443")).toBe(true); - expect(Config.validateExternalURLRules("https://example.com")).toBe(true); - expect(Config.validateExternalURLRules("https://sub.example.com")).toBe(true); - expect(() => Config.validateExternalURLRules("https://sub.example.com/folder/")).toThrow(); - expect(Config.validateExternalURLRules("https://sub.example.com/folder")).toBe(true); + expect(() => + Config.validateExternalURLRules("http://example.com") + ).toThrow(); + expect(() => + Config.validateExternalURLRules("http://example.com:1234") + ).toThrow(); + expect(() => + Config.validateExternalURLRules("http://example.com:80") + ).toThrow(); + expect(Config.validateExternalURLRules("https://example.com:443")).toBe( + true + ); + expect(Config.validateExternalURLRules("https://example.com")).toBe( + true + ); + expect(Config.validateExternalURLRules("https://sub.example.com")).toBe( + true + ); + expect(() => + Config.validateExternalURLRules("https://sub.example.com/folder/") + ).toThrow(); + expect( + Config.validateExternalURLRules("https://sub.example.com/folder") + ).toBe(true); }); it("config value set", () => { @@ -40,7 +54,9 @@ describe("Config", () => { it("config value set with default", () => { const config = Config.getCleanInstance(); config.config = {}; - expect(config.cfg("app_url", "https://example.com")).toBe("https://example.com"); + expect(config.cfg("app_url", "https://example.com")).toBe( + "https://example.com" + ); expect(config.cfg("app_url", "")).toBe(""); expect(config.cfg("app_url")).toBeUndefined(); @@ -81,15 +97,15 @@ describe("Config", () => { it("hasValue", () => { const config = Config.getCleanInstance(); - + config.config = {}; - + expect(config.hasValue("password")).toBe(false); - + // config value is set config.setConfig("password", "test"); expect(config.hasValue("password")).toBe(true); - + // config value is empty string config.setConfig("password", ""); expect(config.hasValue("password")).toBe(false); @@ -126,7 +142,16 @@ describe("Config", () => { // env value is undefined process.env.TCD_PASSWORD = ""; expect(config.hasValue("password")).toBe(false); - + }); + + it("choice values", () => { + const config = Config.getCleanInstance(); + config.config = {}; + expect(() => + config.setConfig("date_format", "dd-mm-yyyy") + ).toThrowError(); + config.setConfig("date_format", "dd-MM-yyyy"); + expect(config.cfg("date_format")).toBe("dd-MM-yyyy"); }); it("setting exists", () => { @@ -135,5 +160,4 @@ describe("Config", () => { expect(Config.getSettingField("app_url")).toHaveProperty("text"); expect(Config.getSettingField("app_url1")).toBeUndefined(); }); - -}); \ No newline at end of file +}); From b045400061439ee12aebfa4e439a2f3c9e5f1592 Mon Sep 17 00:00:00 2001 From: "CATERPILLAR\\Braxen" Date: Wed, 15 Nov 2023 13:30:48 +0100 Subject: [PATCH 2/8] fix more config tests --- server/src/Core/Config.ts | 18 ++++++++++-- server/src/Helpers/Types.ts | 4 +++ server/tests/config.test.ts | 56 ++++++++++++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/server/src/Core/Config.ts b/server/src/Core/Config.ts index 1fc4dbc2..ce9e2af3 100644 --- a/server/src/Core/Config.ts +++ b/server/src/Core/Config.ts @@ -1,6 +1,7 @@ import { debugLog } from "@/Helpers/Console"; import { GetRunningProcesses, execSimple } from "@/Helpers/Execute"; import { is_docker } from "@/Helpers/System"; +import { isNumber } from "@/Helpers/Types"; import type { SettingField } from "@common/Config"; import { settingsFields } from "@common/ServerConfig"; import type { AxiosResponse } from "axios"; @@ -331,15 +332,28 @@ export class Config { setting.stripslash && typeof newValue === "string" ) { - newValue = newValue.replace(/\/$/, ""); + newValue = newValue.replace(/\/$/, "").replace(/\\$/, ""); } if (setting.type === "number" && typeof newValue === "string") { + if (!isNumber(newValue)) { + throw new Error( + `Invalid value for setting '${key}': ${newValue}` + ); + } newValue = parseInt(newValue); } if (setting.type === "boolean" && typeof newValue === "string") { - newValue = newValue === "true" || newValue === "1"; + if (newValue === "true" || newValue === "1") { + newValue = true; + } else if (newValue === "false" || newValue === "0") { + newValue = false; + } else { + throw new Error( + `Invalid value for setting '${key}': ${newValue}` + ); + } } if (setting.type == "array") { diff --git a/server/src/Helpers/Types.ts b/server/src/Helpers/Types.ts index 420df10d..571dea84 100644 --- a/server/src/Helpers/Types.ts +++ b/server/src/Helpers/Types.ts @@ -52,3 +52,7 @@ export function isKickChannel( export function isError(compareData: unknown): compareData is Error { return compareData instanceof Error; } + +export function isNumber(compareData: string): boolean { + return !isNaN(Number(compareData)); +} diff --git a/server/tests/config.test.ts b/server/tests/config.test.ts index dc94ab47..b86d3e50 100644 --- a/server/tests/config.test.ts +++ b/server/tests/config.test.ts @@ -147,11 +147,65 @@ describe("Config", () => { it("choice values", () => { const config = Config.getCleanInstance(); config.config = {}; + + // object + config.setConfig("date_format", "dd-MM-yyyy"); + expect(config.cfg("date_format")).toBe("dd-MM-yyyy"); expect(() => config.setConfig("date_format", "dd-mm-yyyy") ).toThrowError(); - config.setConfig("date_format", "dd-MM-yyyy"); expect(config.cfg("date_format")).toBe("dd-MM-yyyy"); + + // array + config.setConfig("vod_container", "mkv"); + expect(config.cfg("vod_container")).toBe("mkv"); + expect(() => config.setConfig("vod_container", "asdf")).toThrowError(); + expect(config.cfg("vod_container")).toBe("mkv"); + }); + + it("number values", () => { + const config = Config.getCleanInstance(); + config.config = {}; + + // number + config.setConfig("server_port", 1234); + expect(config.cfg("server_port")).toBe(1234); + expect(() => config.setConfig("server_port", "asdf")).toThrowError(); + expect(config.cfg("server_port")).toBe(1234); + + // cast to number + config.setConfig("server_port", "1234"); + expect(config.cfg("server_port")).toBe(1234); + }); + + it("boolean values", () => { + const config = Config.getCleanInstance(); + config.config = {}; + + // boolean + config.setConfig("trust_proxy", true); + expect(config.cfg("trust_proxy")).toBe(true); + expect(() => config.setConfig("trust_proxy", "asdf")).toThrowError(); + expect(config.cfg("trust_proxy")).toBe(true); + + // cast to boolean + config.setConfig("trust_proxy", "1"); + expect(config.cfg("trust_proxy")).toBe(true); + config.setConfig("trust_proxy", "0"); + expect(config.cfg("trust_proxy")).toBe(false); + }); + + it("stripslashes", () => { + const config = Config.getCleanInstance(); + config.config = {}; + + // windows + config.setConfig("bin_dir", "C:\\Program Files\\ffmpeg\\bin\\"); + expect(config.cfg("bin_dir")).toBe("C:\\Program Files\\ffmpeg\\bin"); + + // linux + config.setConfig("bin_dir", "/usr/bin/"); + expect(config.cfg("bin_dir")).toBe("/usr/bin"); }); it("setting exists", () => { From 541b698a40a98647f3f9411dbbd8fe38f3a30439 Mon Sep 17 00:00:00 2001 From: "CATERPILLAR\\Braxen" Date: Wed, 15 Nov 2023 14:07:41 +0100 Subject: [PATCH 3/8] actually use date format, some static color changes --- client-vue/src/assets/_variables.scss | 2 + client-vue/src/assets/style.scss | 17 +++++++++ .../src/components/reusables/DBoolean.vue | 35 ++++++++++++++++++ .../components/streamer/ClipDownloadModal.vue | 5 ++- .../streamer/VideoDownloadModal.vue | 5 ++- .../src/components/vod/VodItemChapters.vue | 5 ++- .../src/components/vod/VodItemStatus.vue | 2 +- .../src/components/vod/VodItemVideoInfo.vue | 37 +++++++++++-------- client-vue/src/main.ts | 2 + common/ServerConfig.ts | 18 ++++++++- server/src/Providers/Twitch.ts | 7 ---- server/tests/config.test.ts | 8 ++-- 12 files changed, 111 insertions(+), 32 deletions(-) create mode 100644 client-vue/src/components/reusables/DBoolean.vue diff --git a/client-vue/src/assets/_variables.scss b/client-vue/src/assets/_variables.scss index b8020e87..5898acbc 100644 --- a/client-vue/src/assets/_variables.scss +++ b/client-vue/src/assets/_variables.scss @@ -16,6 +16,8 @@ $boxart-height: 190px; --body-font: Roboto, Helvetica, Arial; --text-darker: #3d3d3d; + --color-danger: #f14668; + --header-color: #111; --header-font: "Montserrat", sans-serif; diff --git a/client-vue/src/assets/style.scss b/client-vue/src/assets/style.scss index 3868a6be..db72ab53 100644 --- a/client-vue/src/assets/style.scss +++ b/client-vue/src/assets/style.scss @@ -134,6 +134,7 @@ svg[data-icon] { } */ +/** @deprecated */ .text-is-error { color: #f00; &.flashing { @@ -141,6 +142,22 @@ svg[data-icon] { } } +.has-text-danger { + color: var(--color-danger); +} + +.has-text-warning { + color: #f80; +} + +.has-text-success { + color: #1db51d; +} + +.has-bg-danger { + background-color: var(--color-danger); +} + .is-warning { color: #f80; } diff --git a/client-vue/src/components/reusables/DBoolean.vue b/client-vue/src/components/reusables/DBoolean.vue new file mode 100644 index 00000000..97ed73ef --- /dev/null +++ b/client-vue/src/components/reusables/DBoolean.vue @@ -0,0 +1,35 @@ + + + + + \ No newline at end of file diff --git a/client-vue/src/components/streamer/ClipDownloadModal.vue b/client-vue/src/components/streamer/ClipDownloadModal.vue index 5f084c66..cd78ec21 100644 --- a/client-vue/src/components/streamer/ClipDownloadModal.vue +++ b/client-vue/src/components/streamer/ClipDownloadModal.vue @@ -20,7 +20,7 @@ {{ formatNumberShort(clip.view_count, 0) }} views • {{ formatDistanceToNow(new Date(clip.created_at)) }} ago - ({{ format(new Date(clip.created_at), "yyyy-MM-dd HH:mm:ss") }}) + ({{ format(new Date(clip.created_at), `${store.cfg('locale.date-format')} ${store.cfg('locale.time-format')}`) }})

by {{ clip.creator_name }}, playing {{ clip.game_id }} @@ -57,6 +57,7 @@ import { useI18n } from "vue-i18n"; import type { ApiResponse } from "@common/Api/Api"; import type { ChannelTypes } from "@/twitchautomator"; import { formatDistanceToNow, format } from "date-fns"; +import { useStore } from "@/store"; library.add(faSpinner, faTwitch); const props = defineProps<{ @@ -68,6 +69,8 @@ const { t } = useI18n(); const onlineClips = ref([]); const loading = ref(false); +const store = useStore(); + // clips async function fetchTwitchClips() { if (!props.streamer) return; diff --git a/client-vue/src/components/streamer/VideoDownloadModal.vue b/client-vue/src/components/streamer/VideoDownloadModal.vue index 03e781da..3f6cb975 100644 --- a/client-vue/src/components/streamer/VideoDownloadModal.vue +++ b/client-vue/src/components/streamer/VideoDownloadModal.vue @@ -29,7 +29,7 @@ {{ formatDistanceToNow(new Date(vod.created_at)) }} ago - ({{ format(new Date(vod.created_at), "yyyy-MM-dd HH:mm:ss") }}) + ({{ format(new Date(vod.created_at), `${store.cfg('locale.date-format')} ${store.cfg('locale.time-format')}`) }})

{{ vod.description }}

    @@ -68,6 +68,7 @@ import { useI18n } from "vue-i18n"; import type { ApiResponse } from "@common/Api/Api"; import type { ChannelTypes } from "@/twitchautomator"; import { formatDistanceToNow, format } from "date-fns"; +import { useStore } from "@/store"; library.add(faSpinner); const props = defineProps<{ @@ -80,6 +81,8 @@ const onlineVods = ref([]); const loading = ref(false); const quality = ref("best"); +const store = useStore(); + // videos async function fetchTwitchVods() { if (!props.streamer) return; diff --git a/client-vue/src/components/vod/VodItemChapters.vue b/client-vue/src/components/vod/VodItemChapters.vue index cde6c7e5..e6e5b38c 100644 --- a/client-vue/src/components/vod/VodItemChapters.vue +++ b/client-vue/src/components/vod/VodItemChapters.vue @@ -176,7 +176,10 @@ -
    No chapters found
    +
    + + No chapters found +
    diff --git a/client-vue/src/components/vod/VodItemStatus.vue b/client-vue/src/components/vod/VodItemStatus.vue index 4dfc7316..bd800d91 100644 --- a/client-vue/src/components/vod/VodItemStatus.vue +++ b/client-vue/src/components/vod/VodItemStatus.vue @@ -86,7 +86,7 @@
  • {{ t("vod.video-info.downloaded") }}: - {{ vod.is_vod_downloaded ? t("boolean.yes") : t("boolean.no") }} + +
@@ -366,6 +371,7 @@ function youtubePlaylistLink(playlist_id: string): string { diff --git a/client-vue/src/main.ts b/client-vue/src/main.ts index 48515f3a..6fbfb7c6 100644 --- a/client-vue/src/main.ts +++ b/client-vue/src/main.ts @@ -15,6 +15,7 @@ import i18n from "./plugins/i18n"; import LoadingBox from "@/components/reusables/LoadingBox.vue"; import DButton from "@/components/reusables/DButton.vue"; import DSelect from "@/components/reusables/DSelect.vue"; +import DBoolean from "@/components/reusables/DBoolean.vue"; if (import.meta.env.BASE_URL !== undefined) { axios.defaults.baseURL = import.meta.env.BASE_URL; @@ -30,6 +31,7 @@ createApp(App) .component("LoadingBox", LoadingBox) .component("DButton", DButton) .component("DSelect", DSelect) + .component("DBoolean", DBoolean) // .mixin(titleMixin) // .mixin(helpers) .mount("#app"); diff --git a/common/ServerConfig.ts b/common/ServerConfig.ts index 7e63cdc0..d62bbe6b 100644 --- a/common/ServerConfig.ts +++ b/common/ServerConfig.ts @@ -110,8 +110,8 @@ export const settingsFields: Record = { help: "Enable this if your server is not exposed to the internet, aka no EventSub support.", }, - date_format: { - group: "Basic", + "locale.date-format": { + group: "Locale", text: "Date format", type: "object", default: "yyyy-MM-dd", @@ -124,6 +124,20 @@ export const settingsFields: Record = { "dd-MM yyyy": "31-12 2021", "MM-dd yyyy": "12-31 2021", }, + migrate: "date_format", + }, + + "locale.time-format": { + group: "Locale", + text: "Time format", + type: "object", + default: "HH:mm:ss", + choices: { + "HH:mm:ss": "23:59:59 (default)", + "HH:mm": "23:59", + "hh:mm:ss a": "11:59:59 PM", + "hh:mm a": "11:59 PM", + }, }, motd: { diff --git a/server/src/Providers/Twitch.ts b/server/src/Providers/Twitch.ts index 57b9ab2c..9a985bf5 100644 --- a/server/src/Providers/Twitch.ts +++ b/server/src/Providers/Twitch.ts @@ -86,13 +86,6 @@ export class TwitchHelper { "oauth_user_refresh.json" ); - /** @deprecated */ - static readonly accessTokenExpire = 60 * 60 * 24 * 60 * 1000; // 60 days - /** @deprecated */ - static readonly accessTokenRefresh = 60 * 60 * 24 * 30 * 1000; // 30 days - - /** @deprecated */ - static readonly PHP_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSSSS"; static readonly TWITCH_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; static readonly TWITCH_DATE_FORMAT_MS = "yyyy-MM-dd'T'HH:mm:ss'.'SSS'Z'"; diff --git a/server/tests/config.test.ts b/server/tests/config.test.ts index b86d3e50..6da58979 100644 --- a/server/tests/config.test.ts +++ b/server/tests/config.test.ts @@ -149,12 +149,12 @@ describe("Config", () => { config.config = {}; // object - config.setConfig("date_format", "dd-MM-yyyy"); - expect(config.cfg("date_format")).toBe("dd-MM-yyyy"); + config.setConfig("locale.date-format", "dd-MM-yyyy"); + expect(config.cfg("locale.date-format")).toBe("dd-MM-yyyy"); expect(() => - config.setConfig("date_format", "dd-mm-yyyy") + config.setConfig("locale.date-format", "dd-mm-yyyy") ).toThrowError(); - expect(config.cfg("date_format")).toBe("dd-MM-yyyy"); + expect(config.cfg("locale.date-format")).toBe("dd-MM-yyyy"); // array config.setConfig("vod_container", "mkv"); From d2e9b1df6ac0ac899f723122b39e1a60f340e07b Mon Sep 17 00:00:00 2001 From: Braxen Date: Sat, 18 Nov 2023 23:57:40 +0100 Subject: [PATCH 4/8] rework channel loading and tests --- server/src/Core/LiveStreamDVR.ts | 27 +- .../Core/Providers/Twitch/TwitchChannel.ts | 104 ++++---- server/tests/api.test.ts | 231 +++++++++++++++--- server/tests/environment.ts | 199 +++++++++++---- 4 files changed, 442 insertions(+), 119 deletions(-) diff --git a/server/src/Core/LiveStreamDVR.ts b/server/src/Core/LiveStreamDVR.ts index 6525590a..db5a1069 100644 --- a/server/src/Core/LiveStreamDVR.ts +++ b/server/src/Core/LiveStreamDVR.ts @@ -210,6 +210,25 @@ export class LiveStreamDVR { // TwitchHelper.refreshUserAccessToken(); } + public static migrateChannelConfig(channel: ChannelConfig): void { + if (channel.provider == "twitch") { + if (!channel.internalName && channel.login) { + channel.internalName = channel.login; + } + } else if (channel.provider == "youtube") { + if (!channel.internalName && channel.channel_id) { + channel.internalName = channel.channel_id; + } + if (!channel.internalId && channel.channel_id) { + channel.internalId = channel.channel_id; + } + } else if (channel.provider == "kick") { + if (!channel.internalName && channel.slug) { + channel.internalName = channel.slug; + } + } + } + /** * @test disable * @returns @@ -255,6 +274,8 @@ export class LiveStreamDVR { ); needsSave = true; } + + /* if (!channel.internalName || !channel.internalId) { if (channel.provider == "twitch") { channel.internalName = channel.login || ""; @@ -267,6 +288,8 @@ export class LiveStreamDVR { // throw new Error(`Channel ${channel.uuid} does not have an internalName`); } } + */ + LiveStreamDVR.migrateChannelConfig(channel); } this.channels_config = data; @@ -324,9 +347,7 @@ export class LiveStreamDVR { let ch: TwitchChannel; try { - ch = await TwitchChannel.loadFromLogin( - channel.internalName - ); + ch = await TwitchChannel.load(channel.uuid); } catch (th) { log( LOGLEVEL.FATAL, diff --git a/server/src/Core/Providers/Twitch/TwitchChannel.ts b/server/src/Core/Providers/Twitch/TwitchChannel.ts index 6d86d0dd..ab20fc95 100644 --- a/server/src/Core/Providers/Twitch/TwitchChannel.ts +++ b/server/src/Core/Providers/Twitch/TwitchChannel.ts @@ -1653,32 +1653,43 @@ export class TwitchChannel extends BaseChannel { // TODO: load by uuid? public static async loadAbstract( - channel_id: string + // channel_id: string + uuid: string ): Promise { - log( - LOGLEVEL.DEBUG, - "tw.channel.loadAbstract", - `Load channel ${channel_id}` - ); + log(LOGLEVEL.DEBUG, "tw.channel.loadAbstract", `Load channel ${uuid}`); const channel_memory = LiveStreamDVR.getInstance() .getChannels() .find( (channel): channel is TwitchChannel => - isTwitchChannel(channel) && - channel.internalId === channel_id + isTwitchChannel(channel) && channel.uuid === uuid ); if (channel_memory) { log( LOGLEVEL.WARNING, "tw.channel.loadAbstract", - `Channel ${channel_id} already loaded` + `Channel ${uuid} (${channel_memory.internalName}) already exists in memory, returning` ); return channel_memory; } + const channel_config = LiveStreamDVR.getInstance().channels_config.find( + (c) => c.provider == "twitch" && c.uuid === uuid + ); + + if (!channel_config) + throw new Error(`Could not find channel config for uuid ${uuid}`); + + const channel_id = + channel_config.internalId || + (await this.channelIdFromLogin(channel_config.internalName)); + + if (!channel_id) + throw new Error( + `Could not get channel id for login ${channel_config.internalName}` + ); + const channel = new this(); - // channel.userid = channel_id; const channel_data = await this.getUserDataById(channel_id); if (!channel_data) @@ -1688,16 +1699,6 @@ export class TwitchChannel extends BaseChannel { const channel_login = channel_data.login; - const channel_config = LiveStreamDVR.getInstance().channels_config.find( - (c) => - c.provider == "twitch" && - (c.login === channel_login || c.internalName === channel_login) - ); - if (!channel_config) - throw new Error( - `Could not find channel config in memory for channel login: ${channel_login}` - ); - channel.uuid = channel_config.uuid; channel.channel_data = channel_data; channel.config = channel_config; @@ -1826,11 +1827,11 @@ export class TwitchChannel extends BaseChannel { public static async create( config: TwitchChannelConfig ): Promise { + // check if channel already exists in config const exists_config = LiveStreamDVR.getInstance().channels_config.find( (ch) => ch.provider == "twitch" && - (ch.login === config.login || - ch.internalName === config.login || + (ch.login === config.internalName || ch.internalName === config.internalName) ); if (exists_config) @@ -1838,6 +1839,7 @@ export class TwitchChannel extends BaseChannel { `Channel ${config.internalName} already exists in config` ); + // check if channel already exists in memory const exists_channel = LiveStreamDVR.getInstance() .getChannels() .find( @@ -1850,6 +1852,7 @@ export class TwitchChannel extends BaseChannel { `Channel ${config.internalName} already exists in channels` ); + // fetch channel data const data = await TwitchChannel.getUserDataByLogin( config.internalName ); @@ -1863,7 +1866,8 @@ export class TwitchChannel extends BaseChannel { LiveStreamDVR.getInstance().channels_config.push(config); LiveStreamDVR.getInstance().saveChannelsConfig(); - const channel = await TwitchChannel.loadFromLogin(config.internalName); + // const channel = await TwitchChannel.loadFromLogin(config.internalName); + const channel = await TwitchChannel.load(config.uuid); if (!channel || !channel.internalName) throw new Error( `Channel ${config.internalName} could not be loaded` @@ -2006,7 +2010,7 @@ export class TwitchChannel extends BaseChannel { let response; if (!TwitchHelper.hasAxios()) { - throw new Error("Axios is not initialized"); + throw new Error("Axios is not initialized (getStreams)"); } try { @@ -2047,26 +2051,24 @@ export class TwitchChannel extends BaseChannel { return json.data ?? false; } - /** - * Load channel class using login, don't call this. Used internally. - * - * @internal - * @param login - * @returns - */ - public static async loadFromLogin(login: string): Promise { - if (!login) throw new Error("Streamer login is empty"); - if (typeof login !== "string") - throw new TypeError("Streamer login is not a string"); - log( - LOGLEVEL.DEBUG, - "tw.channel.loadFromLogin", - `Load from login ${login}` + public static async load(uuid: string): Promise { + /* + const channel_config = LiveStreamDVR.getInstance().channels_config.find( + (c) => c.uuid === uuid ); - const channel_id = await this.channelIdFromLogin(login); - if (!channel_id) - throw new Error(`Could not get channel id from login: ${login}`); - return this.loadAbstract(channel_id); // $channel; + if (!channel_config) + throw new Error(`Could not find channel config for uuid: ${uuid}`); + + const channel_data = await this.getUserDataByLogin( + channel_config.internalName + ); + + if (!channel_data) + throw new Error( + `Could not get channel data for channel login: ${channel_config.internalName}` + ); + */ + return await this.loadAbstract(uuid); } public static async channelIdFromLogin( @@ -2141,6 +2143,10 @@ export class TwitchChannel extends BaseChannel { `Fetching user data for ${method} ${identifier}, force: ${force}` ); + if (identifier == undefined || identifier == null || identifier == "") { + throw new Error(`getUserDataProxy: identifier is empty`); + } + // check cache first if (!force) { const channelData = @@ -2202,7 +2208,7 @@ export class TwitchChannel extends BaseChannel { */ if (!TwitchHelper.hasAxios()) { - throw new Error("Axios is not initialized"); + throw new Error("Axios is not initialized (getUserDataProxy)"); } let response; @@ -2478,7 +2484,7 @@ export class TwitchChannel extends BaseChannel { ); if (!TwitchHelper.hasAxios()) { - throw new Error("Axios is not initialized"); + throw new Error("Axios is not initialized (getChannelDataById)"); } let response; @@ -2660,7 +2666,9 @@ export class TwitchChannel extends BaseChannel { }; if (!TwitchHelper.hasAxios()) { - throw new Error("Axios is not initialized"); + throw new Error( + "Axios is not initialized (subscribeToIdWithWebhook)" + ); } let response; @@ -3006,7 +3014,9 @@ export class TwitchChannel extends BaseChannel { }; if (!TwitchHelper.hasAxios()) { - throw new Error("Axios is not initialized"); + throw new Error( + "Axios is not initialized (subscribeToIdWithWebsocket)" + ); } let response; diff --git a/server/tests/api.test.ts b/server/tests/api.test.ts index 85ba132d..e708bf92 100644 --- a/server/tests/api.test.ts +++ b/server/tests/api.test.ts @@ -17,7 +17,7 @@ import "./environment"; // jest.mock("../src/Core/TwitchChannel"); let app: Express | undefined; -let spy1: jest.SpyInstance | undefined; +// let spy1: jest.SpyInstance | undefined; // jest.mock("../src/Providers/Twitch"); // jest.mock("../src/Core/Config"); @@ -60,26 +60,30 @@ beforeAll(async () => { app.use("", baserouter); // TwitchChannel.getChannelDataProxy + /* spy1 = jest .spyOn(TwitchChannel, "getUserDataProxy") - .mockImplementation(() => { - return Promise.resolve({ - provider: "twitch", - id: "12345", - login: "test", - display_name: "test", - type: "", - broadcaster_type: "partner", - description: "test", - profile_image_url: "test", - offline_image_url: "test", - view_count: 0, - created_at: "test", - _updated: 1234, - cache_avatar: "test", - cache_offline_image: "", - } as UserData); - }); + .mockImplementation( + (method: string, identifier: string, force: boolean) => { + return Promise.resolve({ + provider: "twitch", + id: "1234", + login: identifier, + display_name: identifier, + type: "", + broadcaster_type: "partner", + description: "test", + profile_image_url: "test", + offline_image_url: "test", + view_count: 0, + created_at: "test", + _updated: 1234, + cache_avatar: "test", + cache_offline_image: "", + } as UserData); + } + ); + */ }); // afterEach(() => { @@ -90,7 +94,7 @@ afterAll(() => { Config.destroyInstance(); LiveStreamDVR.shutdown("test", true); app = undefined; - spy1?.mockRestore(); + // spy1?.mockRestore(); jest.restoreAllMocks(); }); @@ -188,9 +192,47 @@ describe("channels", () => { download_vod_at_end_quality: "best", }; + const add_data2 = { + provider: "twitch", + internalName: "test2", + quality: "best 1080p60", + match: "", + download_chat: true, + live_chat: false, + burn_chat: false, + no_capture: false, + no_cleanup: true, + max_storage: 2, + max_vods: 5, + download_vod_at_end: false, + download_vod_at_end_quality: "best", + }; + it("should add a channel in isolated mode", async () => { Config.getInstance().setConfig("app_url", ""); Config.getInstance().setConfig("isolated_mode", true); + + const spy = jest + .spyOn(TwitchChannel, "getUserDataProxy") + .mockReturnValue( + Promise.resolve({ + provider: "twitch", + id: "1234", + login: "test", + display_name: "test", + type: "", + broadcaster_type: "partner", + description: "test", + profile_image_url: "test", + offline_image_url: "test", + view_count: 0, + created_at: "test", + _updated: 1234, + cache_avatar: "test", + cache_offline_image: "", + } as UserData) + ); + const res3 = await request(app).post("/api/v0/channels").send(add_data); expect(res3.body.message).toContain("'test' created"); expect(res3.body.data).toHaveProperty("displayName"); @@ -198,10 +240,14 @@ describe("channels", () => { LiveStreamDVR.getInstance().clearChannels(); LiveStreamDVR.getInstance().channels_config = []; + + spy.mockRestore(); }); - it("should not add a channel", async () => { - spy1?.mockClear(); + it("should not add a channel because of quality mismatch", async () => { + // spy1?.mockClear(); + + const spy = jest.spyOn(TwitchChannel, "getUserDataProxy"); const res = await request(app).post("/api/v0/channels").send({ provider: "twitch", @@ -217,17 +263,43 @@ describe("channels", () => { expect(res.status).toBe(400); // expect(res.body.message).toContain("Invalid quality"); - expect(spy1).not.toHaveBeenCalled(); + // expect(spy1).not.toHaveBeenCalled(); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); }); it("should fail adding channel due to subscribe stuff", async () => { // both disabled Config.getInstance().setConfig("app_url", ""); Config.getInstance().setConfig("isolated_mode", false); + + const spy = jest + .spyOn(TwitchChannel, "getUserDataProxy") + .mockReturnValue( + Promise.resolve({ + provider: "twitch", + id: "1234", + login: "test", + display_name: "test", + type: "", + broadcaster_type: "partner", + description: "test", + profile_image_url: "test", + offline_image_url: "test", + view_count: 0, + created_at: "test", + _updated: 1234, + cache_avatar: "test", + cache_offline_image: "", + } as UserData) + ); + const res1 = await request(app).post("/api/v0/channels").send(add_data); expect(res1.body.message).toContain("no app_url"); expect(res1.status).toBe(400); + spy.mockRestore(); + // debug app url // Config.getInstance().setConfig("app_url", "debug"); // Config.getInstance().setConfig("isolated_mode", false); @@ -253,9 +325,31 @@ describe("channels", () => { it("should add a channel", async () => { Config.getInstance().setConfig("app_url", "https://example.com"); Config.getInstance().setConfig("isolated_mode", false); + + const spy = jest + .spyOn(TwitchChannel, "getUserDataProxy") + .mockReturnValue( + Promise.resolve({ + provider: "twitch", + id: "1234", + login: "test", + display_name: "test", + type: "", + broadcaster_type: "partner", + description: "test", + profile_image_url: "test", + offline_image_url: "test", + view_count: 0, + created_at: "test", + _updated: 1234, + cache_avatar: "test", + cache_offline_image: "", + } as UserData) + ); + const res4 = await request(app).post("/api/v0/channels").send(add_data); expect(res4.body.message).toContain("'test' created"); - expect(res4.body.data).toHaveProperty("display_name"); + expect(res4.body.data).toHaveProperty("displayName"); expect(res4.status).toBe(200); uuid = res4.body.data.uuid; @@ -264,8 +358,10 @@ describe("channels", () => { // TwitchChannel.channels = []; // TwitchChannel.channels_config = []; - expect(spy1).toHaveBeenCalled(); + // expect(spy1).toHaveBeenCalled(); + expect(spy).toHaveBeenCalled(); expect(TwitchChannel.subscribeToIdWithWebhook).toHaveBeenCalled(); + spy.mockRestore(); Config.getInstance().setConfig("app_url", ""); Config.getInstance().setConfig("isolated_mode", false); @@ -289,7 +385,7 @@ describe("channels", () => { expect(uuid).not.toBe(""); const channel_res = await request(app).get(`/api/v0/channels/${uuid}`); expect(channel_res.status).toBe(200); - expect(channel_res.body.data).toHaveProperty("display_name"); + expect(channel_res.body.data).toHaveProperty("displayName"); }); it("added channel should be in channels list", async () => { @@ -297,18 +393,97 @@ describe("channels", () => { const channels_res = await request(app).get("/api/v0/channels"); expect(channels_res.status).toBe(200); expect(channels_res.body.data.streamer_list).toHaveLength(1); - expect(channels_res.body.data.streamer_list[0].display_name).toBe( + expect(channels_res.body.data.streamer_list[0].displayName).toBe( "test" ); }); it("should remove a channel", async () => { const res = await request(app).delete(`/api/v0/channels/${uuid}`); + expect(res.status).toBe(200); expect(res.body.message).toContain("'test' deleted"); expect(res.body.status).toBe("OK"); - expect(res.status).toBe(200); expect(TwitchChannel.unsubscribeFromIdWithWebhook).toHaveBeenCalled(); }); + + it("two channels should be added", async () => { + LiveStreamDVR.getInstance().clearChannels(); + LiveStreamDVR.getInstance().channels_config = []; + Config.getInstance().setConfig("app_url", "https://example.com"); + + let spy = jest + .spyOn(TwitchChannel, "getUserDataProxy") + .mockImplementation( + (method: string, identifier: string, force: boolean) => + Promise.resolve({ + provider: "twitch", + id: "1234", + login: "test", + display_name: "test", + type: "", + broadcaster_type: "partner", + description: "test", + profile_image_url: "test", + offline_image_url: "test", + view_count: 0, + created_at: "test", + _updated: 1234, + cache_avatar: "test", + cache_offline_image: "", + }) + ); + + const res1 = await request(app).post("/api/v0/channels").send(add_data); + expect(res1.body.message).toContain("'test' created"); + expect(res1.status).toBe(200); + expect(TwitchChannel.subscribeToIdWithWebhook).toHaveBeenCalled(); + + spy.mockRestore(); + + spy = jest + .spyOn(TwitchChannel, "getUserDataProxy") + .mockImplementation( + (method: string, identifier: string, force: boolean) => + Promise.resolve({ + provider: "twitch", + id: "1234", + login: "test2", + display_name: "test2", + type: "", + broadcaster_type: "partner", + description: "test", + profile_image_url: "test", + offline_image_url: "test", + view_count: 0, + created_at: "test", + _updated: 1234, + cache_avatar: "test", + cache_offline_image: "", + }) + ); + + const res2 = await request(app) + .post("/api/v0/channels") + .send(add_data2); + expect(res2.body.message).toContain("'test2' created"); + expect(res2.status).toBe(200); + expect(TwitchChannel.subscribeToIdWithWebhook).toHaveBeenCalled(); + + spy.mockRestore(); + + const channels_res = await request(app).get("/api/v0/channels"); + expect(channels_res.status).toBe(200); + expect(channels_res.body.data.streamer_list).toHaveLength(2); + expect(channels_res.body.data.streamer_list[0].displayName).toBe( + "test" + ); + expect(channels_res.body.data.streamer_list[1].displayName).toBe( + "test2" + ); + + LiveStreamDVR.getInstance().clearChannels(); + LiveStreamDVR.getInstance().channels_config = []; + }); }); describe("auth", () => { diff --git a/server/tests/environment.ts b/server/tests/environment.ts index cc9ea810..3d4aab82 100644 --- a/server/tests/environment.ts +++ b/server/tests/environment.ts @@ -1,17 +1,16 @@ +import { Config } from "../src/Core/Config"; +import { Job } from "../src/Core/Job"; +import { KeyValue } from "../src/Core/KeyValue"; +import { LiveStreamDVR } from "../src/Core/LiveStreamDVR"; import * as LogModule from "../src/Core/Log"; import { BaseChannel } from "../src/Core/Providers/Base/BaseChannel"; import { BaseVOD } from "../src/Core/Providers/Base/BaseVOD"; import { TwitchChannel } from "../src/Core/Providers/Twitch/TwitchChannel"; +import { TwitchGame } from "../src/Core/Providers/Twitch/TwitchGame"; import { TwitchVOD } from "../src/Core/Providers/Twitch/TwitchVOD"; import { YouTubeVOD } from "../src/Core/Providers/YouTube/YouTubeVOD"; -import { Config } from "../src/Core/Config"; -import { LiveStreamDVR } from "../src/Core/LiveStreamDVR"; import { Scheduler } from "../src/Core/Scheduler"; import { TwitchHelper } from "../src/Providers/Twitch"; -import { AppRoot, DataRoot } from "../src/Core/BaseConfig"; -import { KeyValue } from "../src/Core/KeyValue"; -import { TwitchGame } from "../src/Core/Providers/Twitch/TwitchGame"; -import { Job } from "../src/Core/Job"; /* // dangerous methods that need to be mocked/disabled for tests @@ -50,52 +49,170 @@ Job.loadJobsFromCache // mock methods beforeAll(() => { - jest.spyOn(Config.prototype, "loadConfig").mockImplementation(function(this: Config) { + jest.spyOn(Config.prototype, "loadConfig").mockImplementation(function ( + this: Config + ) { this.generateConfig(); return true; }); - jest.spyOn(Config.prototype, "saveConfig").mockImplementation(() => { return true; }); - jest.spyOn(Config.prototype, "startWatchingConfig").mockImplementation(() => { console.debug("Disable start watching config"); return; }); - jest.spyOn(Config, "checkBuiltDependencies").mockImplementation(() => { return; }); - jest.spyOn(Config, "checkAppRoot").mockImplementation(() => { return; }); - jest.spyOn(Config, "createFolders").mockImplementation(() => { return; }); - jest.spyOn(KeyValue.prototype, "save").mockImplementation(() => { return; }); - jest.spyOn(KeyValue.prototype, "load").mockImplementation(() => { return; }); - jest.spyOn(LiveStreamDVR.prototype, "loadChannelsConfig").mockImplementation(() => { return true; }); - jest.spyOn(LiveStreamDVR.prototype, "saveChannelsConfig").mockImplementation(() => { return true; }); - jest.spyOn(LiveStreamDVR.prototype, "startDiskSpaceInterval").mockImplementation(() => { return true; }); - jest.spyOn(LiveStreamDVR, "checkBinaryVersions").mockImplementation(() => { return Promise.resolve(); }); - jest.spyOn(LiveStreamDVR, "checkVersion").mockImplementation(() => { return; }); + jest.spyOn(Config.prototype, "saveConfig").mockImplementation(() => { + return true; + }); + jest.spyOn(Config.prototype, "startWatchingConfig").mockImplementation( + () => { + console.debug("Disable start watching config"); + return; + } + ); + jest.spyOn(Config, "checkBuiltDependencies").mockImplementation(() => { + return; + }); + jest.spyOn(Config, "checkAppRoot").mockImplementation(() => { + return; + }); + jest.spyOn(Config, "createFolders").mockImplementation(() => { + return; + }); + jest.spyOn(KeyValue.prototype, "save").mockImplementation(() => { + return; + }); + jest.spyOn(KeyValue.prototype, "load").mockImplementation(() => { + return; + }); + jest.spyOn( + LiveStreamDVR.prototype, + "loadChannelsConfig" + ).mockImplementation(() => { + return true; + }); + jest.spyOn( + LiveStreamDVR.prototype, + "saveChannelsConfig" + ).mockImplementation(() => { + return true; + }); + jest.spyOn( + LiveStreamDVR.prototype, + "startDiskSpaceInterval" + ).mockImplementation(() => { + return true; + }); + jest.spyOn(LiveStreamDVR, "checkBinaryVersions").mockImplementation(() => { + return Promise.resolve(); + }); + jest.spyOn(LiveStreamDVR, "checkVersion").mockImplementation(() => { + return; + }); // jest.spyOn(Log, "logAdvanced").mockImplementation((level, module, text, meta) => { // console.log(`[TEST][${level}] ${module}: ${text}`); // return; // }); // logAdvanced is now a regular export at the top of the file, not a class method - jest.spyOn(LogModule, "log").mockImplementation((level, module, text, meta) => { - console.log(`[TEST][${level}] ${module}: ${text}`); + jest.spyOn(LogModule, "log").mockImplementation( + (level, module, text, meta) => { + console.log(`[TEST][${level}] ${module}: ${text}`); + return; + } + ); + jest.spyOn(Scheduler, "defaultJobs").mockImplementation(() => { + return; + }); + jest.spyOn(BaseChannel.prototype, "broadcastUpdate").mockImplementation( + () => { + return; + } + ); + jest.spyOn(BaseChannel.prototype, "saveVodDatabase").mockImplementation( + () => { + return; + } + ); + jest.spyOn(BaseChannel.prototype, "findClips").mockImplementation(() => { + return Promise.resolve(); + }); + jest.spyOn(BaseChannel.prototype, "makeFolder").mockImplementation(() => { + return; + }); + jest.spyOn(BaseVOD.prototype, "startWatching").mockImplementation(() => { + console.debug("Disable start watching basevod"); + return Promise.resolve(true); + }); + jest.spyOn(BaseVOD.prototype, "broadcastUpdate").mockImplementation(() => { return; }); - jest.spyOn(Scheduler, "defaultJobs").mockImplementation(() => { return; }); - jest.spyOn(BaseChannel.prototype, "broadcastUpdate").mockImplementation(() => { return; }); - jest.spyOn(BaseChannel.prototype, "saveVodDatabase").mockImplementation(() => { return; }); - jest.spyOn(BaseChannel.prototype, "findClips").mockImplementation(() => { return Promise.resolve(); }); - jest.spyOn(BaseChannel.prototype, "makeFolder").mockImplementation(() => { return; }); - jest.spyOn(BaseVOD.prototype, "startWatching").mockImplementation(() => { console.debug("Disable start watching basevod"); return Promise.resolve(true); }); - jest.spyOn(BaseVOD.prototype, "broadcastUpdate").mockImplementation(() => { return; }); - jest.spyOn(TwitchChannel.prototype, "startWatching").mockImplementation(() => { console.debug("Disable start watching twitchchannel"); return Promise.resolve(); }); - jest.spyOn(TwitchChannel, "loadChannelsCache").mockImplementation(() => { return true; }); - jest.spyOn(TwitchChannel, "getUserDataProxy").mockImplementation(() => { return Promise.resolve(false); }); - jest.spyOn(TwitchVOD.prototype, "startWatching").mockImplementation(() => { console.debug("Disable start watching twitchvod"); return Promise.resolve(true); }); - jest.spyOn(TwitchVOD.prototype, "saveJSON").mockImplementation(() => { return Promise.resolve(true); }); - jest.spyOn(YouTubeVOD.prototype, "saveJSON").mockImplementation(() => { return Promise.resolve(true); }); - jest.spyOn(TwitchChannel, "subscribeToIdWithWebhook").mockImplementation(() => { return Promise.resolve(true); }); - jest.spyOn(TwitchChannel, "unsubscribeFromIdWithWebhook").mockImplementation(() => { return Promise.resolve(true); }); - jest.spyOn(TwitchChannel, "subscribeToIdWithWebsocket").mockImplementation(() => { return Promise.resolve(true); }); + jest.spyOn(TwitchChannel.prototype, "startWatching").mockImplementation( + () => { + console.debug("Disable start watching twitchchannel"); + return Promise.resolve(); + } + ); + jest.spyOn(TwitchChannel, "loadChannelsCache").mockImplementation(() => { + return true; + }); + /* + jest.spyOn(TwitchChannel, "getUserDataProxy").mockImplementation(() => { + return Promise.resolve(false); + }); + jest.spyOn(TwitchChannel, "getUserDataByLogin").mockImplementation( + (login: string, force?: boolean) => { + return Promise.resolve({ + login: login, + _updated: 1, + cache_offline_image: "", + profile_image_url: "", + offline_image_url: "", + created_at: "", + id: "1234", + avatar_cache: "", + avatar_thumb: "", + broadcaster_type: "partner", + display_name: login, + type: "", + description: "", + view_count: 0, + }); + } + ); + */ + jest.spyOn(TwitchVOD.prototype, "startWatching").mockImplementation(() => { + console.debug("Disable start watching twitchvod"); + return Promise.resolve(true); + }); + jest.spyOn(TwitchVOD.prototype, "saveJSON").mockImplementation(() => { + return Promise.resolve(true); + }); + jest.spyOn(YouTubeVOD.prototype, "saveJSON").mockImplementation(() => { + return Promise.resolve(true); + }); + jest.spyOn(TwitchChannel, "subscribeToIdWithWebhook").mockImplementation( + () => { + return Promise.resolve(true); + } + ); + jest.spyOn( + TwitchChannel, + "unsubscribeFromIdWithWebhook" + ).mockImplementation(() => { + return Promise.resolve(true); + }); + jest.spyOn(TwitchChannel, "subscribeToIdWithWebsocket").mockImplementation( + () => { + return Promise.resolve(true); + } + ); // jest.spyOn(TwitchChannel, "unsubscribeFromIdWithWebsocket").mockImplementation(() => { return Promise.resolve(true); }); - jest.spyOn(TwitchHelper, "getAccessToken").mockImplementation(() => { return Promise.resolve("test"); }); - jest.spyOn(TwitchGame, "populateFavouriteGames").mockImplementation(() => { return Promise.resolve(); }); - jest.spyOn(TwitchGame, "populateGameDatabase").mockImplementation(() => { return Promise.resolve(); }); - jest.spyOn(Job, "loadJobsFromCache").mockImplementation(() => { return; }); + jest.spyOn(TwitchHelper, "getAccessToken").mockImplementation(() => { + return Promise.resolve("test"); + }); + jest.spyOn(TwitchGame, "populateFavouriteGames").mockImplementation(() => { + return Promise.resolve(); + }); + jest.spyOn(TwitchGame, "populateGameDatabase").mockImplementation(() => { + return Promise.resolve(); + }); + jest.spyOn(Job, "loadJobsFromCache").mockImplementation(() => { + return; + }); // mock data consts // jest.spyOn(AppRoot, "get").mockImplementation(() => { return "test"; }); From 45284ef6c4d5eb81c0ad8c1704c639976357520a Mon Sep 17 00:00:00 2001 From: Braxen Date: Sun, 19 Nov 2023 10:20:22 +0100 Subject: [PATCH 5/8] youtube channel cleanup --- client-vue/package.json | 2 +- .../streamer/VideoDownloadModal.vue | 2 +- .../core/Providers/YouTube/YouTubeChannel.ts | 4 +- common/Api/Client.ts | 4 +- server/package.json | 2 +- server/src/Controllers/Channels.ts | 2 +- server/src/Core/LiveStreamDVR.ts | 33 ++--- .../Core/Providers/YouTube/YouTubeChannel.ts | 119 +++++++++--------- 8 files changed, 89 insertions(+), 79 deletions(-) diff --git a/client-vue/package.json b/client-vue/package.json index c8f26e40..7d18facc 100644 --- a/client-vue/package.json +++ b/client-vue/package.json @@ -1,6 +1,6 @@ { "name": "livestreamdvr-client", - "version": "2.4.0", + "version": "2.4.1", "private": true, "homepage": "https://github.com/MrBrax/LiveStreamDVR", "scripts": { diff --git a/client-vue/src/components/streamer/VideoDownloadModal.vue b/client-vue/src/components/streamer/VideoDownloadModal.vue index 3f6cb975..e4bd1d7a 100644 --- a/client-vue/src/components/streamer/VideoDownloadModal.vue +++ b/client-vue/src/components/streamer/VideoDownloadModal.vue @@ -119,7 +119,7 @@ async function fetchYouTubeVods() { let response; try { - response = await axios.get(`/api/v0/youtubeapi/videos/${props.streamer.channel_id}`); + response = await axios.get(`/api/v0/youtubeapi/videos/${props.streamer.internalId}`); } catch (error) { if (axios.isAxiosError(error)) { console.error("fetchYouTubeVods error", error.response); diff --git a/client-vue/src/core/Providers/YouTube/YouTubeChannel.ts b/client-vue/src/core/Providers/YouTube/YouTubeChannel.ts index d870d697..976d4928 100644 --- a/client-vue/src/core/Providers/YouTube/YouTubeChannel.ts +++ b/client-vue/src/core/Providers/YouTube/YouTubeChannel.ts @@ -6,8 +6,8 @@ import YouTubeVOD from "./YouTubeVOD"; export default class YouTubeChannel extends BaseChannel { uuid = ""; readonly provider = "youtube"; - channel_id = ""; - display_name = ""; + // channel_id = ""; + // display_name = ""; // quality: VideoQuality[] = []; profile_image_url = ""; diff --git a/common/Api/Client.ts b/common/Api/Client.ts index 7a397265..5893f5f0 100644 --- a/common/Api/Client.ts +++ b/common/Api/Client.ts @@ -239,8 +239,8 @@ export interface ApiTwitchChannel extends ApiBaseChannel { export interface ApiYouTubeChannel extends ApiBaseChannel { provider: "youtube"; - channel_id: string; - display_name: string; + // channel_id: string; + // display_name: string; // quality: VideoQuality[] | undefined; vods_list: ApiYouTubeVod[]; profile_image_url: string; diff --git a/server/package.json b/server/package.json index c618d299..bb8548d4 100644 --- a/server/package.json +++ b/server/package.json @@ -1,6 +1,6 @@ { "name": "livestreamdvr-server", - "version": "1.7.0", + "version": "1.7.1", "description": "", "main": "index.ts", "scripts": { diff --git a/server/src/Controllers/Channels.ts b/server/src/Controllers/Channels.ts index 3b8d27bd..aa6cbd97 100644 --- a/server/src/Controllers/Channels.ts +++ b/server/src/Controllers/Channels.ts @@ -241,7 +241,7 @@ export function UpdateChannel( const channel_config: YouTubeChannelConfig = { uuid: channel.uuid, provider: "youtube", - channel_id: channel.channel_id || "", + // channel_id: channel.channel_id || "", internalId: channel.internalId, internalName: channel.internalName, quality: quality, diff --git a/server/src/Core/LiveStreamDVR.ts b/server/src/Core/LiveStreamDVR.ts index db5a1069..954e1a06 100644 --- a/server/src/Core/LiveStreamDVR.ts +++ b/server/src/Core/LiveStreamDVR.ts @@ -336,24 +336,28 @@ export class LiveStreamDVR { public async loadChannels(): Promise { log(LOGLEVEL.INFO, "dvr.loadChannels", "Loading channels..."); if (this.channels_config.length > 0) { - for (const channel of this.channels_config) { + for (const channelConfig of this.channels_config) { log( LOGLEVEL.INFO, "dvr.loadChannels", - `Loading channel ${channel.uuid}, provider ${channel.provider}...` + `Loading channel ${channelConfig.uuid}, provider ${channelConfig.provider}...` ); - if (!channel.provider || channel.provider == "twitch") { + if ( + !channelConfig.provider || + channelConfig.provider == "twitch" + ) { let ch: TwitchChannel; try { - ch = await TwitchChannel.load(channel.uuid); + ch = await TwitchChannel.load(channelConfig.uuid); } catch (th) { log( LOGLEVEL.FATAL, "dvr.load.tw", `TW Channel ${ - channel.internalName || channel.login + channelConfig.internalName || + channelConfig.login } could not be loaded: ${th}` ); console.error(th); @@ -369,7 +373,8 @@ export class LiveStreamDVR { LOGLEVEL.SUCCESS, "dvr.load.tw", `Loaded channel ${ - channel.internalName || channel.login + channelConfig.internalName || + channelConfig.login } with ${ch.getVods().length} vods` ); if (ch.no_capture) { @@ -377,7 +382,8 @@ export class LiveStreamDVR { LOGLEVEL.WARNING, "dvr.load.tw", `Channel ${ - channel.internalName || channel.login + channelConfig.internalName || + channelConfig.login } is configured to not capture streams.` ); } @@ -386,23 +392,22 @@ export class LiveStreamDVR { LOGLEVEL.FATAL, "dvr.load.tw", `Channel ${ - channel.internalName || channel.login + channelConfig.internalName || + channelConfig.login } could not be added, please check logs.` ); break; } - } else if (channel.provider == "youtube") { + } else if (channelConfig.provider == "youtube") { let ch: YouTubeChannel; try { - ch = await YouTubeChannel.loadFromId( - channel.internalId - ); + ch = await YouTubeChannel.load(channelConfig.uuid); } catch (th) { log( LOGLEVEL.FATAL, "dvr.load.yt", - `YT Channel ${channel.internalName} could not be loaded: ${th}` + `YT Channel ${channelConfig.internalName} could not be loaded: ${th}` ); console.error(th); continue; @@ -431,7 +436,7 @@ export class LiveStreamDVR { log( LOGLEVEL.FATAL, "dvr.load.yt", - `Channel ${channel.channel_id} could not be added, please check logs.` + `Channel ${channelConfig.channel_id} could not be added, please check logs.` ); break; } diff --git a/server/src/Core/Providers/YouTube/YouTubeChannel.ts b/server/src/Core/Providers/YouTube/YouTubeChannel.ts index 1bf884e4..50a18221 100644 --- a/server/src/Core/Providers/YouTube/YouTubeChannel.ts +++ b/server/src/Core/Providers/YouTube/YouTubeChannel.ts @@ -30,7 +30,7 @@ export class YouTubeChannel extends BaseChannel { public provider: Providers = "youtube"; - public channel_id = ""; + // public channel_id = ""; public vods_list: YouTubeVOD[] = []; @@ -52,26 +52,29 @@ export class YouTubeChannel extends BaseChannel { } static async create(config: YouTubeChannelConfig): Promise { + // check if channel exists in config const exists_config = LiveStreamDVR.getInstance().channels_config.find( - (ch) => - ch.provider == "youtube" && ch.channel_id === config.channel_id + (configEntry) => + configEntry.provider == "youtube" && + (configEntry.channel_id === config.internalId || + configEntry.internalId === config.internalId) ); if (exists_config) throw new Error( - `Channel ${config.channel_id} already exists in config` + `Channel ${config.internalId} already exists in config` ); - // const exists_channel = TwitchChannel.channels.find(ch => ch.login === config.login); + // check if channel exists in memory const exists_channel = LiveStreamDVR.getInstance() .getChannels() .find( (channel): channel is YouTubeChannel => isYouTubeChannel(channel) && - channel.channel_id === config.channel_id + channel.internalId === config.internalId ); if (exists_channel) throw new Error( - `Channel ${config.channel_id} already exists in channels` + `Channel ${config.internalId} already exists in channels` ); const data = await YouTubeChannel.getUserDataById(config.internalId); @@ -85,8 +88,8 @@ export class YouTubeChannel extends BaseChannel { LiveStreamDVR.getInstance().channels_config.push(config); LiveStreamDVR.getInstance().saveChannelsConfig(); - const channel = await YouTubeChannel.loadFromId(config.internalId); - if (!channel || !channel.channel_id) + const channel = await YouTubeChannel.load(config.uuid); + if (!channel || !channel.internalId) throw new Error(`Channel ${config.internalId} could not be loaded`); if ( @@ -95,12 +98,12 @@ export class YouTubeChannel extends BaseChannel { !Config.getInstance().cfg("isolated_mode") ) { try { - await YouTubeChannel.subscribe(channel.channel_id); + await YouTubeChannel.subscribe(channel.internalId); } catch (error) { log( LOGLEVEL.ERROR, "yt.channel", - `Failed to subscribe to channel ${channel.channel_id}: ${ + `Failed to subscribe to channel ${channel.internalId}: ${ (error as Error).message }` ); @@ -118,25 +121,27 @@ export class YouTubeChannel extends BaseChannel { log( LOGLEVEL.WARNING, "yt.channel", - `Not subscribing to ${channel.channel_id} due to debug app_url.` + `Not subscribing to ${channel.internalId} due to debug app_url.` ); } else if (Config.getInstance().cfg("isolated_mode")) { log( LOGLEVEL.WARNING, "yt.channel", - `Not subscribing to ${channel.channel_id} due to isolated mode.` + `Not subscribing to ${channel.internalId} due to isolated mode.` ); } else { log( LOGLEVEL.ERROR, "yt.channel", - `Can't subscribe to ${channel.channel_id} due to either no app_url or isolated mode disabled.` + `Can't subscribe to ${channel.internalId} due to either no app_url or isolated mode disabled.` ); + + // remove channel from config LiveStreamDVR.getInstance().channels_config = LiveStreamDVR.getInstance().channels_config.filter( - (ch) => - ch.provider == "youtube" && - ch.channel_id !== config.channel_id + (configEntry) => + configEntry.provider == "youtube" && + configEntry.internalId !== config.internalId ); // remove channel from config LiveStreamDVR.getInstance().saveChannelsConfig(); throw new Error( @@ -158,58 +163,58 @@ export class YouTubeChannel extends BaseChannel { return channel; } - public static async loadFromId( - channel_id: string - ): Promise { - if (!channel_id) throw new Error("Streamer login is empty"); - if (typeof channel_id !== "string") - throw new TypeError("Streamer id is not a string"); - log(LOGLEVEL.DEBUG, "yt.channel", `Load from login ${channel_id}`); + public static async load(uuid: string): Promise { + // if (!channel_id) throw new Error("Streamer login is empty"); + // if (typeof channel_id !== "string") + // throw new TypeError("Streamer id is not a string"); + // log(LOGLEVEL.DEBUG, "yt.channel", `Load from login ${channel_id}`); // const channel_id = await this.channelIdFromLogin(channel_id); // if (!channel_id) throw new Error(`Could not get channel id from login: ${channel_id}`); - return await this.loadAbstract(channel_id); // $channel; + return await this.loadAbstract(uuid); } public static async loadAbstract( - channel_id: string + // channel_id: string + uuid: string ): Promise { - log(LOGLEVEL.DEBUG, "yt.channel", `Load channel ${channel_id}`); + log(LOGLEVEL.DEBUG, "yt.channel", `Load channel ${uuid}`); const channel_memory = LiveStreamDVR.getInstance() .getChannels() .find( (channel): channel is YouTubeChannel => - isYouTubeChannel(channel) && - channel.channel_id === channel_id + isYouTubeChannel(channel) && channel.uuid === uuid ); if (channel_memory) { log( LOGLEVEL.WARNING, "yt.channel", - `Channel ${channel_id} already loaded` + `Channel ${uuid} (${channel_memory.internalName}) already exists in memory, returning` ); return channel_memory; } - const channel = new this(); - channel.channel_id = channel_id; - - const channel_data = await this.getUserDataById(channel_id); - if (!channel_data) + const channel_config = LiveStreamDVR.getInstance().channels_config.find( + (c) => c.provider == "youtube" && c.uuid === uuid + ); + if (!channel_config) throw new Error( - `Could not get channel data for channel id: ${channel_id}` + `Could not find channel config for channel uuid: ${uuid}` ); - // const channel_login = channel_data.login; + const channel = new this(); + // channel.channel_id = channel_id; - const channel_config = LiveStreamDVR.getInstance().channels_config.find( - (c) => c.provider == "youtube" && c.channel_id === channel_id + const channel_data = await this.getUserDataById( + channel_config.internalId ); - if (!channel_config) + if (!channel_data) throw new Error( - `Could not find channel config in memory for channel id: ${channel_id}` + `Could not get channel data for channel id: ${channel_config.internalId}` ); + // const channel_login = channel_data.login; + channel.uuid = channel_config.uuid; channel.channel_data = channel_data; channel.config = channel_config; @@ -220,25 +225,25 @@ export class YouTubeChannel extends BaseChannel { if ( await KeyValue.getInstance().getBoolAsync( - `yt.${channel.channel_id}.online` + `yt.${channel.internalId}.online` ) ) { log( LOGLEVEL.WARNING, "yt.channel", - `Channel ${channel.channel_id} is online, stale?` + `Channel ${channel.internalId} is online, stale?` ); } if ( await KeyValue.getInstance().hasAsync( - `yt.${channel.channel_id}.channeldata` + `yt.${channel.internalId}.channeldata` ) ) { log( LOGLEVEL.WARNING, "yt.channel", - `Channel ${channel.channel_id} has stale chapter data.` + `Channel ${channel.internalId} has stale chapter data.` ); } @@ -441,7 +446,7 @@ export class YouTubeChannel extends BaseChannel { .getChannels() .find( (ch): ch is YouTubeChannel => - isYouTubeChannel(ch) && ch.channel_id === channel_id + isYouTubeChannel(ch) && ch.internalId === channel_id ); } @@ -585,8 +590,8 @@ export class YouTubeChannel extends BaseChannel { return { ...(await super.toAPI()), provider: "youtube", - channel_id: this.channel_id || "", - display_name: this.displayName || "", + // channel_id: this.channel_id || "", + // display_name: this.displayName || "", description: this.description || "", profile_image_url: this.channel_data?.thumbnails?.default?.url || "", @@ -609,13 +614,13 @@ export class YouTubeChannel extends BaseChannel { get saves_vods(): boolean { return KeyValue.getInstance().getBool( - `yt.${this.channel_id}.saves_vods` + `yt.${this.internalId}.saves_vods` ); } public getChapterData(): BaseVODChapterJSON | undefined { const cd = KeyValue.getInstance().get( - `yt.${this.channel_id}.chapterdata` + `yt.${this.internalId}.chapterdata` ); return cd ? (JSON.parse(cd) as BaseVODChapterJSON) : undefined; } @@ -636,7 +641,7 @@ export class YouTubeChannel extends BaseChannel { * @returns Empty VOD */ public async createVOD(filename: string): Promise { - if (!this.channel_id) throw new Error("Channel id is not set"); + if (!this.internalId) throw new Error("Channel id is not set"); // if (!this.login) throw new Error("Channel login is not set"); if (!this.internalName) throw new Error("Channel display_name is not set"); @@ -644,7 +649,7 @@ export class YouTubeChannel extends BaseChannel { log( LOGLEVEL.INFO, "yt.channel.createVOD", - `Create VOD JSON for ${this.channel_id}: ${path.basename( + `Create VOD JSON for ${this.internalId}: ${path.basename( filename )} @ ${path.dirname(filename)}` ); @@ -764,7 +769,7 @@ export class YouTubeChannel extends BaseChannel { } public async refreshData(): Promise { - if (!this.channel_id) throw new Error("Channel id not set"); + if (!this.internalId) throw new Error("Channel id not set"); log( LOGLEVEL.INFO, "channel.refreshData", @@ -772,13 +777,13 @@ export class YouTubeChannel extends BaseChannel { ); const channel_data = await YouTubeChannel.getUserDataById( - this.channel_id, + this.internalId, true ); if (channel_data) { this.channel_data = channel_data; - this.channel_id = channel_data.id; + // this.internalId = channel_data.id; // this.login = channel_data.login; // this.display_name = channel_data.display_name; // this.profile_image_url = channel_data.profile_image_url; @@ -791,7 +796,7 @@ export class YouTubeChannel extends BaseChannel { } public async getVideos(): Promise { - return await YouTubeVOD.getVideosProxy(this.channel_id); + return await YouTubeVOD.getVideosProxy(this.internalId); } public async getStreams(): Promise { @@ -815,7 +820,7 @@ export class YouTubeChannel extends BaseChannel { let searchResponse; try { searchResponse = await service.search.list({ - channelId: this.channel_id, + channelId: this.internalId, eventType: "live", type: ["video"], part: ["id", "snippet"], From a81bc561135cff2cb0cc1be949438fa166a3c458 Mon Sep 17 00:00:00 2001 From: Braxen Date: Sun, 19 Nov 2023 21:45:11 +0100 Subject: [PATCH 6/8] silly format paths tests --- server/src/Helpers/Format.test.ts | 38 ++++++++----------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/server/src/Helpers/Format.test.ts b/server/src/Helpers/Format.test.ts index 9e79034c..f6477e4c 100644 --- a/server/src/Helpers/Format.test.ts +++ b/server/src/Helpers/Format.test.ts @@ -186,37 +186,19 @@ describe("Validate", () => { describe("validateRelativePath", () => { it("should not validate absolute directory path", () => { - expect( - validateRelativePath(path.join("C:\\", "test", "test")) - ).toEqual(false); - expect( - validateRelativePath(path.join("C:\\", "test", "test\\")) - ).toEqual(false); - expect( - validateRelativePath(path.join("C:\\", "test", "test\\\\")) - ).toEqual(false); - expect( - validateRelativePath(path.join("C:\\", "test/test")) - ).toEqual(false); - expect(validateRelativePath(path.join("/test", "test"))).toEqual( - false - ); - expect(validateRelativePath(path.join("/test\0", "test"))).toEqual( - false - ); + expect(validateRelativePath("C:\\test\\test")).toEqual(false); + expect(validateRelativePath("C:\\test\\test\\")).toEqual(false); + expect(validateRelativePath("C:\\test\\test\\\\")).toEqual(false); + expect(validateRelativePath("C:\\test/test")).toEqual(false); + expect(validateRelativePath("/test/test")).toEqual(false); + expect(validateRelativePath("/test\0/test")).toEqual(false); }); it("should validate relative directory path", () => { - expect(validateRelativePath(path.join("test", "test"))).toEqual( - true - ); - expect(validateRelativePath(path.join("test", "test\\"))).toEqual( - true - ); - expect(validateRelativePath(path.join("test", "test\\\\"))).toEqual( - true - ); - expect(validateRelativePath(path.join("test/test"))).toEqual(true); + expect(validateRelativePath("test\\test")).toEqual(true); + expect(validateRelativePath("test\\test\\")).toEqual(true); + expect(validateRelativePath("test\\test\\\\")).toEqual(true); + expect(validateRelativePath("test/test")).toEqual(true); }); it("should fail relative directory path with ..", () => { From 058a2246d3a4e14a165f6552289c39d879bd720a Mon Sep 17 00:00:00 2001 From: Braxen Date: Sun, 19 Nov 2023 21:50:45 +0100 Subject: [PATCH 7/8] Update Format.test.ts --- server/src/Helpers/Format.test.ts | 51 ++++++++----------------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/server/src/Helpers/Format.test.ts b/server/src/Helpers/Format.test.ts index f6477e4c..9ba50260 100644 --- a/server/src/Helpers/Format.test.ts +++ b/server/src/Helpers/Format.test.ts @@ -7,7 +7,6 @@ import { } from "./Format"; import { formatString } from "@common/Format"; -import path from "path"; import { validateAbsolutePath, @@ -153,34 +152,18 @@ describe("Sanitize", () => { describe("Validate", () => { describe("validateAbsolutePath", () => { it("should validate absolute directory path", () => { - expect( - validateAbsolutePath(path.join("C:\\", "test", "test")) - ).toEqual(true); - expect( - validateAbsolutePath(path.join("C:\\", "test", "test\\")) - ).toEqual(true); - expect( - validateAbsolutePath(path.join("C:\\", "test", "test\\\\")) - ).toEqual(true); - expect( - validateAbsolutePath(path.join("C:\\", "test/test")) - ).toEqual(true); - expect(validateAbsolutePath(path.join("C:\\\0\\test"))).toEqual( - false - ); + expect(validateAbsolutePath("C:\\test\\test")).toEqual(true); + expect(validateAbsolutePath("C:\\test\\test\\")).toEqual(true); + expect(validateAbsolutePath("C:\\test\\test\\\\")).toEqual(true); + expect(validateAbsolutePath("C:\\test/test")).toEqual(true); + expect(validateAbsolutePath("C:\\\0\\test")).toEqual(false); }); it("should not validate relative directory path", () => { - expect(validateAbsolutePath(path.join("test", "test"))).toEqual( - false - ); - expect(validateAbsolutePath(path.join("test", "test\\"))).toEqual( - false - ); - expect(validateAbsolutePath(path.join("test", "test\\\\"))).toEqual( - false - ); - expect(validateAbsolutePath(path.join("test/test"))).toEqual(false); + expect(validateAbsolutePath("test\\test")).toEqual(false); + expect(validateAbsolutePath("test\\test\\")).toEqual(false); + expect(validateAbsolutePath("test\\test\\\\")).toEqual(false); + expect(validateAbsolutePath("test/test")).toEqual(false); }); }); @@ -202,18 +185,10 @@ describe("Validate", () => { }); it("should fail relative directory path with ..", () => { - expect(validateRelativePath(path.join("..", "test"))).toEqual( - false - ); - expect(validateRelativePath(path.join("..", "test\\"))).toEqual( - false - ); - expect(validateRelativePath(path.join("..", "test\\\\"))).toEqual( - false - ); - expect(validateRelativePath(path.join("..", "test/test"))).toEqual( - false - ); + expect(validateRelativePath("..\\test")).toEqual(false); + expect(validateRelativePath("..\\test\\")).toEqual(false); + expect(validateRelativePath("..\\test\\\\")).toEqual(false); + expect(validateRelativePath("../test/test")).toEqual(false); expect(validateRelativePath("test../test")).toEqual(true); }); }); From 2e9f19caac2940c9823d55d952f5321ac93cd2e0 Mon Sep 17 00:00:00 2001 From: Braxen Date: Sun, 19 Nov 2023 22:35:30 +0100 Subject: [PATCH 8/8] try to fix validate absolute path function, probably dangerous --- server/src/Helpers/Filesystem.ts | 9 ++++++++- server/src/Helpers/Format.test.ts | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/Helpers/Filesystem.ts b/server/src/Helpers/Filesystem.ts index b76dd14e..19a0660a 100644 --- a/server/src/Helpers/Filesystem.ts +++ b/server/src/Helpers/Filesystem.ts @@ -51,7 +51,14 @@ export function sanitizeFilename(filename: string): string { */ export function validateAbsolutePath(dir: string): boolean { - return path.isAbsolute(dir) && !dir.match(/\0/); + // return path.isAbsolute(dir) && !dir.match(/\0/); + const normalDir = path.normalize(dir); + if (normalDir.match(/\0/)) return false; + return ( + path.isAbsolute(normalDir) || + normalDir.startsWith("/") || + new RegExp(/^[a-zA-Z]:\\/).test(normalDir) + ); } /** diff --git a/server/src/Helpers/Format.test.ts b/server/src/Helpers/Format.test.ts index 9ba50260..ff14d076 100644 --- a/server/src/Helpers/Format.test.ts +++ b/server/src/Helpers/Format.test.ts @@ -157,6 +157,8 @@ describe("Validate", () => { expect(validateAbsolutePath("C:\\test\\test\\\\")).toEqual(true); expect(validateAbsolutePath("C:\\test/test")).toEqual(true); expect(validateAbsolutePath("C:\\\0\\test")).toEqual(false); + expect(validateAbsolutePath("/test/test")).toEqual(true); + expect(validateAbsolutePath("/test\0/test")).toEqual(false); }); it("should not validate relative directory path", () => {