From faa2e839a11219edc3f53e07ea4903f04061d373 Mon Sep 17 00:00:00 2001 From: ftk Date: Tue, 12 Jul 2022 16:35:20 +0300 Subject: [PATCH 1/9] storyboards: row,cols,fps support (not merged into yt-dlp yet) --- src/thumbnailer_server.lua | 13 ++++++++----- src/thumbnailer_shared.lua | 24 ++++++++++++++++-------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/thumbnailer_server.lua b/src/thumbnailer_server.lua index 565b4d1..120bfe4 100644 --- a/src/thumbnailer_server.lua +++ b/src/thumbnailer_server.lua @@ -173,7 +173,7 @@ function do_worker_job(state_json_string, frames_json_string) local file_duration = mp.get_property_native("duration") local file_path = thumb_state.worker_input_path - if thumb_state.is_remote and thumb_state.storyboard_url == nil then + if thumb_state.is_remote and thumb_state.storyboard == nil then if (thumbnail_func == create_thumbnail_ffmpeg) then msg.warn("Thumbnailing remote path, falling back on mpv.") end @@ -212,13 +212,16 @@ function do_worker_job(state_json_string, frames_json_string) if need_thumbnail_generation then local success - if thumb_state.storyboard_url ~= nil then + if thumb_state.storyboard ~= nil then -- get atlas and then split it into thumbnails - local rows = 5 - local cols = 5 + local rows = thumb_state.storyboard.rows + local cols = thumb_state.storyboard.cols local atlas_idx = math.floor(thumb_idx/(cols*rows)) local atlas_path = thumb_state.thumbnail_template:format(atlas_idx) .. ".atlas" - local url = thumb_state.storyboard_url[atlas_idx+1].url + local url = thumb_state.storyboard.fragments[atlas_idx+1].url + if url == nil then + url = thumb_state.storyboard.fragment_base_url .. "/" .. thumb_state.storyboard.fragments[atlas_idx+1].path + end local ret = thumbnail_func(url, 0, thumb_state.thumbnail_size, atlas_path, { no_scale=true }) success = check_output(ret, atlas_path, thumbnail_func == create_thumbnail_mpv) if success then diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index 0e034e2..df4a48a 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -26,7 +26,7 @@ local Thumbnailer = { worker_extra = {}, -- Storyboard urls - storyboard_url = nil, + storyboard = nil, }, -- Set in register_client worker_register_timeout = nil, @@ -42,7 +42,7 @@ function Thumbnailer:clear_state() self.state.finished_thumbnails = 0 self.state.thumbnails = {} self.state.worker_extra = {} - self.state.storyboard_url = nil + self.state.storyboard = nil end @@ -91,13 +91,21 @@ function Thumbnailer:check_storyboard_async(callback) mp.command_native_async({name="subprocess", args=sb_cmd, capture_stdout=true}, function(success, sb_json) if success and sb_json.status == 0 then local sb = utils.parse_json(sb_json.stdout) - if sb ~= nil and sb.duration and sb.width and sb.height and #sb.fragments > 1 then - self.state.storyboard_url = sb.fragments + if sb ~= nil and sb.duration and sb.width and sb.height and sb.fragments and #sb.fragments > 0 then + self.state.storyboard = {} + self.state.storyboard.fragments = sb.fragments + self.state.storyboard.fragment_base_url = sb.fragment_base_url + self.state.storyboard.rows = sb.rows or 5 + self.state.storyboard.cols = sb.columns or 5 self.state.thumbnail_size = {w=sb.width, h=sb.height} - -- estimate the count of thumbnails - -- assume 5x5 atlas (sb0) - self.state.thumbnail_delta = sb.fragments[1].duration / (5*5) -- first atlas is always full - self.state.thumbnail_count = math.floor(sb.duration / self.state.thumbnail_delta) + if sb.fps then + self.state.thumbnail_count = math.floor(sb.fps * sb.duration) + self.state.thumbnail_delta = sb.duration / self.state.thumbnail_count + else + -- estimate the count of thumbnails + self.state.thumbnail_delta = sb.fragments[1].duration / (self.state.storyboard.rows*self.state.storyboard.cols) -- first atlas is always full + self.state.thumbnail_count = math.floor(sb.duration / self.state.thumbnail_delta) + end -- Prefill individual thumbnail states self.state.thumbnails = {} for i = 1, self.state.thumbnail_count do From dc670f73a73306cd8ebfe0c22a7b2d8564605f54 Mon Sep 17 00:00:00 2001 From: ftk Date: Tue, 12 Jul 2022 16:59:34 +0300 Subject: [PATCH 2/9] storyboards: limit thumbnails count --- src/options.lua | 2 ++ src/thumbnailer_server.lua | 32 ++++++++++++++++++++------------ src/thumbnailer_shared.lua | 13 ++++++++++++- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/options.lua b/src/options.lua index 5e6dae1..22f4f39 100644 --- a/src/options.lua +++ b/src/options.lua @@ -121,6 +121,8 @@ local thumbnailer_options = { -- Enable storyboards (requires yt-dlp in PATH). Currently only supports YouTube storyboard_enable = true, + -- Max thumbnails for storyboards. It only skips processing some of the downloaded thumbnails and doesn't make it much faster + storyboard_max_thumbnail_count = 800, } read_options(thumbnailer_options, SCRIPT_NAME) diff --git a/src/thumbnailer_server.lua b/src/thumbnailer_server.lua index 120bfe4..c878e2f 100644 --- a/src/thumbnailer_server.lua +++ b/src/thumbnailer_server.lua @@ -125,24 +125,26 @@ function check_output(ret, output_path, is_mpv) end -- split cols x N atlas in BGRA format into many thumbnail files -function split_atlas(atlas_path, cols, thumbnail_size, template, offset) +function split_atlas(atlas_path, cols, thumbnail_size, output_name) local atlas = io.open(atlas_path, "rb") local atlas_filesize = atlas:seek("end") local atlas_pictures = math.floor(atlas_filesize / (4 * thumbnail_size.w * thumbnail_size.h)) + local stride = 4 * thumbnail_size.w * math.min(cols, atlas_pictures) for pic = 0, atlas_pictures-1 do - local idx = offset + pic local x_start = (pic % cols) * thumbnail_size.w local y_start = math.floor(pic / cols) * thumbnail_size.h - local thumb_file = io.open(template:format(idx), "wb") - local stride = 4 * thumbnail_size.w * math.min(cols, atlas_pictures) - for line = 0, thumbnail_size.h - 1 do - atlas:seek("set", 4 * x_start + (y_start + line) * stride) - local data = atlas:read(thumbnail_size.w * 4) - if data ~= nil then - thumb_file:write(data) + local filename = output_name(pic) + if filename ~= nil then + local thumb_file = io.open(filename, "wb") + for line = 0, thumbnail_size.h - 1 do + atlas:seek("set", 4 * x_start + (y_start + line) * stride) + local data = atlas:read(thumbnail_size.w * 4) + if data ~= nil then + thumb_file:write(data) + end end + thumb_file:close() end - thumb_file:close() end atlas:close() end @@ -216,7 +218,8 @@ function do_worker_job(state_json_string, frames_json_string) -- get atlas and then split it into thumbnails local rows = thumb_state.storyboard.rows local cols = thumb_state.storyboard.cols - local atlas_idx = math.floor(thumb_idx/(cols*rows)) + local div = thumb_state.storyboard.divisor + local atlas_idx = math.floor(thumb_idx * div /(cols*rows)) local atlas_path = thumb_state.thumbnail_template:format(atlas_idx) .. ".atlas" local url = thumb_state.storyboard.fragments[atlas_idx+1].url if url == nil then @@ -225,7 +228,12 @@ function do_worker_job(state_json_string, frames_json_string) local ret = thumbnail_func(url, 0, thumb_state.thumbnail_size, atlas_path, { no_scale=true }) success = check_output(ret, atlas_path, thumbnail_func == create_thumbnail_mpv) if success then - split_atlas(atlas_path, cols, thumb_state.thumbnail_size, thumb_state.thumbnail_template, atlas_idx * cols * rows) + split_atlas(atlas_path, cols, thumb_state.thumbnail_size, function(idx) + if (atlas_idx * cols * rows + idx) % div ~= 0 then + return nil + end + return thumb_state.thumbnail_template:format(math.floor((atlas_idx * cols * rows + idx) / div)) + end) os.remove(atlas_path) end else diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index df4a48a..d187533 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -100,12 +100,23 @@ function Thumbnailer:check_storyboard_async(callback) self.state.thumbnail_size = {w=sb.width, h=sb.height} if sb.fps then self.state.thumbnail_count = math.floor(sb.fps * sb.duration) - self.state.thumbnail_delta = sb.duration / self.state.thumbnail_count else -- estimate the count of thumbnails self.state.thumbnail_delta = sb.fragments[1].duration / (self.state.storyboard.rows*self.state.storyboard.cols) -- first atlas is always full self.state.thumbnail_count = math.floor(sb.duration / self.state.thumbnail_delta) end + + local divisor = 1 -- only save every n-th thumbnail + if thumbnailer_options.storyboard_max_thumbnail_count then + while self.state.thumbnail_count / divisor > thumbnailer_options.storyboard_max_thumbnail_count do + divisor = divisor + 1 + end + end + self.state.storyboard.divisor = divisor + self.state.thumbnail_count = math.floor(self.state.thumbnail_count / divisor) + self.state.thumbnail_delta = sb.duration / self.state.thumbnail_count + + -- Prefill individual thumbnail states self.state.thumbnails = {} for i = 1, self.state.thumbnail_count do From de69e1d0dfdabc8b24b327bd91ba07f32be4f6c2 Mon Sep 17 00:00:00 2001 From: ftk Date: Wed, 13 Jul 2022 12:53:18 +0300 Subject: [PATCH 3/9] use XDG cache directory on unix --- src/options.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.lua b/src/options.lua index 22f4f39..7b200e2 100644 --- a/src/options.lua +++ b/src/options.lua @@ -1,6 +1,6 @@ local SCRIPT_NAME = "mpv_thumbnail_script" -local default_cache_base = ON_WINDOWS and os.getenv("TEMP") or "/tmp/" +local default_cache_base = ON_WINDOWS and os.getenv("TEMP") or (os.getenv("XDG_CACHE_HOME") or "/tmp/") local thumbnailer_options = { -- The thumbnail directory From e1a6e8435665cb1dbd1c24e8c458e17d33239ff0 Mon Sep 17 00:00:00 2001 From: ftk Date: Thu, 14 Jul 2022 15:54:35 +0300 Subject: [PATCH 4/9] simplify code, add youtube workaround --- src/thumbnailer_shared.lua | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index d187533..1cf4003 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -99,18 +99,21 @@ function Thumbnailer:check_storyboard_async(callback) self.state.storyboard.cols = sb.columns or 5 self.state.thumbnail_size = {w=sb.width, h=sb.height} if sb.fps then - self.state.thumbnail_count = math.floor(sb.fps * sb.duration) + self.state.thumbnail_count = math.floor(sb.fps * sb.duration + 0.5) -- round + -- hack: youtube always adds 1 black frame at the end... + if sb.extractor == "youtube" then + self.state.thumbnail_count = self.state.thumbnail_count - 1 + end else -- estimate the count of thumbnails - self.state.thumbnail_delta = sb.fragments[1].duration / (self.state.storyboard.rows*self.state.storyboard.cols) -- first atlas is always full + -- assume first atlas is always full + self.state.thumbnail_delta = sb.fragments[1].duration / (self.state.storyboard.rows*self.state.storyboard.cols) self.state.thumbnail_count = math.floor(sb.duration / self.state.thumbnail_delta) end local divisor = 1 -- only save every n-th thumbnail if thumbnailer_options.storyboard_max_thumbnail_count then - while self.state.thumbnail_count / divisor > thumbnailer_options.storyboard_max_thumbnail_count do - divisor = divisor + 1 - end + divisor = math.ceil(self.state.thumbnail_count / thumbnailer_options.storyboard_max_thumbnail_count) end self.state.storyboard.divisor = divisor self.state.thumbnail_count = math.floor(self.state.thumbnail_count / divisor) From 381733895f810c102bc0adb8f65f6ba002bebd93 Mon Sep 17 00:00:00 2001 From: ftk Date: Sat, 16 Jul 2022 22:52:47 +0300 Subject: [PATCH 5/9] fix youtube playlist urls --- src/thumbnailer_shared.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index 1cf4003..fda63d7 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -84,7 +84,7 @@ end function Thumbnailer:check_storyboard_async(callback) if thumbnailer_options.storyboard_enable and self.state.is_remote then msg.info("Trying to get storyboard info...") - local sb_cmd = {"yt-dlp", "--format", "sb0", "--dump-json", + local sb_cmd = {"yt-dlp", "--format", "sb0", "--dump-json", "--no-playlist", "--extractor-args", "youtube:skip=hls,dash,translated_subs", -- yt speedup "--", mp.get_property_native("path")} From 80684ae893a6dbea9119197531d6f45fff40518d Mon Sep 17 00:00:00 2001 From: ftk Date: Sun, 17 Jul 2022 00:09:49 +0300 Subject: [PATCH 6/9] fix #24 --- src/thumbnailer_shared.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index fda63d7..92e347e 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -63,7 +63,7 @@ function Thumbnailer:on_thumb_ready(index) end function Thumbnailer:on_thumb_progress(index) - self.state.thumbnails[index] = math.max(self.state.thumbnails[index], 0) + self.state.thumbnails[index] = self.state.thumbnails[index] == 1 end function Thumbnailer:on_start_file() From 9a52b9650f00e74df8a9b856a18c50b2adb52706 Mon Sep 17 00:00:00 2001 From: ftk Date: Sun, 17 Jul 2022 00:10:08 +0300 Subject: [PATCH 7/9] dont call yt-dlp twice --- src/thumbnailer_shared.lua | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index 92e347e..489664a 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -76,6 +76,17 @@ function Thumbnailer:on_video_change(params) if params ~= nil then if not self.state.ready then self:update_state() + self:check_storyboard_async(function() + local duration = mp.get_property_native("duration") + local max_duration = thumbnailer_options.autogenerate_max_duration + + if duration ~= nil and self.state.available and thumbnailer_options.autogenerate then + -- Notify if autogenerate is on and video is not too long + if duration < max_duration or max_duration == 0 then + self:start_worker_jobs() + end + end + end) end end end @@ -336,18 +347,7 @@ function Thumbnailer:register_client() -- Notify workers to generate thumbnails when video loads/changes mp.observe_property("video-dec-params", "native", function(name, params) - Thumbnailer:on_video_change(params) - self:check_storyboard_async(function() - local duration = mp.get_property_native("duration") - local max_duration = thumbnailer_options.autogenerate_max_duration - - if duration ~= nil and self.state.available and thumbnailer_options.autogenerate then - -- Notify if autogenerate is on and video is not too long - if duration < max_duration or max_duration == 0 then - self:start_worker_jobs() - end - end - end) + self:on_video_change(params) end) local thumb_script_key = not thumbnailer_options.disable_keybinds and "T" or nil From 424344bafe0c18589c4e257caad4bb099d537fc1 Mon Sep 17 00:00:00 2001 From: ftk Date: Mon, 18 Jul 2022 18:30:37 +0300 Subject: [PATCH 8/9] boolean fix --- src/thumbnailer_shared.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index 489664a..5c8a333 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -63,7 +63,7 @@ function Thumbnailer:on_thumb_ready(index) end function Thumbnailer:on_thumb_progress(index) - self.state.thumbnails[index] = self.state.thumbnails[index] == 1 + self.state.thumbnails[index] = (self.state.thumbnails[index] == 1) and 1 or 0 end function Thumbnailer:on_start_file() From c3404967354ba27887393ce7c72ed7ea3576b0fc Mon Sep 17 00:00:00 2001 From: ftk Date: Wed, 27 Jul 2022 14:02:30 +0300 Subject: [PATCH 9/9] allow upscaling storyboards --- src/options.lua | 2 ++ src/thumbnailer_server.lua | 16 +++++++++++----- src/thumbnailer_shared.lua | 12 +++++++++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/options.lua b/src/options.lua index 7b200e2..b340316 100644 --- a/src/options.lua +++ b/src/options.lua @@ -123,6 +123,8 @@ local thumbnailer_options = { storyboard_enable = true, -- Max thumbnails for storyboards. It only skips processing some of the downloaded thumbnails and doesn't make it much faster storyboard_max_thumbnail_count = 800, + -- Most storyboard thumbnails are 160x90. Enabling this allows upscaling them up to thumbnail_height + storyboard_upscale = false, } read_options(thumbnailer_options, SCRIPT_NAME) diff --git a/src/thumbnailer_server.lua b/src/thumbnailer_server.lua index c878e2f..45fca41 100644 --- a/src/thumbnailer_server.lua +++ b/src/thumbnailer_server.lua @@ -55,7 +55,9 @@ function create_thumbnail_mpv(file_path, timestamp, size, output_path, options) -- Optionally disable subtitles (thumbnailer_options.mpv_no_sub and "--no-sub" or nil), - (options.no_scale == nil and ("--vf=scale=%d:%d"):format(size.w, size.h) or nil), + (options.relative_scale == nil + and ("--vf=scale=%d:%d"):format(size.w, size.h) + or ("--vf=scale=iw*%d:ih*%d"):format(size.w, size.h)), "--vf-add=format=bgra", "--of=rawvideo", @@ -69,7 +71,7 @@ end function create_thumbnail_ffmpeg(file_path, timestamp, size, output_path, options) options = options or {} - local ffmpeg_command = skip_nil({ + local ffmpeg_command = { "ffmpeg", "-loglevel", "quiet", "-noaccurate_seek", @@ -79,13 +81,17 @@ function create_thumbnail_ffmpeg(file_path, timestamp, size, output_path, option "-frames:v", "1", "-an", - (options.no_scale == nil and "-vf" or nil), (options.no_scale == nil and ("scale=%d:%d"):format(size.w, size.h) or nil), + "-vf", + (options.relative_scale == nil + and ("scale=%d:%d"):format(size.w, size.h) + or ("scale=iw*%d:ih*%d"):format(size.w, size.h)), + "-c:v", "rawvideo", "-pix_fmt", "bgra", "-f", "rawvideo", "-y", output_path - }) + } return utils.subprocess({args=ffmpeg_command}) end @@ -225,7 +231,7 @@ function do_worker_job(state_json_string, frames_json_string) if url == nil then url = thumb_state.storyboard.fragment_base_url .. "/" .. thumb_state.storyboard.fragments[atlas_idx+1].path end - local ret = thumbnail_func(url, 0, thumb_state.thumbnail_size, atlas_path, { no_scale=true }) + local ret = thumbnail_func(url, 0, { w=thumb_state.storyboard.scale, h=thumb_state.storyboard.scale }, atlas_path, { relative_scale=true }) success = check_output(ret, atlas_path, thumbnail_func == create_thumbnail_mpv) if success then split_atlas(atlas_path, cols, thumb_state.thumbnail_size, function(idx) diff --git a/src/thumbnailer_shared.lua b/src/thumbnailer_shared.lua index 5c8a333..646f71b 100644 --- a/src/thumbnailer_shared.lua +++ b/src/thumbnailer_shared.lua @@ -108,7 +108,7 @@ function Thumbnailer:check_storyboard_async(callback) self.state.storyboard.fragment_base_url = sb.fragment_base_url self.state.storyboard.rows = sb.rows or 5 self.state.storyboard.cols = sb.columns or 5 - self.state.thumbnail_size = {w=sb.width, h=sb.height} + if sb.fps then self.state.thumbnail_count = math.floor(sb.fps * sb.duration + 0.5) -- round -- hack: youtube always adds 1 black frame at the end... @@ -122,6 +122,16 @@ function Thumbnailer:check_storyboard_async(callback) self.state.thumbnail_count = math.floor(sb.duration / self.state.thumbnail_delta) end + -- Storyboard upscaling factor + local scale = 1 + if thumbnailer_options.storyboard_upscale then + -- BUG: sometimes mpv crashes when asked for non-integer scaling and BGRA format (something related to zimg?) + -- use integer scaling for now + scale = math.max(1, math.floor(thumbnailer_options.thumbnail_height / sb.height)) + end + self.state.thumbnail_size = {w=sb.width*scale, h=sb.height*scale} + self.state.storyboard.scale = scale + local divisor = 1 -- only save every n-th thumbnail if thumbnailer_options.storyboard_max_thumbnail_count then divisor = math.ceil(self.state.thumbnail_count / thumbnailer_options.storyboard_max_thumbnail_count)