chore(DownloadCaptions): set a legit user agent & relax job

CaptainFact · Feb 24, 2025 · 44a1dea · 44a1dea
1 parent dc253a0
commit 44a1dea
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 9 deletions.
diff --git a/apps/cf/lib/videos/captions_fetcher_youtube.ex b/apps/cf/lib/videos/captions_fetcher_youtube.ex
@@ -9,6 +9,8 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   require Logger
   import SweetXml
 
+  @user_agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/135.0"
+
   @impl true
   def fetch(%{youtube_id: youtube_id, language: language}) do
     with {:ok, data} <- fetch_youtube_data(youtube_id),
@@ -27,7 +29,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   defp fetch_youtube_data(video_id) do
     url = "https://www.youtube.com/watch?v=#{video_id}"
 
-    case HTTPoison.get(url, []) do
+    case HTTPoison.get(url, [{"User-Agent", @user_agent}]) do
       {:ok, %HTTPoison.Response{body: body}} ->
         {:ok, body}
 
@@ -59,7 +61,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   end
 
   defp fetch_transcript(base_url) do
-    case HTTPoison.get(base_url, []) do
+    case HTTPoison.get(base_url, [{"User-Agent", @user_agent}]) do
       {:ok, %HTTPoison.Response{body: body}} ->
         {:ok, body}
 

diff --git a/apps/cf_jobs/config/config.exs b/apps/cf_jobs/config/config.exs
@@ -38,14 +38,14 @@ config :cf_jobs, CF.Jobs.Scheduler,
       schedule: {:extended, "*/5"},
       task: {CF.Jobs.CreateNotifications, :update, []},
       overlap: false
-    ]
+    ],
     # Captions
-    # download_captions: [
-    #   # every minute
-    #   schedule: "*/1 * * * *",
-    #   task: {CF.Jobs.DownloadCaptions, :update, []},
-    #   overlap: false
-    # ]
+    download_captions: [
+      # every 8h
+      schedule: "0 */8 * * *",
+      task: {CF.Jobs.DownloadCaptions, :update, []},
+      overlap: false
+    ]
   ]
 
 # Configure Postgres pool size

diff --git a/apps/cf_jobs/lib/jobs/download_captions.ex b/apps/cf_jobs/lib/jobs/download_captions.ex
@@ -36,6 +36,7 @@ defmodule CF.Jobs.DownloadCaptions do
     |> Enum.map(fn video ->
       Logger.info("Downloading captions for video #{video.id}")
       CF.Videos.download_captions(video)
+      Process.sleep(1000)
     end)
 
     {:reply, :ok, :ok}