Make --archive save the links to the downloaded songs as soon as the songs are downloaded. Closes #2196

spotdl/download/downloader.py

@@ -232,6 +232,9 @@ def __init__(
         # Initialize archive
         self.url_archive = Archive()
         if self.settings["archive"]:
+            # Ensure the archive file is created if it doesn't exist
+            self.url_archive.initialize(self.settings["archive"])
+            # Attempt to load the existing URLs from the archive
             self.url_archive.load(self.settings["archive"])
 
         logger.debug("Archive: %d urls", len(self.url_archive))
@@ -273,7 +276,6 @@ def download_multiple_songs(
             logger.info(
                 "Fetching %d album%s", len(albums), "s" if len(albums) > 1 else ""
             )
-
             songs.extend(songs_from_albums(list(albums)))
 
             # Remove duplicates
@@ -285,48 +287,49 @@ def download_multiple_songs(
 
         logger.debug("Downloading %d songs", len(songs))
 
+        # Filter out already downloaded songs
         if self.settings["archive"]:
             songs = [song for song in songs if song.url not in self.url_archive]
             logger.debug("Filtered %d songs with archive", len(songs))
 
         self.progress_handler.set_song_count(len(songs))
 
-        # Create tasks list
-        tasks = [self.pool_download(song) for song in songs]
-
-        # Call all task asynchronously, and wait until all are finished
-        results = list(self.loop.run_until_complete(asyncio.gather(*tasks)))
+        results = []
+        try:
+            # Create tasks list
+            tasks = [self.pool_download(song) for song in songs]
 
-        # Print errors
-        if self.settings["print_errors"]:
-            for error in self.errors:
-                logger.error(error)
+            # Call all tasks asynchronously, and wait until all are finished
+            results = list(self.loop.run_until_complete(asyncio.gather(*tasks)))
 
-        if self.settings["save_errors"]:
-            with open(
-                self.settings["save_errors"], "a", encoding="utf-8"
-            ) as error_file:
-                if len(self.errors) > 0:
-                    error_file.write(
-                        f"{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}\n"
-                    )
+            # Print errors
+            if self.settings["print_errors"]:
                 for error in self.errors:
-                    error_file.write(f"{error}\n")
-
-            logger.info("Saved errors to %s", self.settings["save_errors"])
-
-        # Save archive
-        if self.settings["archive"]:
-            for result in results:
-                if result[1] or self.settings["add_unavailable"]:
-                    self.url_archive.add(result[0].url)
+                    logger.error(error)
+
+            if self.settings["save_errors"]:
+                with open(
+                    self.settings["save_errors"], "w", encoding="utf-8"
+                ) as error_file:
+                    error_file.write("\n".join(self.errors))
+                logger.info("Saved errors to %s", self.settings["save_errors"])
+
+        except Exception as e:
+            logger.error(f"An error occurred: {str(e)}")
+
+        finally:
+            # Save archive incrementally after each successful download
+            if self.settings["archive"]:
+                for result in results:
+                    # Check if the download was successful
+                    if result[1] or self.settings["add_unavailable"]:
+                        # Add the URL to the archive and write to the file
+                        self.url_archive.add(result[0].url)
+                        self.url_archive.add_entry(
+                            self.settings["archive"], result[0].url
+                        )
 
-            self.url_archive.save(self.settings["archive"])
-            logger.info(
-                "Saved archive with %d urls to %s",
-                len(self.url_archive),
-                self.settings["archive"],
-            )
+                logger.info("Archive saved with %d URLs", len(self.url_archive))
 
         # Create m3u playlist
         if self.settings["m3u"]:
@@ -335,7 +338,6 @@ def download_multiple_songs(
                 for song, path in results
                 if path or self.settings["add_unavailable"]
             ]
-
             gen_m3u_files(
                 song_list,
                 self.settings["m3u"],
@@ -350,7 +352,6 @@ def download_multiple_songs(
         if self.settings["save_file"]:
             with open(self.settings["save_file"], "w", encoding="utf-8") as save_file:
                 json.dump([song.json for song, _ in results], save_file, indent=4)
-
             logger.info("Saved results to %s", self.settings["save_file"])
 
         return results

spotdl/utils/archive.py

@@ -47,3 +47,26 @@ def save(self, file: str) -> bool:
                 archive.write(f"{element}\n")
 
         return True
+
+    def initialize(self, file: str) -> None:
+        """
+        Create the archive file if it doesn't exist.
+
+        ### Arguments
+        - file: the file name of the archive
+        """
+        if not Path(file).exists():
+            with open(file, "w", encoding="utf-8") as archive:
+                archive.write("")
+
+    def add_entry(self, file: str, url: str) -> None:
+        """
+        Adds an entry to the archive file and flushes it immediately.
+
+        ### Arguments
+        - file: the file name of the archive
+        - url: the URL to append to the archive file
+        """
+        with open(file, "a", encoding="utf-8") as archive:
+            archive.write(f"{url}\n")
+            archive.flush()