refactor: fix TMDb image relations

Fixed the TMDb image relations to support a proper many-to-many mapping between entities of different types and images. The previous implementation was flawed in that it assumed only a single entity of a given type would be linked to the same image at all times. That is wrong. I'll assume they're using some sort of hashing on their side to automagically de-duplicate exact images between entities, leading to images being re-used between entities at semi-random, but that's just an estimated guess, and I don't actually know how their internals work.

Shoko.Server/API/v3/Controllers/ActionController.cs

@@ -1,17 +1,11 @@
-using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Quartz;
 using Shoko.Server.API.Annotations;
 using Shoko.Server.API.v3.Models.Shoko;
-using Shoko.Server.Providers.AniDB;
-using Shoko.Server.Providers.AniDB.Interfaces;
-using Shoko.Server.Providers.AniDB.UDP.Info;
 using Shoko.Server.Providers.TMDB;
 using Shoko.Server.Providers.TraktTV;
 using Shoko.Server.Repositories;
@@ -38,23 +32,31 @@ public class ActionController : BaseController
     private readonly ActionService _actionService;
     private readonly AnimeGroupService _groupService;
     private readonly TraktTVHelper _traktHelper;
-    private readonly TmdbMetadataService _tmdbService;
+    private readonly TmdbMetadataService _tmdbMetadataService;
     private readonly TmdbLinkingService _tmdbLinkingService;
+    private readonly TmdbImageService _tmdbImageService;
     private readonly ISchedulerFactory _schedulerFactory;
-    private readonly IRequestFactory _requestFactory;
-    private readonly AnimeSeriesService _seriesService;
 
-    public ActionController(ILogger<ActionController> logger, TraktTVHelper traktHelper, TmdbMetadataService tmdbService, TmdbLinkingService tmdbLinkingService, ISchedulerFactory schedulerFactory,
-        IRequestFactory requestFactory, ISettingsProvider settingsProvider, ActionService actionService, AnimeSeriesService seriesService, AnimeGroupCreator groupCreator, AnimeGroupService groupService) : base(settingsProvider)
+    public ActionController(
+        ILogger<ActionController> logger,
+        TraktTVHelper traktHelper,
+        TmdbMetadataService tmdbMetadataService,
+        TmdbLinkingService tmdbLinkingService,
+        TmdbImageService tmdbImageService,
+        ISchedulerFactory schedulerFactory,
+        ISettingsProvider settingsProvider,
+        ActionService actionService,
+        AnimeGroupCreator groupCreator,
+        AnimeGroupService groupService
+    ) : base(settingsProvider)
     {
         _logger = logger;
         _traktHelper = traktHelper;
-        _tmdbService = tmdbService;
+        _tmdbMetadataService = tmdbMetadataService;
         _tmdbLinkingService = tmdbLinkingService;
+        _tmdbImageService = tmdbImageService;
         _schedulerFactory = schedulerFactory;
-        _requestFactory = requestFactory;
         _actionService = actionService;
-        _seriesService = seriesService;
         _groupCreator = groupCreator;
         _groupService = groupService;
     }
@@ -155,7 +157,7 @@ public ActionResult UpdateAllImages()
     [HttpGet("SearchForTmdbMatches")]
     public ActionResult SearchForTmdbMatches()
     {
-        Task.Factory.StartNew(() => _tmdbService.ScanForMatches());
+        Task.Factory.StartNew(() => _tmdbMetadataService.ScanForMatches());
         return Ok();
     }
 
@@ -166,7 +168,7 @@ public ActionResult SearchForTmdbMatches()
     [HttpGet("UpdateAllTmdbMovies")]
     public ActionResult UpdateAllTmdbMovies()
     {
-        Task.Factory.StartNew(() => _tmdbService.UpdateAllMovies(true, true));
+        Task.Factory.StartNew(() => _tmdbMetadataService.UpdateAllMovies(true, true));
         return Ok();
     }
 
@@ -178,7 +180,7 @@ public ActionResult UpdateAllTmdbMovies()
     [HttpGet("PurgeAllUnusedTmdbMovies")]
     public ActionResult PurgeAllUnusedTmdbMovies()
     {
-        Task.Factory.StartNew(() => _tmdbService.PurgeAllUnusedMovies());
+        Task.Factory.StartNew(() => _tmdbMetadataService.PurgeAllUnusedMovies());
         return Ok();
     }
 
@@ -190,7 +192,7 @@ public ActionResult PurgeAllUnusedTmdbMovies()
     [HttpGet("PurgeAllTmdbMovieCollections")]
     public ActionResult PurgeAllTmdbMovieCollections()
     {
-        Task.Factory.StartNew(() => _tmdbService.PurgeAllMovieCollections());
+        Task.Factory.StartNew(() => _tmdbMetadataService.PurgeAllMovieCollections());
         return Ok();
     }
 
@@ -201,7 +203,7 @@ public ActionResult PurgeAllTmdbMovieCollections()
     [HttpGet("UpdateAllTmdbShows")]
     public ActionResult UpdateAllTmdbShows()
     {
-        Task.Factory.StartNew(() => _tmdbService.UpdateAllShows(true, true));
+        Task.Factory.StartNew(() => _tmdbMetadataService.UpdateAllShows(true, true));
         return Ok();
     }
 
@@ -211,7 +213,19 @@ public ActionResult UpdateAllTmdbShows()
     [HttpGet("DownloadMissingTmdbPeople")]
     public ActionResult DownloadMissingTmdbPeople()
     {
-        Task.Factory.StartNew(() => _tmdbService.RepairMissingPeople());
+        Task.Factory.StartNew(() => _tmdbMetadataService.RepairMissingPeople());
+        return Ok();
+    }
+
+    /// <summary>
+    /// Purge all unused TMDB Images that are not linked to anything.
+    /// </summary>
+    /// <returns></returns>
+    [Authorize("admin")]
+    [HttpGet("PurgeAllUnusedTmdbImages")]
+    public ActionResult PurgeAllUnusedTmdbImages()
+    {
+        Task.Factory.StartNew(() => _tmdbImageService.PurgeAllUnusedImages());
         return Ok();
     }
 
@@ -223,7 +237,7 @@ public ActionResult DownloadMissingTmdbPeople()
     [HttpGet("PurgeAllUnusedTmdbShows")]
     public ActionResult PurgeAllUnusedTmdbShows()
     {
-        Task.Factory.StartNew(() => _tmdbService.PurgeAllUnusedShows());
+        Task.Factory.StartNew(() => _tmdbMetadataService.PurgeAllUnusedShows());
         return Ok();
     }
 
@@ -235,7 +249,7 @@ public ActionResult PurgeAllUnusedTmdbShows()
     [HttpGet("PurgeAllTmdbShowAlternateOrderings")]
     public ActionResult PurgeAllTmdbShowAlternateOrderings()
     {
-        Task.Factory.StartNew(() => _tmdbService.PurgeAllShowEpisodeGroups());
+        Task.Factory.StartNew(() => _tmdbMetadataService.PurgeAllShowEpisodeGroups());
         return Ok();
     }
 
@@ -250,12 +264,13 @@ public ActionResult PurgeAllTmdbShowAlternateOrderings()
     [HttpGet("PurgeAllTmdbLinks")]
     public ActionResult PurgeAllTmdbLinks([FromQuery] bool removeShowLinks = true, [FromQuery] bool removeMovieLinks = true, [FromQuery] bool? resetAutoLinkingState = null)
     {
-        if (removeShowLinks || removeMovieLinks)
-            Task.Factory.StartNew(() => _tmdbLinkingService.RemoveAllLinks(removeShowLinks, removeMovieLinks));
-
-        if (resetAutoLinkingState.HasValue)
-            Task.Factory.StartNew(() => _tmdbLinkingService.ResetAutoLinkingState(resetAutoLinkingState.Value));
-
+        Task.Run(() =>
+        {
+            if (removeShowLinks || removeMovieLinks)
+                _tmdbLinkingService.RemoveAllLinks(removeShowLinks, removeMovieLinks);
+            if (resetAutoLinkingState.HasValue)
+                _tmdbLinkingService.ResetAutoLinkingState(resetAutoLinkingState.Value);
+        });
         return Ok();
     }
 

Shoko.Server/API/v3/Controllers/TmdbController.cs

@@ -196,16 +196,12 @@ public ActionResult<TmdbMovie> GetTmdbMovieByMovieID(
     /// Remove the local copy of the metadata for a TMDB movie.
     /// </summary>
     /// <param name="movieID">TMDB Movie ID.</param>
-    /// <param name="removeImageFiles">Also remove images related to the show.</param>
     /// <returns></returns>
     [Authorize("admin")]
     [HttpDelete("Movie/{movieID}")]
-    public async Task<ActionResult> RemoveTmdbMovieByMovieID(
-        [FromRoute] int movieID,
-        [FromQuery] bool removeImageFiles = true
-    )
+    public async Task<ActionResult> RemoveTmdbMovieByMovieID([FromRoute] int movieID)
     {
-        await _tmdbMetadataService.SchedulePurgeOfMovie(movieID, removeImageFiles);
+        await _tmdbMetadataService.SchedulePurgeOfMovie(movieID);
 
         return NoContent();
     }
@@ -1008,16 +1004,12 @@ public ActionResult<TmdbShow> GetTmdbShowByShowID(
     /// Remove the local copy of the metadata for a TMDB show.
     /// </summary>
     /// <param name="showID">TMDB Movie ID.</param>
-    /// <param name="removeImageFiles">Also remove images related to the show.</param>
     /// <returns></returns>
     [Authorize("admin")]
     [HttpDelete("Show/{showID}")]
-    public async Task<ActionResult> RemoveTmdbShowByShowID(
-        [FromRoute] int showID,
-        [FromQuery] bool removeImageFiles = true
-    )
+    public async Task<ActionResult> RemoveTmdbShowByShowID([FromRoute] int showID)
     {
-        await _tmdbMetadataService.SchedulePurgeOfShow(showID, removeImageFiles);
+        await _tmdbMetadataService.SchedulePurgeOfShow(showID);
 
         return NoContent();
     }

Shoko.Server/Databases/DatabaseFixes.cs

@@ -853,4 +853,46 @@ public static void RecreateAnimeCharactersAndCreators()
 
         _logger.Info($"Done recreating characters and creator relations for {animeList.Count} anidb anime entries.");
     }
+
+    public static void ScheduleTmdbImageUpdates()
+    {
+        var tmdbMetadataService = Utils.ServiceContainer.GetRequiredService<TmdbMetadataService>();
+        var tmdbMovies = RepoFactory.TMDB_Movie.GetAll();
+        var tmdbShows = RepoFactory.TMDB_Show.GetAll();
+        var movies = tmdbMovies.Count;
+        var shows = tmdbShows.Count;
+        var str = ServerState.Instance.ServerStartingStatus;
+        ServerState.Instance.ServerStartingStatus = $"{str} - 0 / {movies} movies - 0 / {shows} shows";
+        _logger.Info($"Scheduling tmdb image updates for {movies} tmdb movies and {shows} tmdb shows...");
+
+        var count = 0;
+        foreach (var tmdbMovie in tmdbMovies)
+        {
+            if (++count % 10 == 0 || count == movies)
+            {
+                _logger.Info($"Scheduling tmdb image updates for tmdb movies... ({count}/{movies})");
+                ServerState.Instance.ServerStartingStatus = $"{str} - {count} / {movies} movies - 0 / {shows} shows";
+            }
+
+            tmdbMetadataService.ScheduleDownloadAllMovieImages(tmdbMovie.Id)
+                .GetAwaiter()
+                .GetResult();
+        }
+
+        count = 0;
+        foreach (var tmdbShow in tmdbShows)
+        {
+            if (++count % 10 == 0 || count == shows)
+            {
+                _logger.Info($"Scheduling tmdb image updates for tmdb shows... ({count}/{shows})");
+                ServerState.Instance.ServerStartingStatus = $"{str} - {movies} / {movies} movies - {count} / {shows} shows";
+            }
+
+            tmdbMetadataService.ScheduleDownloadAllShowImages(tmdbShow.Id)
+                .GetAwaiter()
+                .GetResult();
+        }
+
+        _logger.Info($"Done scheduling tmdb image updates for {movies} tmdb movies and {shows} tmdb shows.");
+    }
 }

Shoko.Server/Databases/MySQL.cs

@@ -27,7 +27,7 @@ namespace Shoko.Server.Databases;
 public class MySQL : BaseDatabase<MySqlConnection>
 {
     public override string Name { get; } = "MySQL";
-    public override int RequiredVersion { get; } = 149;
+    public override int RequiredVersion { get; } = 150;
 
     private List<DatabaseCommand> createVersionTable = new()
     {
@@ -926,6 +926,18 @@ public class MySQL : BaseDatabase<MySqlConnection>
         new(148, 03, DatabaseFixes.RecreateAnimeCharactersAndCreators),
         new(149, 1, MySQLFixUTF8MB4),
         new(149, 2, SetDefaultCollationToUTF8MB4),
+        new(150, 01, "CREATE TABLE `TMDB_Image_Entity` (`TMDB_Image_EntityID` INT NOT NULL AUTO_INCREMENT, `TmdbEntityID` INT NULL, `TmdbEntityType` INT NOT NULL, `ImageType` INT NOT NULL, `RemoteFileName` VARCHAR(128) NOT NULL, `Ordering` INT NOT NULL, `ReleasedAt` DATE NULL, PRIMARY KEY (`TMDB_Image_EntityID`));"),
+        new(150, 02, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbMovieID`;"),
+        new(150, 03, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbEpisodeID`;"),
+        new(150, 04, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbSeasonID`;"),
+        new(150, 05, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbShowID`;"),
+        new(150, 06, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbCollectionID`;"),
+        new(150, 07, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbNetworkID`;"),
+        new(150, 08, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbCompanyID`;"),
+        new(150, 09, "ALTER TABLE `TMDB_Image` DROP COLUMN `TmdbPersonID`;"),
+        new(150, 10, "ALTER TABLE `TMDB_Image` DROP COLUMN `ForeignType`;"),
+        new(150, 11, "ALTER TABLE `TMDB_Image` DROP COLUMN `ImageType`;"),
+        new(150, 12, DatabaseFixes.ScheduleTmdbImageUpdates),
     };
 
     private DatabaseCommand linuxTableVersionsFix = new("RENAME TABLE versions TO Versions;");

Shoko.Server/Databases/SQLServer.cs

@@ -28,7 +28,7 @@ namespace Shoko.Server.Databases;
 public class SQLServer : BaseDatabase<SqlConnection>
 {
     public override string Name { get; } = "SQLServer";
-    public override int RequiredVersion { get; } = 141;
+    public override int RequiredVersion { get; } = 142;
 
     public override void BackupDatabase(string fullfilename)
     {
@@ -878,6 +878,18 @@ public override bool HasVersionsTable()
         new DatabaseCommand(141, 01, "ALTER TABLE AniDB_Character ADD Type int NOT NULL DEFAULT 0;"),
         new DatabaseCommand(141, 02, "ALTER TABLE AniDB_Character ADD LastUpdated datetime2 NOT NULL DEFAULT '1970-01-01 00:00:00';"),
         new DatabaseCommand(141, 03, DatabaseFixes.RecreateAnimeCharactersAndCreators),
+        new DatabaseCommand(142, 01, "CREATE TABLE TMDB_Image_Entity (TMDB_Image_EntityID INT IDENTITY(1,1), TmdbEntityID INT NULL, TmdbEntityType INT NOT NULL, ImageType INT NOT NULL, RemoteFileName NVARCHAR(128) NOT NULL, Ordering INT NOT NULL, ReleasedAt DATE NULL);"),
+        new DatabaseCommand(142, 02, "ALTER TABLE TMDB_Image DROP COLUMN TmdbMovieID;"),
+        new DatabaseCommand(142, 03, "ALTER TABLE TMDB_Image DROP COLUMN TmdbEpisodeID;"),
+        new DatabaseCommand(142, 04, "ALTER TABLE TMDB_Image DROP COLUMN TmdbSeasonID;"),
+        new DatabaseCommand(142, 05, "ALTER TABLE TMDB_Image DROP COLUMN TmdbShowID;"),
+        new DatabaseCommand(142, 06, "ALTER TABLE TMDB_Image DROP COLUMN TmdbCollectionID;"),
+        new DatabaseCommand(142, 07, "ALTER TABLE TMDB_Image DROP COLUMN TmdbNetworkID;"),
+        new DatabaseCommand(142, 08, "ALTER TABLE TMDB_Image DROP COLUMN TmdbCompanyID;"),
+        new DatabaseCommand(142, 09, "ALTER TABLE TMDB_Image DROP COLUMN TmdbPersonID;"),
+        new DatabaseCommand(142, 10, "ALTER TABLE TMDB_Image DROP COLUMN ForeignType;"),
+        new DatabaseCommand(142, 11, "ALTER TABLE TMDB_Image DROP COLUMN ImageType;"),
+        new DatabaseCommand(142, 12, DatabaseFixes.ScheduleTmdbImageUpdates),
     };
 
     private static void AlterImdbMovieIDType()

Shoko.Server/Databases/SQLite.cs

@@ -28,7 +28,7 @@ public class SQLite : BaseDatabase<SqliteConnection>
 {
     public override string Name => "SQLite";
 
-    public override int RequiredVersion => 131;
+    public override int RequiredVersion => 132;
 
     public override void BackupDatabase(string fullfilename)
     {
@@ -845,6 +845,18 @@ public override void CreateDatabase()
         new(131, 01, "ALTER TABLE AniDB_Character ADD COLUMN Type INTEGER NOT NULL DEFAULT 0;"),
         new(131, 02, "ALTER TABLE AniDB_Character ADD COLUMN LastUpdated DATETIME NOT NULL DEFAULT '1970-01-01 00:00:00';"),
         new(131, 03, DatabaseFixes.RecreateAnimeCharactersAndCreators),
+        new(132, 01, "CREATE TABLE TMDB_Image_Entity (TMDB_Image_EntityID INTEGER PRIMARY KEY AUTOINCREMENT, TmdbEntityID INTEGER NULL, TmdbEntityType INTEGER NOT NULL, ImageType INTEGER NOT NULL, RemoteFileName TEXT NOT NULL, Ordering INTEGER NOT NULL, ReleasedAt DATE NULL);"),
+        new(132, 02, "ALTER TABLE TMDB_Image DROP COLUMN TmdbMovieID;"),
+        new(132, 03, "ALTER TABLE TMDB_Image DROP COLUMN TmdbEpisodeID;"),
+        new(132, 04, "ALTER TABLE TMDB_Image DROP COLUMN TmdbSeasonID;"),
+        new(132, 05, "ALTER TABLE TMDB_Image DROP COLUMN TmdbShowID;"),
+        new(132, 06, "ALTER TABLE TMDB_Image DROP COLUMN TmdbCollectionID;"),
+        new(132, 07, "ALTER TABLE TMDB_Image DROP COLUMN TmdbNetworkID;"),
+        new(132, 08, "ALTER TABLE TMDB_Image DROP COLUMN TmdbCompanyID;"),
+        new(132, 09, "ALTER TABLE TMDB_Image DROP COLUMN TmdbPersonID;"),
+        new(132, 10, "ALTER TABLE TMDB_Image DROP COLUMN ForeignType;"),
+        new(132, 11, "ALTER TABLE TMDB_Image DROP COLUMN ImageType;"),
+        new(132, 12, DatabaseFixes.ScheduleTmdbImageUpdates),
     };
 
     private static Tuple<bool, string> MigrateRenamers(object connection)

Shoko.Server/Extensions/ModelClients.cs

@@ -206,7 +206,7 @@ public static MovieDB_Fanart ToClientFanart(this TMDB_Image image)
             ImageSize = "original",
             ImageType = "backdrop",
             ImageWidth = image.Width,
-            MovieId = image.TmdbMovieID ?? 0,
+            MovieId = 0,
             URL = image.RemoteFileName,
         };
 
@@ -220,7 +220,7 @@ public static MovieDB_Poster ToClientPoster(this TMDB_Image image)
             ImageSize = "original",
             ImageType = "poster",
             ImageWidth = image.Width,
-            MovieId = image.TmdbMovieID ?? 0,
+            MovieId = 0,
             URL = image.RemoteFileName,
         };
 

Shoko.Server/Mappings/TMDB/TMDB_ImageMap.cs

@@ -1,8 +1,6 @@
 using FluentNHibernate.Mapping;
-using Shoko.Plugin.Abstractions.Enums;
 using Shoko.Server.Databases.NHibernate;
 using Shoko.Server.Models.TMDB;
-using Shoko.Server.Server;
 
 namespace Shoko.Server.Mappings;
 
@@ -15,17 +13,7 @@ public TMDB_ImageMap()
         Not.LazyLoad();
         Id(x => x.TMDB_ImageID);
 
-        Map(x => x.TmdbMovieID);
-        Map(x => x.TmdbEpisodeID);
-        Map(x => x.TmdbSeasonID);
-        Map(x => x.TmdbShowID);
-        Map(x => x.TmdbCollectionID);
-        Map(x => x.TmdbNetworkID);
-        Map(x => x.TmdbCompanyID);
-        Map(x => x.TmdbPersonID);
         Map(x => x.IsEnabled);
-        Map(x => x.ForeignType).Not.Nullable().CustomType<ForeignEntityType>();
-        Map(x => x.ImageType).Not.Nullable().CustomType<ImageEntityType>();
         Map(x => x.Width).Not.Nullable();
         Map(x => x.Height).Not.Nullable();
         Map(x => x.Language).Not.Nullable().CustomType<TitleLanguageConverter>();