Fix tbl merge bug and generic file merging

- Fixed TBL Merging bug related to expanded TBL files
- Added generic file merging for a bunch of files for P3P, P4G and P5R
- Add testing units for new generic merging

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/ByteResolver.cs

@@ -0,0 +1,13 @@
+using Sewer56.StructuredDiff.Interfaces;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers;
+
+public struct ByteResolver : IEncoderFieldResolver
+{
+    public bool Resolve(nuint offset, out int moveBy, out int length)
+    {
+        moveBy = 0;
+        length = 1;
+        return false;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/GenericPatcher.cs

@@ -0,0 +1,90 @@
+using Persona.Merger.Utilities;
+using Reloaded.Memory.Streams;
+using static Persona.Merger.Patching.Tbl.FieldResolvers.TblPatcherCommon;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers.Generic;
+
+/// <summary>
+/// Utility class for patching P5R TBL files.
+/// </summary>
+public struct GenericPatcher
+{
+    /// <summary>
+    /// The table to patch.
+    /// </summary>
+    public byte[] TblData { get; set; }
+
+    public GenericPatcher(byte[] tblData)
+    {
+        TblData = tblData;
+    }
+
+    /// <summary>
+    /// Generates a table patch.
+    /// </summary>
+    /// <param name="otherTbl">Data of the new table.</param>
+    public unsafe TblPatch GeneratePatchGeneric(byte[] otherTbl, int ResolverSize)
+    {
+        fixed (byte* otherTblData = &otherTbl[0])
+        fixed (byte* tblData = &TblData[0])
+        {
+            var patch = new TblPatch();
+
+            var originalSegments = stackalloc PointerLengthTuple[1]; // using pointer to elide bounds checks below
+            var newSegments = stackalloc PointerLengthTuple[1];
+            PopulateGeneric(tblData, TblData.Length, originalSegments);
+            PopulateGeneric(otherTblData, otherTbl.Length, newSegments);
+
+            if (ResolverSize == 2) DiffSegment(patch, newSegments[0], originalSegments[0], new ShortResolver());
+            else if (ResolverSize == 4) DiffSegment(patch, newSegments[0], originalSegments[0], new IntResolver());
+            else DiffSegment(patch, newSegments[0], originalSegments[0], new ByteResolver());
+
+            return patch;
+        }
+    }
+
+    /// <summary>
+    /// Applies a list of table patches.
+    /// </summary>
+    /// <param name="patches">List of patches to apply.</param>
+    public unsafe byte[] ApplyGeneric(List<TblPatch> patches)
+    {
+        fixed (byte* tblData = &TblData[0])
+        {
+            // Get original segments.
+            var segmentCount = 1;
+            var originalSegments = stackalloc PointerLengthTuple[segmentCount]; // using pointer to elide bounds checks below
+            PopulateGeneric(tblData, segmentCount, originalSegments);
+
+            // Convert original segments into Memory<T>.
+            var segments = ConvertSegmentsToMemoryGeneric(segmentCount, originalSegments, tblData, TblData);
+
+            // Apply Patch(es).
+            for (int x = 0; x < segmentCount; x++)
+                ApplyPatch(patches, x, segments);
+
+            // Produce new file.
+            var fileSize = 0;
+            foreach (var segment in segments)
+                fileSize += segment.Length;
+
+            var result = GC.AllocateUninitializedArray<byte>(fileSize);
+            using var memoryStream = new ExtendedMemoryStream(result, true);
+            foreach (var segment in segments)
+            {
+                // memoryStream.WriteBigEndianPrimitive(segment.Length);
+                memoryStream.Write(segment.Span);
+                // memoryStream.AddPadding(P5RTblSegmentFinder.TblSegmentAlignment);
+            }
+
+            return result;
+        }
+    }
+
+    public unsafe static void PopulateGeneric(byte* tblPointer, int length, PointerLengthTuple* segments)
+    {
+        ref var currentSegment = ref segments[0];
+        currentSegment.Pointer = tblPointer;
+        currentSegment.Length = length;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/IntResolver.cs

@@ -0,0 +1,15 @@
+using Sewer56.StructuredDiff.Interfaces;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers;
+
+public struct IntResolver : IEncoderFieldResolver
+{
+    public bool Resolve(nuint offset, out int moveBy, out int length)
+    {
+        // All data are u32s.
+        var fourByteAligned = offset / 4 * 4;
+        moveBy = (int)(offset - fourByteAligned);
+        length = 4;
+        return true;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/ShortResolver.cs

@@ -0,0 +1,14 @@
+using Sewer56.StructuredDiff.Interfaces;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers;
+
+public struct ShortResolver : IEncoderFieldResolver
+{
+    public bool Resolve(nuint offset, out int moveBy, out int length)
+    {
+        var twoByteAligned = offset / 2 * 2;
+        moveBy = (int)(offset - twoByteAligned);
+        length = 2;
+        return true;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/TblPatcherCommon.cs

@@ -26,15 +26,38 @@ internal static unsafe Memory<byte>[] ConvertSegmentsToMemory(int segmentCount,
 
         return segments;
     }
+    internal static unsafe Memory<byte>[] ConvertSegmentsToMemoryGeneric(int segmentCount, PointerLengthTuple* originalSegments, byte* tblData, byte[] tblDataArr)
+    {
+        var segments = new Memory<byte>[segmentCount];
+        for (int x = 0; x < segmentCount; x++)
+        {
+            ref var originalSegment = ref originalSegments[x];
+            segments[x] = new Memory<byte>(tblDataArr, 0, tblDataArr.Length);
+        }
+
+        return segments;
+    }
 
     /// <summary>
     /// Applies a given list of table patches to the current file's TBL segments.
     /// </summary>
     internal static unsafe void ApplyPatch(List<TblPatch> patches, int x, Memory<byte>[] segments)
     {
+        patches.Sort((a, b) => a.SegmentDiffs[x].LengthAfterPatch.CompareTo(b.SegmentDiffs[x].LengthAfterPatch));
+
+        int newLength = 0;
+
+        foreach (var patch in CollectionsMarshal.AsSpan(patches))
+        {
+            if (patch.SegmentDiffs[x].LengthAfterPatch > newLength)
+            {
+                newLength = patch.SegmentDiffs[x].LengthAfterPatch;
+            }
+        }
+
         foreach (var patch in CollectionsMarshal.AsSpan(patches))
         {
-            var destination = GC.AllocateUninitializedArray<byte>(Math.Max(patch.SegmentDiffs[x].LengthAfterPatch, segments[x].Length));
+            var destination = GC.AllocateUninitializedArray<byte>(Math.Max(newLength, segments[x].Length));
             var patchDiff = patch.SegmentDiffs[x].Data;
 
             fixed (byte* destinationPtr = &destination[0])

Persona.Merger.Tests/P5RAssets.cs

@@ -19,4 +19,14 @@ public static class P5RAssets
     public static readonly string ItemBefore  = Path.Combine(AssetsFolder, "Item", "Before", "ITEM.TBL");
     public static readonly string ItemAfter   = Path.Combine(AssetsFolder, "Item", "After", "ITEM.TBL");
     public static readonly string ItemExtend  = Path.Combine(AssetsFolder, "Item", "Extend", "ITEM.TBL"); // Miku?
+
+    public static readonly string VisualBefore = Path.Combine(AssetsFolder, "Visual", "Before", "VISUAL.TBL");
+    public static readonly string VisualAfter = Path.Combine(AssetsFolder, "Visual", "After", "VISUAL.TBL");
+    public static readonly string VisualAfter2 = Path.Combine(AssetsFolder, "Visual", "After2", "VISUAL.TBL");
+    public static readonly string VisualAfter3 = Path.Combine(AssetsFolder, "Visual", "After3", "VISUAL.TBL");
+
+    public static readonly string PDDBefore = Path.Combine(AssetsFolder, "PDD", "Before", "SHDPERSONAENEMY.PDD");
+    public static readonly string PDDAfter = Path.Combine(AssetsFolder, "PDD", "After", "SHDPERSONAENEMY.PDD");
+    public static readonly string PDDAfter2 = Path.Combine(AssetsFolder, "PDD", "After2", "SHDPERSONAENEMY.PDD");
+    public static readonly string PDDAfter3 = Path.Combine(AssetsFolder, "PDD", "After3", "SHDPERSONAENEMY.PDD");
 }

Persona.Merger.Tests/Parser/P5RTblPatcherTests.cs

@@ -1,5 +1,6 @@
 using Persona.Merger.Patching.Tbl;
 using Persona.Merger.Patching.Tbl.FieldResolvers.P5R;
+using Persona.Merger.Patching.Tbl.FieldResolvers.Generic;
 
 namespace Persona.Merger.Tests.Parser;
 
@@ -37,4 +38,40 @@ public void PatchTbl_Item_Extend()
         var patched = patcher.Apply(new List<TblPatch>() { patch });
         Assert.Equal(after, patched);
     }
+
+    [Fact]
+    public void PatchTbl_Generic()
+    {
+        var patches = new List<TblPatch>();
+
+        var original = File.ReadAllBytes(P5RAssets.PDDBefore);
+        var after = File.ReadAllBytes(P5RAssets.PDDAfter);
+        var after2 = File.ReadAllBytes(P5RAssets.PDDAfter2);
+        var after3 = File.ReadAllBytes(P5RAssets.PDDAfter3);
+        var patcher = new GenericPatcher(original);
+        patches.Add(patcher.GeneratePatchGeneric(after, 4));
+        patches.Add(patcher.GeneratePatchGeneric(after2, 4));
+        var patched = patcher.ApplyGeneric(patches);
+
+        Assert.Equal(patched, after3);
+    }
+
+    [Fact]
+    public void PatchTbl_VISUAL()
+    {
+        var patches = new List<TblPatch>();
+
+        var original = File.ReadAllBytes(P5RAssets.VisualBefore);
+        var after = File.ReadAllBytes(P5RAssets.VisualAfter);
+        var after2 = File.ReadAllBytes(P5RAssets.VisualAfter2);
+        var after3 = File.ReadAllBytes(P5RAssets.VisualAfter3);
+        var patcher = new P5RTblPatcher(original, TblType.Visual);
+        patches.Add(patcher.GeneratePatch(after));
+        patches.Add(patcher.GeneratePatch(after2));
+        var patched = patcher.Apply(patches);
+
+        File.WriteAllBytes("D:\\Game Modding\\R2\\Loader\\VISUAL.TBL", patched);
+
+        Assert.Equal(patched, after3);
+    }
 }

Persona.Merger.Tests/Persona.Merger.Tests.csproj

@@ -42,6 +42,10 @@
 
     <ItemGroup>
       <Folder Include="Assets\P5R\Skill\Extend" />
+      <Folder Include="Assets\P5R\Visual\Before\" />
+      <Folder Include="Assets\P5R\Visual\After\" />
+      <Folder Include="Assets\P5R\Visual\After2\" />
+      <Folder Include="Assets\P5R\Visual\After3\" />
     </ItemGroup>
 
 </Project>

p5rpc.modloader/CHANGELOG.MD

@@ -1,3 +1,8 @@
+# 2.8.0
+
+- Fixed TBL Merging bug related to expanded TBL files
+- Added generic file merging for a bunch of files for P3P, P4G and P5R
+
 # 2.7.1
 
 - Fixed: Introskip on P5R Ver 1.04

p5rpc.modloader/Merging/Tbl/P3PTblMerger.cs

@@ -1,9 +1,11 @@
 using CriFs.V2.Hook.Interfaces;
+using CriFsV2Lib.Definitions.Utilities;
 using FileEmulationFramework.Lib.Utilities;
 using PAK.Stream.Emulator.Interfaces;
 using PAK.Stream.Emulator.Interfaces.Structures.IO;
 using Persona.Merger.Cache;
 using Persona.Merger.Patching.Tbl;
+using Persona.Merger.Patching.Tbl.FieldResolvers.Generic;
 using Persona.Merger.Patching.Tbl.FieldResolvers.P3P;
 using static p5rpc.modloader.Merging.Tbl.TblMerger;
 
@@ -40,7 +42,10 @@ public void Merge(string[] cpks, ICriFsRedirectorApi.BindContext context)
             PatchTbl(pakFiles, @"init_free.bin\battle\PERSONA.TBL", TblType.Persona, cpks),
             PatchTbl(pakFiles, @"init_free.bin\battle\SKILL.TBL", TblType.Skill, cpks),
             PatchTbl(pakFiles, @"init_free.bin\battle\UNIT.TBL", TblType.Unit, cpks),
-            PatchTbl(pakFiles, @"init_free.bin\init\itemtbl.bin", TblType.Item, cpks)
+            PatchTbl(pakFiles, @"init_free.bin\init\itemtbl.bin", TblType.Item, cpks),
+            PatchAnyFileInPak(pakFiles, @"data\facility\combine.bin\ps_model.bin", 4, cpks),
+            PatchAnyFileInPak(pakFiles, @"data\facility\elvgirl.bin\ps_model.bin", 4, cpks),
+            PatchAnyFileInPak(pakFiles, @"data\facility\elvgirl_m.bin\ps_model.bin", 4, cpks)
         };
 
         Task.WhenAll(tasks.Select(x => x.AsTask())).Wait();
@@ -125,6 +130,65 @@ await Task.Run(async () =>
         });
     }
 
+    private async ValueTask PatchAnyFileInPak(RouteGroupTuple[] pakFiles, string tblPath, int ResolverSize, string[] cpks)
+    {
+        var route = tblPath.Substring(0, tblPath.LastIndexOf('\\'));
+        var tblName = Path.GetFileName(tblPath);
+        var candidates = FindInPaks(pakFiles, route, tblName);
+
+        if (candidates.Count == 0) return;
+
+        var extIndex = tblPath.IndexOf('.');
+        var dirIndex = tblPath.IndexOf('\\', extIndex);
+        var pathInCpk = '\\' + tblPath.Substring(0, dirIndex);
+        var pathInPak = tblPath.Substring(dirIndex + 1);
+
+        if (!_utils.TryFindFileInAnyCpk(pathInCpk, cpks, out var cpkPath, out var cpkEntry, out var fileIndex))
+        {
+            _logger.Warning("Unable to find TBL in any CPK {0}", tblPath);
+            return;
+        }
+
+        // Build cache key
+        string[] modIds = { "p5rpc.modloader" };
+        var cacheKey = MergedFileCache.CreateKey(tblPath, modIds);
+        var sources = candidates.Select(x => new CachedFileSource { LastWrite = File.GetLastWriteTime(x) }).ToArray();
+
+        if (_mergedFileCache.TryGet(cacheKey, sources, out var cachedFilePath))
+        {
+            _logger.Info("Loading Merged TBL {0} from Cache ({1})", tblPath, cachedFilePath);
+            _pakEmulator.AddFile(cachedFilePath, route, pathInPak);
+            return;
+        }
+
+        // Else Merge our Data
+        // First we extract.
+        await Task.Run(async () =>
+        {
+            _logger.Info("Merging {0} with key {1}.", tblPath, cacheKey);
+            await using var cpkStream =
+                new FileStream(cpkPath, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
+            using var reader = _criFsApi.GetCriFsLib().CreateCpkReader(cpkStream, false);
+            using var extractedPak = reader.ExtractFile(cpkEntry.Files[fileIndex].File);
+
+            var extractedTbl = _pakEmulator.GetEntry(new MemoryStream(extractedPak.RawArray), pathInPak);
+            if (extractedTbl == null)
+            {
+                _logger.Error($"Unable to extract {pathInPak} from {pathInCpk}");
+                return;
+            }
+
+            // Then we merge
+            byte[] patched;
+            patched = await PatchAny(extractedTbl.Value.ToArray(), candidates, ResolverSize);
+
+            // Then we store in cache.
+            var item = await _mergedFileCache.AddAsync(cacheKey, sources, patched);
+            _pakEmulator.AddFile(Path.Combine(_mergedFileCache.CacheFolder, item.RelativePath), route, pathInPak);
+            _logger.Info("Merge {0} Complete. Cached to {1}.", tblPath, item.RelativePath);
+        });
+    }
+
     private static async Task<byte[]> PatchTable(TblType type, byte[] extractedTable, List<string> candidates)
     {
         var patcher = new P3PTblPatcher(extractedTable, type);
@@ -179,4 +243,16 @@ private async Task<byte[]> PatchAiCalc(byte[] extractedTable, List<string> candi
         var patched = patcher.Apply(patches, TblType.AiCalc, bfs);
         return patched;
     }
+
+    private static async Task<byte[]> PatchAny(byte[] extractedTable,
+        List<string> candidates, int ResolverSize)
+    {
+        var patcher = new GenericPatcher(extractedTable);
+        var patches = new List<TblPatch>(candidates.Count);
+        for (var x = 0; x < candidates.Count; x++)
+            patches.Add(patcher.GeneratePatchGeneric(await File.ReadAllBytesAsync(candidates[x]), ResolverSize));
+
+        var patched = patcher.ApplyGeneric(patches);
+        return patched;
+    }
 }