Merge pull request #37 from DeathChaos25/master

Fix tbl merge bug and add generic file merging

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/ByteResolver.cs

@@ -0,0 +1,13 @@
+using Sewer56.StructuredDiff.Interfaces;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers;
+
+public struct ByteResolver : IEncoderFieldResolver
+{
+    public bool Resolve(nuint offset, out int moveBy, out int length)
+    {
+        moveBy = 0;
+        length = 1;
+        return false;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/GenericPatcher.cs

@@ -0,0 +1,88 @@
+using Persona.Merger.Utilities;
+using Reloaded.Memory.Streams;
+using static Persona.Merger.Patching.Tbl.FieldResolvers.TblPatcherCommon;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers.Generic;
+
+/// <summary>
+/// Utility class for patching P5R TBL files.
+/// </summary>
+public struct GenericPatcher
+{
+    /// <summary>
+    /// The table to patch.
+    /// </summary>
+    public byte[] TblData { get; set; }
+
+    public GenericPatcher(byte[] tblData)
+    {
+        TblData = tblData;
+    }
+
+    /// <summary>
+    /// Generates a table patch.
+    /// </summary>
+    /// <param name="otherTbl">Data of the new table.</param>
+    public unsafe TblPatch GeneratePatchGeneric(byte[] otherTbl, int ResolverSize)
+    {
+        fixed (byte* otherTblData = &otherTbl[0])
+        fixed (byte* tblData = &TblData[0])
+        {
+            var patch = new TblPatch();
+
+            var originalSegments = stackalloc PointerLengthTuple[1]; // using pointer to elide bounds checks below
+            var newSegments = stackalloc PointerLengthTuple[1];
+            PopulateGeneric(tblData, TblData.Length, originalSegments);
+            PopulateGeneric(otherTblData, otherTbl.Length, newSegments);
+
+            if (ResolverSize == 2) DiffSegment(patch, newSegments[0], originalSegments[0], new ShortResolver());
+            else if (ResolverSize == 4) DiffSegment(patch, newSegments[0], originalSegments[0], new IntResolver());
+            else DiffSegment(patch, newSegments[0], originalSegments[0], new ByteResolver());
+
+            return patch;
+        }
+    }
+
+    /// <summary>
+    /// Applies a list of table patches.
+    /// </summary>
+    /// <param name="patches">List of patches to apply.</param>
+    public unsafe byte[] ApplyGeneric(List<TblPatch> patches)
+    {
+        fixed (byte* tblData = &TblData[0])
+        {
+            // Get original segments.
+            var segmentCount = 1;
+            var originalSegments = stackalloc PointerLengthTuple[segmentCount]; // using pointer to elide bounds checks below
+            PopulateGeneric(tblData, segmentCount, originalSegments);
+
+            // Convert original segments into Memory<T>.
+            var segments = ConvertSegmentsToMemoryGeneric(segmentCount, originalSegments, tblData, TblData);
+
+            // Apply Patch(es).
+            for (int x = 0; x < segmentCount; x++)
+                ApplyPatch(patches, x, segments);
+
+            // Produce new file.
+            var fileSize = 0;
+            foreach (var segment in segments)
+                fileSize += segment.Length;
+
+            var result = GC.AllocateUninitializedArray<byte>(fileSize);
+            using var memoryStream = new ExtendedMemoryStream(result, true);
+            foreach (var segment in segments)
+            {
+                memoryStream.Write(segment.Span);
+            }
+
+            return result;
+        }
+    }
+
+    public unsafe static void PopulateGeneric(byte* tblPointer, int length, PointerLengthTuple* segments)
+    {
+        ref var currentSegment = ref segments[0];
+        currentSegment.Pointer = tblPointer;
+        currentSegment.Length = length;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/IntResolver.cs

@@ -0,0 +1,15 @@
+using Sewer56.StructuredDiff.Interfaces;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers;
+
+public struct IntResolver : IEncoderFieldResolver
+{
+    public bool Resolve(nuint offset, out int moveBy, out int length)
+    {
+        // All data are u32s.
+        var fourByteAligned = offset / 4 * 4;
+        moveBy = (int)(offset - fourByteAligned);
+        length = 4;
+        return true;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/Generic/ShortResolver.cs

@@ -0,0 +1,14 @@
+using Sewer56.StructuredDiff.Interfaces;
+
+namespace Persona.Merger.Patching.Tbl.FieldResolvers;
+
+public struct ShortResolver : IEncoderFieldResolver
+{
+    public bool Resolve(nuint offset, out int moveBy, out int length)
+    {
+        var twoByteAligned = offset / 2 * 2;
+        moveBy = (int)(offset - twoByteAligned);
+        length = 2;
+        return true;
+    }
+}

Persona.Merger.Common/Patching/Tbl/FieldResolvers/TblPatcherCommon.cs

@@ -26,24 +26,49 @@ internal static unsafe Memory<byte>[] ConvertSegmentsToMemory(int segmentCount,
 
         return segments;
     }
+    internal static unsafe Memory<byte>[] ConvertSegmentsToMemoryGeneric(int segmentCount, PointerLengthTuple* originalSegments, byte* tblData, byte[] tblDataArr)
+    {
+        var segments = new Memory<byte>[segmentCount];
+        for (int x = 0; x < segmentCount; x++)
+        {
+            ref var originalSegment = ref originalSegments[x];
+            segments[x] = new Memory<byte>(tblDataArr, 0, tblDataArr.Length);
+        }
+
+        return segments;
+    }
 
     /// <summary>
     /// Applies a given list of table patches to the current file's TBL segments.
     /// </summary>
-    internal static unsafe void ApplyPatch(List<TblPatch> patches, int x, Memory<byte>[] segments)
+    /// <param name="patches">A list of patches to apply, these are applied in order of the list</param>
+    /// <param name="segment">The index of the segment in the table to apply the patches to</param>
+    /// <param name="segments">A list of the data of each segment. segments[segment] will be replaced with the patched segment data.</param>
+    internal static unsafe void ApplyPatch(List<TblPatch> patches, int segment, Memory<byte>[] segments)
     {
+        int newLength = 0;
+
+        foreach (var patch in CollectionsMarshal.AsSpan(patches))
+        {
+            if (patch.SegmentDiffs[segment].LengthAfterPatch > newLength)
+            {
+                newLength = patch.SegmentDiffs[segment].LengthAfterPatch;
+            }
+        }
+
         foreach (var patch in CollectionsMarshal.AsSpan(patches))
         {
-            var destination = GC.AllocateUninitializedArray<byte>(Math.Max(patch.SegmentDiffs[x].LengthAfterPatch, segments[x].Length));
-            var patchDiff = patch.SegmentDiffs[x].Data;
+            var destination = GC.AllocateUninitializedArray<byte>(Math.Max(newLength, segments[segment].Length));
+            segments[segment].CopyTo(destination);
+            var patchDiff = patch.SegmentDiffs[segment].Data;
 
             fixed (byte* destinationPtr = &destination[0])
-            fixed (byte* currentSegmentPtr = segments[x].Span)
+            fixed (byte* currentSegmentPtr = segments[segment].Span)
             fixed (byte* patchPtr = patchDiff.Span)
             {
-                S56DiffDecoder.Decode(currentSegmentPtr, patchPtr, destinationPtr, (nuint)patch.SegmentDiffs[x].Data.Length,
-                    out var numWritten);
-                segments[x] = new Memory<byte>(destination, 0, (int)numWritten);
+                S56DiffDecoder.Decode(currentSegmentPtr, patchPtr, destinationPtr, (nuint)patch.SegmentDiffs[segment].Data.Length,
+                    out _);
+                segments[segment] = destination;
             }
         }
     }

Persona.Merger.Tests/P5RAssets.cs

@@ -19,4 +19,14 @@ public static class P5RAssets
     public static readonly string ItemBefore  = Path.Combine(AssetsFolder, "Item", "Before", "ITEM.TBL");
     public static readonly string ItemAfter   = Path.Combine(AssetsFolder, "Item", "After", "ITEM.TBL");
     public static readonly string ItemExtend  = Path.Combine(AssetsFolder, "Item", "Extend", "ITEM.TBL"); // Miku?
+
+    public static readonly string VisualBefore = Path.Combine(AssetsFolder, "Visual", "Before", "VISUAL.TBL");
+    public static readonly string VisualAfter = Path.Combine(AssetsFolder, "Visual", "After", "VISUAL.TBL");
+    public static readonly string VisualAfter2 = Path.Combine(AssetsFolder, "Visual", "After2", "VISUAL.TBL");
+    public static readonly string VisualExpected = Path.Combine(AssetsFolder, "Visual", "Expected", "VISUAL.TBL");
+
+    public static readonly string PDDBefore = Path.Combine(AssetsFolder, "PDD", "Before", "SHDPERSONAENEMY.PDD");
+    public static readonly string PDDAfter = Path.Combine(AssetsFolder, "PDD", "After", "SHDPERSONAENEMY.PDD");
+    public static readonly string PDDAfter2 = Path.Combine(AssetsFolder, "PDD", "After2", "SHDPERSONAENEMY.PDD");
+    public static readonly string PDDExpected = Path.Combine(AssetsFolder, "PDD", "Expected", "SHDPERSONAENEMY.PDD");
 }

Persona.Merger.Tests/Parser/P5RTblPatcherTests.cs

@@ -1,5 +1,6 @@
 using Persona.Merger.Patching.Tbl;
 using Persona.Merger.Patching.Tbl.FieldResolvers.P5R;
+using Persona.Merger.Patching.Tbl.FieldResolvers.Generic;
 
 namespace Persona.Merger.Tests.Parser;
 
@@ -37,4 +38,38 @@ public void PatchTbl_Item_Extend()
         var patched = patcher.Apply(new List<TblPatch>() { patch });
         Assert.Equal(after, patched);
     }
+
+    [Fact]
+    public void PatchTbl_Generic()
+    {
+        var patches = new List<TblPatch>();
+
+        var original = File.ReadAllBytes(P5RAssets.PDDBefore);
+        var after = File.ReadAllBytes(P5RAssets.PDDAfter);
+        var after2 = File.ReadAllBytes(P5RAssets.PDDAfter2);
+        var expected = File.ReadAllBytes(P5RAssets.PDDExpected);
+        var patcher = new GenericPatcher(original);
+        patches.Add(patcher.GeneratePatchGeneric(after, 4));
+        patches.Add(patcher.GeneratePatchGeneric(after2, 4));
+        var patched = patcher.ApplyGeneric(patches);
+
+        Assert.Equal(patched, expected);
+    }
+
+    [Fact]
+    public void PatchTbl_VISUAL()
+    {
+        var patches = new List<TblPatch>();
+
+        var original = File.ReadAllBytes(P5RAssets.VisualBefore);
+        var after = File.ReadAllBytes(P5RAssets.VisualAfter);
+        var after2 = File.ReadAllBytes(P5RAssets.VisualAfter2);
+        var expected = File.ReadAllBytes(P5RAssets.VisualExpected);
+        var patcher = new P5RTblPatcher(original, TblType.Visual);
+        patches.Add(patcher.GeneratePatch(after));
+        patches.Add(patcher.GeneratePatch(after2));
+        var patched = patcher.Apply(patches);
+
+        Assert.Equal(patched, expected);
+    }
 }

Persona.Merger.Tests/Persona.Merger.Tests.csproj

@@ -42,6 +42,10 @@
 
     <ItemGroup>
       <Folder Include="Assets\P5R\Skill\Extend" />
+      <Folder Include="Assets\P5R\Visual\Before\" />
+      <Folder Include="Assets\P5R\Visual\After\" />
+      <Folder Include="Assets\P5R\Visual\After2\" />
+      <Folder Include="Assets\P5R\Visual\Expected\" />
     </ItemGroup>
 
 </Project>

p5rpc.modloader/CHANGELOG.MD

@@ -1,3 +1,8 @@
+# 2.8.0
+
+- Fixed TBL Merging bug related to expanded TBL files
+- Added generic file merging for a bunch of files for P3P, P4G and P5R
+
 # 2.7.1
 
 - Fixed: Introskip on P5R Ver 1.04