Add hooks to debug OpenSSL memory allocations

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs

@@ -3,10 +3,13 @@
 
 using System;
 using System.Diagnostics;
+using System.Globalization;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography.X509Certificates;
 using Microsoft.Win32.SafeHandles;
 
+using TrackedAllocationDelegate = System.Action<System.IntPtr, ulong, System.IntPtr, int>;
+
 internal static partial class Interop
 {
     internal static partial class Crypto
@@ -164,5 +167,52 @@ internal static byte[] GetDynamicBuffer<THandle>(NegativeSizeReadMethod<THandle>
 
             return bytes;
         }
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_GetMemoryUse")]
+        private static partial void GetMemoryUse(ref long memoryUse, ref long allocationCount);
+
+        internal static long GetOpenSslAllocatedMemory()
+        {
+            long used = 0;
+            long count = 0;
+            GetMemoryUse(ref used, ref count);
+            return used;
+        }
+
+        internal static long GetOpenSslAllocationCount()
+        {
+            long used = 0;
+            long count = 0;
+            GetMemoryUse(ref used, ref count);
+            return count;
+        }
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EnableMemoryTracking")]
+        internal static unsafe partial void EnableMemoryTracking(int enable);
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_ForEachTrackedAllocation")]
+        private static unsafe partial void ForEachTrackedAllocation(delegate* unmanaged<IntPtr, ulong, char*, int, IntPtr, void> callback, IntPtr ctx);
+
+        internal static unsafe void ForEachTrackedAllocation(TrackedAllocationDelegate callback)
+        {
+            ForEachTrackedAllocation(&MemoryTrackingCallback, (IntPtr)(&callback));
+        }
+
+        [UnmanagedCallersOnly]
+        private static unsafe void MemoryTrackingCallback(IntPtr ptr, ulong size, char* file, int line, IntPtr ctx)
+        {
+            TrackedAllocationDelegate callback = *(TrackedAllocationDelegate*)ctx;
+            callback(ptr, size, (IntPtr)file, line);
+        }
+
+        internal static unsafe void EnableMemoryTracking()
+        {
+            EnableMemoryTracking(1);
+        }
+
+        internal static unsafe void DisableMemoryTracking()
+        {
+            EnableMemoryTracking(0);
+        }
     }
 }

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/README.md

@@ -0,0 +1,77 @@
+# System.Security.Cryptography.Native
+
+This folder contains C# bindings for native shim (libSystem.Security.Cryptography.Native.OpenSsl.so), shimming functionality provided by the OpenSSL library.
+
+## Memory allocation hooks
+
+One extra feature exposed by the native shim is tracking of memory used by
+OpenSSL by hooking the memory allocation routines via
+`CRYPTO_set_mem_functions`.
+
+The functionality is enabled by setting
+`DOTNET_OPENSSL_MEMORY_DEBUG` to 1. This environment
+variable must be set before launching the program (calling
+`Environment.SetEnvironmentVariable` at the start of the program is not
+sufficient). The diagnostic API is not officially exposed and needs to be
+accessed via private reflection on the `Interop.Crypto` type located in the
+`System.Security.Cryptography` assembly. On this type, you can use following static
+methods:
+
+- `int GetOpenSslAllocatedMemory()`
+    - Gets the total amount of memory allocated by OpenSSL
+- `int GetOpenSslAllocationCount()`
+    - Gets the number of allocations made by OpenSSL
+- `void EnableMemoryTracking()`/`void DisableMemoryTracking()`
+    - toggles tracking of individual live allocations via internal data
+      structures. I.e. will keep track of live memory allocated since the start of
+      tracking.
+- `void ForEachTrackedAllocation(Action<IntPtr, ulong, IntPtr, int> callback)`
+    - Accepts an callback and calls it for each allocation performed since the
+      last `EnableMemoryTracking` call. The order of reported information does not
+      correspond to the order of allocation. This method holds an internal lock
+      which prevents other threads from allocating any memory from OpenSSL.
+    - Callback parameters are
+        - IntPtr - The pointer to the allocated object
+        - ulong - size of the allocation in bytes
+        - IntPtr - Pointer to a null-terminated string (`const char*`) containing the name of the file from which the allocation was made.
+        - int - line number within the file specified by the previous parameter where the allocation was called from.
+
+The debug functionality brings some overhead (header for each allocation,
+locks/synchronization during each allocation) and may cause performance penalty.
+
+### Example usage
+
+```cs
+// all above mentioned APIs are accessible via "private reflection"
+BindingFlags flags = BindingFlags.InvokeMethod | BindingFlags.NonPublic | BindingFlags.Static;
+var cryptoInterop = typeof(RandomNumberGenerator).Assembly.GetTypes().First(t => t.Name == "Crypto");
+
+// enable tracking, this clears up any previously tracked allocations
+cryptoInterop.InvokeMember("EnableMemoryTracking", flags, null, null, null);
+
+// do some work that includes OpenSSL
+HttpClient client = new HttpClient();
+await client.GetAsync("https://www.microsoft.com");
+
+// stop tracking (this step is optional)
+cryptoInterop.InvokeMember("DisableMemoryTracking", flags, null, null, null);
+
+using var process = Process.GetCurrentProcess();
+Console.WriteLine($"Bytes known to GC [{GC.GetTotalMemory(false)}], process working set [{process.WorkingSet64}]");
+Console.WriteLine("OpenSSL - currently allocated memory: {0} B", cryptoInterop.InvokeMember("GetOpenSslAllocatedMemory", flags, null, null, null));
+Console.WriteLine("OpenSSL - total allocations since start: {0}", cryptoInterop.InvokeMember("GetOpenSslAllocationCount", flags, null, null, null));
+
+Dictionary<(IntPtr file, int line), ulong> allAllocations = new();
+Action<IntPtr, ulong, IntPtr, int> callback = (ptr, size, namePtr, line) =>
+{
+    CollectionsMarshal.GetValueRefOrAddDefault(allAllocations, (namePtr, line), out _) += size;
+};
+cryptoInterop.InvokeMember("ForEachTrackedAllocation", flags, null, null, [callback]);
+
+Console.WriteLine("Total allocated OpenSSL memory by location");
+foreach (var ((filenameptr, line), total) in allAllocations.OrderByDescending(kvp => kvp.Value).Take(10))
+{
+    string filename = Marshal.PtrToStringUTF8(filenameptr);
+    Console.WriteLine($"{total:N0} B from {filename}:{line}");
+}
+```

src/libraries/System.Security.Cryptography/src/ILLink/ILLink.Descriptors.LibraryBuild.xml

@@ -0,0 +1,12 @@
+<linker>
+  <assembly fullname="System.Security.Cryptography">
+    <type fullname="Interop/Crypto">
+      <!-- Debug only APIs to be reachable only via private reflection -->
+      <method name="GetOpenSslAllocationCount" />
+      <method name="GetOpenSslAllocatedMemory" />
+      <method name="ForEachTrackedAllocation" />
+      <method name="EnableMemoryTracking" />
+      <method name="DisableMemoryTracking" />
+    </type>
+  </assembly>
+</linker>

src/native/libs/System.Security.Cryptography.Native/CMakeLists.txt

@@ -21,6 +21,7 @@ include_directories(${OPENSSL_INCLUDE_DIR})
 set(NATIVECRYPTO_SOURCES
     apibridge.c
     apibridge_30.c
+    memory_debug.c
     openssl.c
     pal_asn1.c
     pal_bignum.c

src/native/libs/System.Security.Cryptography.Native/apibridge.h

@@ -65,3 +65,9 @@ int32_t local_X509_get_version(const X509* x509);
 int32_t local_X509_up_ref(X509* x509);
 typedef void (*SSL_CTX_keylog_cb_func)(const SSL *ssl, const char *line);
 void local_SSL_CTX_set_keylog_callback(SSL_CTX *ctx, SSL_CTX_keylog_cb_func cb);
+
+typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line);
+typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file, int line);
+typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line);
+
+int CRYPTO_set_mem_functions(CRYPTO_malloc_fn malloc_fn, CRYPTO_realloc_fn realloc_fn, CRYPTO_free_fn free_fn);

src/native/libs/System.Security.Cryptography.Native/entrypoints.c

@@ -5,6 +5,7 @@
 
 // Include System.Security.Cryptography.Native headers
 #include "openssl.h"
+#include "memory_debug.h"
 #include "pal_asn1.h"
 #include "pal_bignum.h"
 #include "pal_bio.h"
@@ -188,6 +189,9 @@ static const Entry s_cryptoNative[] =
     DllImportEntry(CryptoNative_GetECKeyParameters)
     DllImportEntry(CryptoNative_GetMaxMdSize)
     DllImportEntry(CryptoNative_GetMemoryBioSize)
+    DllImportEntry(CryptoNative_GetMemoryUse)
+    DllImportEntry(CryptoNative_EnableMemoryTracking)
+    DllImportEntry(CryptoNative_ForEachTrackedAllocation)
     DllImportEntry(CryptoNative_GetObjectDefinitionByName)
     DllImportEntry(CryptoNative_GetOcspRequestDerSize)
     DllImportEntry(CryptoNative_GetPkcs7Certificates)