add hooks to debug OpenSSL memory

src/libraries/Common/src/Interop/Android/System.Security.Cryptography.Native.Android/Interop.Initialization.cs

@@ -31,7 +31,7 @@
     {
         static CryptoInitializer()
         {
-            if (EnsureOpenSslInitialized() != 0)
+            if (EnsureOpenSslInitialized(null, null, null) != 0)
             {
                 // Ideally this would be a CryptographicException, but we use
                 // OpenSSL in libraries lower than System.Security.Cryptography.

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Initialization.cs

@@ -2,7 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
 using System.Runtime.InteropServices;
+using System.Threading;
 
 internal static partial class Interop
 {
@@ -27,11 +31,36 @@ static OpenSsl()
         }
     }
 
-    internal static partial class CryptoInitializer
+    internal static unsafe partial class CryptoInitializer
     {
-        static CryptoInitializer()
+        internal struct MemoryEntry
         {
-            if (EnsureOpenSslInitialized() != 0)
+            public char* File;
+            public int Size;
+            public int Line;
+        }
+
+        private static readonly bool DebugMemory = GetMemoryDebug("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_DEBUG");
+        private static readonly bool ValidateMemory = GetMemoryDebug("DOTNET_SYSTEM_NET_SECURITY_OPENSSL_MEMORY_VALIDATE");
+        private static readonly IntPtr Offset = sizeof(MemoryEntry);
+        private static HashSet<IntPtr>? _allocations;
+        private static HashSet<IntPtr>? _allocationsDiff;
+        private static bool _trackIncrementalAllocations;
+
+        internal static long TotalAllocatedMemory;
+        internal static long TotalAllocations;
+
+#pragma warning disable CA1810
+        static unsafe CryptoInitializer()
+        {
+            if (DebugMemory)
+            {
+                // we need to prepare everything as some allocations do happen during initialization itself.
+                _allocations = new HashSet<IntPtr>();
+                _allocationsDiff = new HashSet<IntPtr>();
+            }
+
+            if (EnsureOpenSslInitialized(DebugMemory ? &CryptoMalloc : null, DebugMemory ? &CryptoRealloc : null, DebugMemory ? &CryptoFree : null) != 0)
             {
                 // Ideally this would be a CryptographicException, but we use
                 // OpenSSL in libraries lower than System.Security.Cryptography.
@@ -42,13 +71,204 @@ static CryptoInitializer()
                 throw new InvalidOperationException();
             }
         }
+#pragma warning restore CA1810
 
         internal static void Initialize()
         {
             // No-op that exists to provide a hook for other static constructors.
         }
 
+        private static bool GetMemoryDebug(string name)
+        {
+            string? value = Environment.GetEnvironmentVariable(name);
+            if (int.TryParse(value, CultureInfo.InvariantCulture, out int enabled))
+            {
+                if (enabled == 2)
+                {
+                    EnableTracking();
+                }
+                else if (enabled == 3)
+                {
+                    GetIncrementalAllocations();
+                }
+
+                return enabled == 1;
+            }
+
+            return false;
+        }
+
+        internal static void EnableTracking()
+        {
+            _allocationsDiff!.Clear();
+            _trackIncrementalAllocations = true;
+        }
+
+        internal static Tuple<IntPtr, int, string>[] GetIncrementalAllocations()
+        {
+            lock (_allocationsDiff!)
+            {
+                Tuple<IntPtr, int, string>[] allocations = new Tuple<IntPtr, int, string>[_allocationsDiff.Count];
+                int index = 0;
+                foreach (IntPtr ptr in _allocationsDiff)
+                {
+                    Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)ptr, 1);
+                    allocations[index] = new Tuple<IntPtr, int, string>(ptr+Offset, entry[0].Size, $"{Marshal.PtrToStringAnsi((IntPtr)entry[0].File)}:{entry[0].Line}");
+                    index++;
+                }
+
+                return allocations;
+            }
+        }
+
+        internal static void DisableTracking()
+        {
+            _trackIncrementalAllocations = false;
+            _allocationsDiff!.Clear();
+        }
+
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EnsureOpenSslInitialized")]
-        private static partial int EnsureOpenSslInitialized();
+        private static unsafe partial int EnsureOpenSslInitialized(delegate* unmanaged<UIntPtr, char*, int, void*> mallocFunction, delegate* unmanaged<void*, UIntPtr, char*, int, void*> reallocFunction, delegate* unmanaged<void*, void> freeFunction);
+
+        [UnmanagedCallersOnly]
+        internal static unsafe void* CryptoMalloc(UIntPtr size, char* file, int line)
+        {
+            void* ptr =  NativeMemory.Alloc(size + (UIntPtr)Offset);
+            Debug.Assert(ptr != null);
+
+            if (ptr == null)
+            {
+                return null;
+            }
+
+            Span<MemoryEntry> entry = new Span<MemoryEntry>(ptr, 1);
+            entry[0].Line = line;
+            entry[0].File = file;
+            entry[0].Size = (int)size;
+
+            if (ValidateMemory)
+            {
+                lock (_allocations!)
+                {
+                    Debug.Assert(_allocations!.Add((IntPtr)ptr));
+                }
+            }
+            if (_trackIncrementalAllocations)
+            {
+                lock (_allocationsDiff!)
+                {
+                    Debug.Assert(_allocationsDiff!.Add((IntPtr)ptr));
+                }
+            }
+            Interlocked.Add(ref TotalAllocatedMemory, (long)size);
+            Interlocked.Increment(ref TotalAllocations);
+
+            return (void*)((IntPtr)ptr + Offset);
+        }
+
+        [UnmanagedCallersOnly]
+        internal static unsafe void* CryptoRealloc(void* oldPtr, UIntPtr size, char* file, int line)
+        {
+            void * ptr;
+            Span<MemoryEntry> entry;
+
+            if (oldPtr != null)
+            {
+                IntPtr entryPtr = (IntPtr)oldPtr - Offset;
+                entry = new Span<MemoryEntry>((void*)entryPtr, 1);
+
+                if (ValidateMemory)
+                {
+                    lock (_allocations!)
+                    {
+                        if (!_allocations!.Remove(entryPtr))
+                        {
+                            Environment.FailFast($"Failed to find OpenSSL memory 0x{(IntPtr)oldPtr:x}");
+                        }
+                    }
+                }
+
+                if (_trackIncrementalAllocations)
+                {
+                    lock (_allocationsDiff!)
+                    {
+                        // this may fail as we may start tracking after given chunk was allocated
+                        _allocationsDiff!.Remove(entryPtr);
+                    }
+                }
+
+                Interlocked.Add(ref TotalAllocatedMemory, -((long)entry[0].Size));
+                ptr =  NativeMemory.Realloc((void*)entryPtr, size + (UIntPtr)Offset);
+            }
+            else
+            {
+                ptr =  NativeMemory.Alloc(size + (UIntPtr)Offset);
+            }
+
+
+            Debug.Assert(ptr != null);
+            if (ptr == null)
+            {
+                return null;
+            }
+
+            if (ValidateMemory)
+            {
+                lock (_allocations!)
+                {
+                    Debug.Assert(_allocations!.Add((IntPtr)ptr));
+                }
+            }
+            if (_trackIncrementalAllocations)
+            {
+                lock (_allocationsDiff!)
+                {
+                    Debug.Assert(_allocationsDiff!.Add((IntPtr)ptr));
+                }
+            }
+            Interlocked.Add(ref TotalAllocatedMemory, (long)size);
+            Interlocked.Increment(ref TotalAllocations);
+
+            entry = new Span<MemoryEntry>((void*)ptr, 1);
+            entry[0].Line = line;
+            entry[0].File = file;
+            entry[0].Size = (int)size;
+
+            return (void*)((IntPtr)ptr + Offset);
+        }
+
+        [UnmanagedCallersOnly]
+        internal static unsafe void CryptoFree(void* ptr)
+        {
+            if (ptr != null)
+            {
+                IntPtr entryPtr = (IntPtr)ptr - Offset;
+                if (ValidateMemory)
+                {
+                    lock (_allocations!)
+                    {
+                        {
+                            if (!_allocations!.Remove(entryPtr))
+                            {
+                                Environment.FailFast($"Failed to find OpenSSL memory 0x{(IntPtr)ptr:x}");
+                            }
+                        }
+                    }
+                }
+                if (_trackIncrementalAllocations)
+                {
+                    lock (_allocationsDiff!)
+                    {
+                        // this may fail as we may start tracking after given chunk was allocated
+                        _allocationsDiff!.Remove(entryPtr);
+                    }
+                }
+
+                Span<MemoryEntry> entry = new Span<MemoryEntry>((void*)entryPtr, 1);
+                Interlocked.Add(ref TotalAllocatedMemory, -((long)entry[0].Size));
+
+                NativeMemory.Free((void*)entryPtr);
+            }
+        }
     }
 }

src/libraries/System.Net.Security/src/System.Net.Security.csproj

@@ -469,6 +469,7 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetPlatformIdentifier)' != '' and '$(TargetPlatformIdentifier)' != 'windows'">
+    <Reference Include="System.Collections.Generic" />
     <Reference Include="System.Diagnostics.StackTrace" />
     <Reference Include="System.Security.Cryptography" />
     <Reference Include="System.Runtime.Numerics" />

src/native/libs/System.Security.Cryptography.Native/apibridge_30.h

@@ -6,6 +6,10 @@
 #pragma once
 #include "pal_types.h"
 
+typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line);
+typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file, int line);
+typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line);
+
 typedef struct evp_mac_st EVP_MAC;
 typedef struct evp_mac_ctx_st EVP_MAC_CTX;
 

src/native/libs/System.Security.Cryptography.Native/openssl.c

@@ -1467,7 +1467,7 @@
 static int32_t g_initStatus = 1;
 int g_x509_ocsp_index = -1;
 
-static int32_t EnsureOpenSslInitializedCore(void)
+static int32_t EnsureOpenSslInitializedCore(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction)
 {
     int ret = 0;
 
@@ -1476,7 +1476,16 @@
     // Otherwise call the 1.1 one.
 #ifdef FEATURE_DISTRO_AGNOSTIC_SSL
     InitializeOpenSSLShim();
+#endif
+
+    if (mallocFunction != NULL && reallocFunction != NULL && freefunction != NULL)
+    {
+        // This needs to be done before any allocation is done e.g. EnsureOpenSsl* is called.
+        // And it also needs to be after the pointers are loaded for AGNOSTIC_SSL
+        CRYPTO_set_mem_functions(mallocFunction, reallocFunction, freefunction);
+    }
 
+#ifdef FEATURE_DISTRO_AGNOSTIC_SSL
     if (API_EXISTS(SSL_state))
     {
         ret = EnsureOpenSsl10Initialized();
@@ -1501,15 +1510,22 @@
     return ret;
 }
 
+static CRYPTO_malloc_fn  _mallocFunction;
+static CRYPTO_realloc_fn _reallocFunction;
+static CRYPTO_free_fn _freefunction;
+
 static void EnsureOpenSslInitializedOnce(void)
 {
-    g_initStatus = EnsureOpenSslInitializedCore();
+    g_initStatus = EnsureOpenSslInitializedCore(_mallocFunction, _reallocFunction, _freefunction);
 }
 
 static pthread_once_t g_initializeShim = PTHREAD_ONCE_INIT;
 
-int32_t CryptoNative_EnsureOpenSslInitialized(void)
+int32_t CryptoNative_EnsureOpenSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction)
 {
+    _mallocFunction = mallocFunction;
+    _reallocFunction = reallocFunction;
+    _freefunction = freefunction;
     pthread_once(&g_initializeShim, EnsureOpenSslInitializedOnce);
     return g_initStatus;
 }

src/native/libs/System.Security.Cryptography.Native/openssl.h

@@ -69,7 +69,7 @@ PALEXPORT int32_t CryptoNative_GetRandomBytes(uint8_t* buf, int32_t num);
 
 PALEXPORT int32_t CryptoNative_LookupFriendlyNameByOid(const char* oidValue, const char** friendlyName);
 
-PALEXPORT int32_t CryptoNative_EnsureOpenSslInitialized(void);
+PALEXPORT int32_t CryptoNative_EnsureOpenSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction);
 
 PALEXPORT int64_t CryptoNative_OpenSslVersionNumber(void);
 

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -248,6 +248,7 @@ int EVP_DigestSqueeze(EVP_MD_CTX *ctx, unsigned char *out, size_t outlen);
     REQUIRED_FUNCTION(CRYPTO_malloc) \
     LEGACY_FUNCTION(CRYPTO_num_locks) \
     LEGACY_FUNCTION(CRYPTO_set_locking_callback) \
+    REQUIRED_FUNCTION(CRYPTO_set_mem_functions) \
     REQUIRED_FUNCTION(d2i_ASN1_BIT_STRING) \
     REQUIRED_FUNCTION(d2i_BASIC_CONSTRAINTS) \
     REQUIRED_FUNCTION(d2i_EXTENDED_KEY_USAGE) \
@@ -775,6 +776,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define CRYPTO_malloc CRYPTO_malloc_ptr
 #define CRYPTO_num_locks CRYPTO_num_locks_ptr
 #define CRYPTO_set_locking_callback CRYPTO_set_locking_callback_ptr
+#define CRYPTO_set_mem_functions CRYPTO_set_mem_functions_ptr
 #define d2i_ASN1_BIT_STRING d2i_ASN1_BIT_STRING_ptr
 #define d2i_BASIC_CONSTRAINTS d2i_BASIC_CONSTRAINTS_ptr
 #define d2i_EXTENDED_KEY_USAGE d2i_EXTENDED_KEY_USAGE_ptr

src/native/libs/System.Security.Cryptography.Native/pal_ssl.c

@@ -31,8 +31,6 @@ c_static_assert(TLSEXT_STATUSTYPE_ocsp == 1);
     "ECDHE-RSA-AES256-SHA384:" \
     "ECDHE-RSA-AES128-SHA256:" \
 
-int32_t CryptoNative_EnsureOpenSslInitialized(void);
-
 #ifdef NEED_OPENSSL_1_0
 static void EnsureLibSsl10Initialized(void)
 {
@@ -173,9 +171,11 @@ static void DetectCiphersuiteConfiguration(void)
 #endif
 }
 
-void CryptoNative_EnsureLibSslInitialized(void)
+
+void CryptoNative_EnsureLibSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction)
+
 {
-    CryptoNative_EnsureOpenSslInitialized();
+    CryptoNative_EnsureOpenSslInitialized(mallocFunction, reallocFunction, freefunction);
 
     // If portable, call the 1.0 initializer when needed.
     // If 1.0, call it statically.

src/native/libs/System.Security.Cryptography.Native/pal_ssl.h

@@ -130,7 +130,8 @@ typedef void (*SslCtxSetKeylogCallback)(const SSL* ssl, const char *line);
 /*
 Ensures that libssl is correctly initialized and ready to use.
 */
-PALEXPORT void CryptoNative_EnsureLibSslInitialized(void);
+
+PALEXPORT void CryptoNative_EnsureLibSslInitialized(CRYPTO_malloc_fn  mallocFunction, CRYPTO_realloc_fn reallocFunction, CRYPTO_free_fn freefunction);
 
 /*
 Shims the SSLv23_method method.