Fix dispatcher crash/deadlock, instruction analyzer, cache equality, task leak, and redirection protection

- MainThreadDispatcher: guard DispatchHook with try/catch so exceptions never escape to native caller; drain and fault pending work on Dispose; synchronize Execute/ExecuteAsync/Dispose against race/dispose.
- InstructionAnalyzer: require ModRM 0xEC for 0x83/0x81 sub-esp/rsp forms, rejecting unsafe RIP-relative or memory forms.
- PatternScannerCache: implement value equality on CacheKey so repeated scans actually hit cache.
- BackgroundTaskExecutor: add remote allocations to the free list immediately after VirtualAllocEx, before any write that could fail and leak.
- redirect: capture and restore original page protection in Apply/Remove instead of leaving target RWX.
- Regression tests for all six fixes.

Tests: 198 passing, 4 integration/interactive skipped.
This commit is contained in:
kbe
2026-07-22 00:32:19 +02:00
parent 3f0bea6bd4
commit 44a368de9c
9 changed files with 301 additions and 43 deletions
@@ -1,3 +1,5 @@
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using WhiteMagic;
using WhiteMagic.Discovery;
@@ -43,6 +45,48 @@ public class PatternScannerCacheTests
Assert.Equal(addr + 30, first);
Assert.Equal(first, second);
// Value equality must mean the second call reused the cached entry.
var cacheField = typeof(PatternScannerCache).GetField("_cache", BindingFlags.NonPublic | BindingFlags.Instance)!;
var cacheDict = cacheField.GetValue(cache)!;
int count = (int)cacheDict.GetType().GetProperty("Count")!.GetValue(cacheDict)!;
Assert.Equal(1, count);
}
finally
{
pin.Free();
}
}
[Fact]
public void FindCached_value_equality_uses_content_not_reference()
{
using var reader = CreateReader();
var cache = new PatternScannerCache(reader);
byte[] buffer = new byte[256];
buffer[10] = 0xAA;
buffer[11] = 0xBB;
GCHandle pin = GCHandle.Alloc(buffer, GCHandleType.Pinned);
try
{
IntPtr addr = pin.AddrOfPinnedObject();
IntPtr end = addr + buffer.Length;
byte[] pattern1 = { 0xAA, 0xBB };
byte[] pattern2 = { 0xAA, 0xBB };
IntPtr first = cache.FindCached(pattern1, null, addr, end);
IntPtr second = cache.FindCached(pattern2, null, addr, end);
Assert.Equal(addr + 10, first);
Assert.Equal(first, second);
var cacheField = typeof(PatternScannerCache).GetField("_cache", BindingFlags.NonPublic | BindingFlags.Instance)!;
var cacheDict = cacheField.GetValue(cache)!;
int count = (int)cacheDict.GetType().GetProperty("Count")!.GetValue(cacheDict)!;
Assert.Equal(1, count);
}
finally
{