Fix x64 stub ABI and marshal-path sizing; dedupe RPM readers

x64 call stub was ABI-broken: fixed 0x20 frame left rsp misaligned at the
inner call (callee entry rsp ≡ 0, ABI requires ≡ 8) and, for 5+ args, wrote
stack args over the return address. Compute frame K ≡ 8 (mod 16), K ≥
0x20 + 8*stackArgs, so the callee sees a 16-aligned stack and stack args land
above the shadow window. Load register args as full 64-bit imm64 (was imm32,
which truncated pointers > 4 GiB). BuildCallStub now takes nuint[]; x86 range-
checks each arg against uint.MaxValue instead of silently truncating.

MarshalCache conflated managed and unmanaged width in one Size field: the
blittable path needs Unsafe.SizeOf<T> (bool = 1) while the marshal path needs
Marshal.SizeOf<T> (inline ByValTStr/ByValArray expand past the managed
pointer). Add MarshalSize; MemoryBase picks per TypeRequiresMarshal at all four
IO sites. Prevents PtrToStructure/StructureToPtr from over-reading/overwriting
the pinned scratch buffer (heap corruption on write).

Extract shared RPM/WPM into RpmHelper: honor partial reads (dead Array.Resize
removed), consistent write-return semantics; InProcessReader now guards
MainModule like ExternalReader.

Tests: x64 frame-alignment property + inline-marshal round-trip added (both
fail against the pre-fix code); existing x64 byte-expectation tests updated to
the new frame. Build clean, 100/100 pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
kbe
2026-07-21 21:53:22 +02:00
co-authored by Claude Opus 4.8
parent 6fa12d8667
commit cb437ef9b5
7 changed files with 618 additions and 137 deletions
+104 -33
View File
@@ -46,21 +46,35 @@ public sealed class StubAssembler : IAssembler
/// </summary>
/// <param name="stubAddress">Where the stub lands (for E8 rel32 encoding).</param>
/// <param name="targetAddress">Function to call.</param>
/// <param name="arguments">Argument values (uint[] — each 4 or 8 bytes per pointerSize).</param>
/// <param name="arguments">Argument values. For x86 each element holds a 32-bit argument;
/// for x64 each element holds the full 64-bit pointer-sized argument.</param>
/// <param name="pointerSize">4 (x86) or 8 (x64).</param>
/// <param name="convention">Calling convention.</param>
/// <param name="convention">Calling convention (ignored on x64; Windows has a single ABI).</param>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="pointerSize"/> is not 4 or 8,
/// or <paramref name="convention"/> is not known, or the distance between stub and target
/// exceeds the E8 rel32 range.</exception>
public byte[] BuildCallStub(IntPtr stubAddress, IntPtr targetAddress,
uint[] arguments, int pointerSize, CallConvention convention)
nuint[] arguments, int pointerSize, CallConvention convention)
{
var buffer = new List<byte>(64);
var buffer = new List<byte>(96);
if (pointerSize == 4)
{
// X86 args are 32-bit. Truncate nuint down to uint — callers must pass values
// that fit in 32 bits on x86 targets.
uint[] args32 = new uint[arguments.Length];
for (int i = 0; i < arguments.Length; i++)
{
ulong v = arguments[i];
if (v > uint.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(arguments),
$"Argument {i} = 0x{v:X} does not fit in 32 bits (x86 target).");
}
args32[i] = (uint)v;
}
BuildX86Stub(buffer, checked((uint)stubAddress), checked((uint)targetAddress),
arguments, convention);
args32, convention);
}
else if (pointerSize == 8)
{
@@ -153,56 +167,113 @@ public sealed class StubAssembler : IAssembler
buffer.Add(0xC3); // ret
}
/// <summary>
/// Builds a Windows x64 call stub that conforms to the Microsoft x64 ABI:
/// first 4 integer/pointer args in RCX, RDX, R8, R9 (64-bit loads); stack args
/// above a 32-byte shadow space; 16-byte stack alignment at the inner <c>call</c>.
/// </summary>
/// <remarks>
/// <para>Frame derivation. The ABI requires the <em>inner</em> <c>call</c> site to
/// land with call-site rsp ≡ 0 (mod 16), so that <c>call</c> pushes 8 bytes and the
/// callee sees entry rsp ≡ 8 — the value an MSVC prologue (<c>push rbp; sub rsp, 0x20</c>)
/// expects, and the only value for which locals land 16-aligned (SSE-safe).</para>
/// <list type="bullet">
/// <item>Stub entry: rsp ≡ 8 (mod 16).</item>
/// <item>Need post-sub rsp ≡ 0 → sub operand K satisfies K ≡ 8 (mod 16).</item>
/// <item>Frame must hold shadow space (0x20) + stack args (8 bytes each for args 4+).
/// Choose the smallest such K: <c>K = frameBytes + ((8 frameBytes) mod 16 + 16) mod 16</c>.
/// For 05 args, K ∈ {0x28, 0x38}; pattern scales linearly.</item>
/// </list>
/// <code>
/// sub rsp, K ; K ≡ 8 (mod 16), K ≥ 0x20 + 8·stackArgs
/// mov rcx, arg0 ; REX.W + imm64 (10 bytes)
/// mov rdx, arg1 ; REX.W + imm64 (10 bytes)
/// mov r8, arg2 ; REX.WB+ imm64 (10 bytes, REX.R)
/// mov r9, arg3 ; REX.WB+ imm64 (10 bytes, REX.R)
/// mov rax, arg[N] ; REX.W + imm64 (10 bytes)
/// mov [rsp + 0x20 + 8*(N-4)], rax (5/8 bytes)
/// call target (rel32) ( 5 bytes)
/// add rsp, K ( 7 bytes)
/// ret ( 1 byte)
/// </code>
/// </remarks>
private void BuildX64Stub(List<byte> buffer, ulong stubAddr,
ulong target, uint[] args)
ulong target, nuint[] args)
{
// Windows x64 single ABI: first 4 args in RCX, RDX, R8D, R9D.
ulong current = stubAddr;
var regCodes = new byte[] { 0xB9, 0xBA, 0xB8, 0xB9 };
var rexBytes = new byte[] { 0x00, 0x00, 0x41, 0x41 };
// Compute frame size K. K ≡ 8 (mod 16) so that the inner call sees
// post-sub rsp ≡ 0 and delivers target entry rsp ≡ 8 (mod 16).
int stackArgs = Math.Max(0, args.Length - 4);
int frameBytes = 0x20 + 8 * stackArgs;
int k = frameBytes + ((8 - (frameBytes % 16) + 16) % 16);
// sub rsp, imm32 (always imm32 form — constant 7 bytes regardless of K).
buffer.Add(0x48); buffer.Add(0x81); buffer.Add(0xEC);
EmitU32(buffer, (uint)k);
current += 7;
// 64-bit register loads for args 0..3. All encodings are exactly 10 bytes:
// REX.W (0x48) + 0xB9 + imm64 → mov rcx, imm64
// REX.W (0x48) + 0xBA + imm64 → mov rdx, imm64
// REX.WB(0x49) + 0xB8 + imm64 → mov r8, imm64 (REX.R for r8)
// REX.WB(0x49) + 0xB9 + imm64 → mov r9, imm64 (REX.R)
byte[][] regMoves =
[
[0x48, 0xB9],
[0x48, 0xBA],
[0x49, 0xB8],
[0x49, 0xB9],
];
int regCount = Math.Min(args.Length, 4);
for (int i = 0; i < regCount; i++)
{
if (rexBytes[i] != 0)
buffer.Add(rexBytes[i]);
buffer.Add(regCodes[i]);
EmitU32(buffer, args[i]);
current += (rexBytes[i] != 0 ? 6u : 5u);
byte[] prefix = regMoves[i];
buffer.Add(prefix[0]);
buffer.Add(prefix[1]);
EmitU64(buffer, args[i]);
current += (uint)(prefix.Length + 8);
}
// Push remaining args in reverse order
for (int i = args.Length - 1; i >= 4; i--)
// Stack args: written at [post-sub-rsp + 0x20 + 8*(i-4)], i.e. above the
// shadow window, where the inner call's callee expects them.
for (int i = 4; i < args.Length; i++)
{
current += 5;
buffer.Add(0x68);
EmitU32(buffer, args[i]);
int offset = 0x20 + (i - 4) * 8;
buffer.Add(0x48); buffer.Add(0xB8); // mov rax, imm64
EmitU64(buffer, args[i]);
current += 10;
buffer.Add(0x48); buffer.Add(0x89); // mov [rsp + disp], rax
if (offset <= 127)
{
buffer.Add(0x44); buffer.Add(0x24); // ModRM: [rsp + disp8]
buffer.Add((byte)offset);
current += 5;
}
else
{
buffer.Add(0x84); buffer.Add(0x24); // ModRM: [rsp + disp32]
EmitU32(buffer, (uint)offset);
current += 8;
}
}
// call rel32
long distance = (long)target - (long)(current + 5);
if (distance < int.MinValue || distance > int.MaxValue)
if (distance is < int.MinValue or > int.MaxValue)
{
throw new ArgumentOutOfRangeException(
"target and stub are >2 GiB apart; E8 rel32 cannot encode this distance.");
}
buffer.Add(0xE8);
EmitU32(buffer, (uint)distance);
current += 5;
// Pop any args pushed on stack (x64 is caller-clean)
int stackArgs = args.Length > 4 ? args.Length - 4 : 0;
if (stackArgs > 0)
{
int bytes = stackArgs * 8;
buffer.Add(0x48); // REX.W
buffer.Add(bytes <= 127 ? (byte)0x83 : (byte)0x81); // add r/m64, imm8/imm32
buffer.Add(0xC4); // rsp
if (bytes <= 127)
buffer.Add((byte)bytes);
else
EmitU32(buffer, (uint)bytes);
}
// Tear down the frame symmetrically.
buffer.Add(0x48); buffer.Add(0x81); buffer.Add(0xC4);
EmitU32(buffer, (uint)k);
buffer.Add(0xC3);
}
+64 -45
View File
@@ -1,88 +1,107 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace WhiteMagic;
/// <summary>
/// Computes and caches marshal-related metadata for type <typeparamref name="T"/>
/// exactly once. <see cref="MemoryBase.Read{T}"/> and <see cref="MemoryBase.Write{T}"/>
/// branch on these cached flags to decide between blittable <c>Span</c>/<c>MemoryMarshal</c>
/// paths and the fallback marshal path.
/// Caches the widths marshalling decisions for type <typeparamref name="T"/>
/// once, at static-constructor time. <see cref="MemoryBase.Read{T}"/> and
/// <see cref="MemoryBase.Write{T}"/> branch on <see cref="TypeRequiresMarshal"/>
/// and pick the appropriate width from this cache.
/// </summary>
/// <typeparam name="T">The type to cache metadata for.</typeparam>
public static class MarshalCache<T>
{
/// <summary>The unmanaged size of <typeparamref name="T"/> in bytes.</summary>
/// <summary>
/// The blittable (managed layout) width of <typeparamref name="T"/>. This is
/// what <see cref="MemoryMarshal.Read{T}"/> / <see cref="MemoryMarshal.Write{T}"/>
/// actually consume. Equals <see cref="Unsafe.SizeOf{T}"/> in the general case,
/// with fixed-width overrides for <see cref="bool"/>, <see cref="char"/>, and
/// enums so the cache value matches the primitive layout width used by those
/// paths.
/// </summary>
public static readonly int Size;
/// <summary>The unmanaged size of <typeparamref name="T"/> as an unsigned integer.</summary>
public static readonly uint SizeU;
/// <summary>
/// The unmanaged (interop) width via <see cref="Marshal.SizeOf"/>. The marshal
/// path (<see cref="Marshal.PtrToStructure"/>/<see cref="Marshal.StructureToPtr"/>)
/// reads/writes this many bytes. Exceeds <see cref="Size"/> whenever a struct
/// carries inline unmanaged data that the marshaler expands — inline
/// <c>ByValTStr</c>/<c>ByValArray</c> buffers, <c>bool</c> fields (4 bytes per
/// default Win32 BOOL marshaling vs 1 byte managed), etc. For types that do
/// not go through the marshal path, this field is still populated but unused
/// by MemoryBase.
/// </summary>
/// <remarks>
/// <c>Marshal.SizeOf</c> throws for some reference-containing shapes (e.g. a
/// bare <see cref="string"/>). When that happens, we fall back to
/// <see cref="Size"/> — the fallback path is unreachable from production code
/// because types with a reference field always have
/// <see cref="TypeRequiresMarshal"/> true, so MemoryBase reads this field only
/// when it is known to be populated.
/// </remarks>
public static readonly int MarshalSize;
/// <summary>
/// <see langword="true"/> when <typeparamref name="T"/> cannot be copied through the
/// blittable <see cref="System.Runtime.InteropServices.MemoryMarshal"/> path and must
/// use <see cref="Marshal.PtrToStructure"/>/<see cref="Marshal.StructureToPtr"/> instead.
/// This is the case when a top-level field carries <see cref="MarshalAsAttribute"/>, or
/// when <typeparamref name="T"/> contains a managed reference
/// (<see cref="System.Runtime.CompilerServices.RuntimeHelpers.IsReferenceOrContainsReferences{T}"/>).
/// <see langword="true"/> when <typeparamref name="T"/> cannot be copied through
/// the blittable <see cref="System.Runtime.InteropServices.MemoryMarshal"/> path
/// and must fall back to <see cref="Marshal.PtrToStructure"/> /
/// <see cref="Marshal.StructureToPtr"/>. This is the case when a top-level field
/// carries <see cref="MarshalAsAttribute"/>, or when <typeparamref name="T"/>
/// contains a managed reference
/// (<see cref="RuntimeHelpers.IsReferenceOrContainsReferences{T}"/>).
/// </summary>
/// <remarks>
/// The <see cref="MarshalAsAttribute"/> check inspects only top-level fields; a
/// <see cref="MarshalAsAttribute"/> on a field of a nested struct is not detected.
/// Reference-containing nested structs are still caught, because the reference check
/// propagates through nested value types.
/// <see cref="MarshalAsAttribute"/> on a field of a nested struct is not
/// detected. Reference-containing nested structs are still caught, because the
/// reference check propagates through nested value types.
/// </remarks>
public static readonly bool TypeRequiresMarshal;
/// <summary><see langword="true"/> when <typeparamref name="T"/> is <see cref="IntPtr"/>.</summary>
public static readonly bool IsIntPtr;
/// <summary>The underlying type code of <typeparamref name="T"/>.</summary>
public static readonly TypeCode TypeCode;
/// <summary>
/// The effective type that the marshaler uses. For an enum this is the underlying
/// integer type; for all other types it is <typeparamref name="T"/> itself.
/// </summary>
public static readonly Type RealType;
static MarshalCache()
{
TypeCode = Type.GetTypeCode(typeof(T));
if (typeof(T) == typeof(bool))
{
Size = 1;
RealType = typeof(T);
}
else if (typeof(T) == typeof(char))
{
// Marshal.SizeOf(char) is 1 (ANSI), but the blittable path reads/writes a
// char as a 2-byte UTF-16 code unit. Size must match the blittable width.
// Marshal.SizeOf<char> reports 1 (ANSI char), but the blittable
// MemoryMarshal path reads/writes a char as a 2-byte UTF-16 code unit.
Size = 2;
RealType = typeof(T);
}
else if (typeof(T).IsEnum)
{
Type underlying = typeof(T).GetEnumUnderlyingType();
Size = Marshal.SizeOf(underlying);
RealType = underlying;
TypeCode = Type.GetTypeCode(underlying);
Size = Marshal.SizeOf(typeof(T).GetEnumUnderlyingType());
}
else
{
Size = Marshal.SizeOf(typeof(T));
RealType = typeof(T);
// The blittable path goes through MemoryMarshal, which uses the CLR
// managed layout. Use Unsafe.SizeOf<T> so Size agrees with that
// layout — Marshal.SizeOf<T> disagrees when a struct contains a
// `bool` field (unmanaged 4 vs managed 1).
Size = Unsafe.SizeOf<T>();
}
SizeU = (uint)Size;
IsIntPtr = RealType == typeof(IntPtr);
bool hasMarshalAsField =
RealType.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
typeof(T).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
.Any(f => f.GetCustomAttributes(typeof(MarshalAsAttribute), true).Length != 0);
TypeRequiresMarshal =
hasMarshalAsField || System.Runtime.CompilerServices.RuntimeHelpers.IsReferenceOrContainsReferences<T>();
hasMarshalAsField || RuntimeHelpers.IsReferenceOrContainsReferences<T>();
// MarshalSize is only consulted when TypeRequiresMarshal is true; for the
// rare case where Marshal.SizeOf refuses a shape (ref-containing structs),
// fall back to the managed size so the field stays populated.
try
{
MarshalSize = Marshal.SizeOf<T>();
}
catch (ArgumentException)
{
MarshalSize = Size;
}
}
}
+4 -4
View File
@@ -37,7 +37,7 @@ public abstract class MemoryBase : IDisposable
if (isRelative)
address = GetAbsolute(address);
int size = MarshalCache<T>.Size;
int size = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
byte[] raw = ReadBytes(address, size);
if (raw.Length < size)
@@ -56,7 +56,7 @@ public abstract class MemoryBase : IDisposable
if (isRelative)
address = GetAbsolute(address);
int size = MarshalCache<T>.Size;
int size = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
byte[] raw;
if (MarshalCache<T>.TypeRequiresMarshal)
@@ -81,7 +81,7 @@ public abstract class MemoryBase : IDisposable
if (isRelative)
address = GetAbsolute(address);
int elementSize = MarshalCache<T>.Size;
int elementSize = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
long totalSize = (long)elementSize * count;
ArgumentOutOfRangeException.ThrowIfGreaterThan(totalSize, int.MaxValue, nameof(count));
@@ -127,7 +127,7 @@ public abstract class MemoryBase : IDisposable
if (values is null || values.Length == 0)
return true;
int elementSize = MarshalCache<T>.Size;
int elementSize = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
long total = (long)elementSize * values.Length;
ArgumentOutOfRangeException.ThrowIfGreaterThan(total, int.MaxValue, nameof(values));
int totalSize = (int)total;
+61
View File
@@ -0,0 +1,61 @@
using System.Runtime.InteropServices;
using WhiteMagic.Native;
namespace WhiteMagic;
/// <summary>
/// Shared ReadProcessMemory / WriteProcessMemory wrappers used by both
/// <see cref="ExternalReader"/> and <see cref="InProcessReader"/>. Kept in a single
/// location to keep the two readers byte-for-byte consistent on partial-read handling,
/// write-return semantics, and failure modes.
/// </summary>
internal static class RpmHelper
{
/// <summary>
/// Reads up to <paramref name="count"/> bytes from <paramref name="address"/> in
/// the process identified by <paramref name="handle"/>. Returns:
/// <list type="bullet">
/// <item>An empty array if <see cref="NativeMethods.ReadProcessMemory"/> fails and
/// reports zero bytes read.</item>
/// <item>A truncated array of exactly <c>bytesRead</c> bytes when the call returns
/// <see langword="false"/> but the OS has placed a partial copy in the buffer
/// (for example, <c>ERROR_PARTIAL_COPY</c>).</item>
/// <item>The full buffer on success.</item>
/// </list>
/// </summary>
public static byte[] ReadBytes(SafeMemoryHandle handle, IntPtr address, int count)
{
byte[] buffer = new byte[count];
bool ok = NativeMethods.ReadProcessMemory(handle, address, buffer, count, out nint bytesRead);
if (!ok && bytesRead == 0)
{
return [];
}
if ((int)bytesRead < count)
{
// Either a successful short read, or a failed-but-partial RPM. In both
// cases honor the bytes the OS actually produced rather than padding.
byte[] partial = new byte[(int)bytesRead];
Buffer.BlockCopy(buffer, 0, partial, 0, (int)bytesRead);
return partial;
}
return buffer;
}
/// <summary>
/// Writes <paramref name="bytes"/> to <paramref name="address"/> in the process
/// identified by <paramref name="handle"/>. Returns the number of bytes actually
/// written, or 0 on total failure.
/// </summary>
public static int WriteBytes(SafeMemoryHandle handle, IntPtr address, ReadOnlySpan<byte> bytes)
{
if (!NativeMethods.WriteProcessMemory(handle, address, bytes, bytes.Length, out nint written))
{
return 0;
}
return (int)written;
}
}