Fix x64 stub ABI and marshal-path sizing; dedupe memory readers

x64 call stub was ABI-broken: fixed 0x20 frame left rsp misaligned at the
inner call (callee entry rsp ≡ 0, ABI requires ≡ 8) and, for 5+ args, wrote
stack args over the return address. Compute frame K ≡ 8 (mod 16), K ≥
0x20 + 8*stackArgs, so the callee sees a 16-aligned stack and stack args land
above the shadow window. Load register args as full 64-bit imm64 (was imm32,
which truncated pointers > 4 GiB). BuildCallStub now takes nuint[]; x86 range-
checks each arg against uint.MaxValue instead of silently truncating.

MarshalCache conflated managed and unmanaged width in one Size field: the
blittable path needs Unsafe.SizeOf<T> (bool = 1) while the marshal path needs
Marshal.SizeOf<T> (inline ByValTStr/ByValArray expand past the managed
pointer). Add MarshalSize; MemoryBase picks per TypeRequiresMarshal at all four
IO sites. Prevents PtrToStructure/StructureToPtr from over-reading/overwriting
the pinned scratch buffer (heap corruption on write).

Extract shared RPM/WPM into RpmHelper: honor partial reads (dead Array.Resize
removed), consistent write-return semantics; InProcessReader now guards
MainModule like ExternalReader.

Tests: x64 frame-alignment property + inline-marshal round-trip added (both
fail against the pre-fix code); existing x64 byte-expectation tests updated to
the new frame. Build clean, 100/100 pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
kbe
2026-07-21 22:30:10 +02:00
co-authored by Claude Opus 4.8
parent 184dec86ca
commit 12b9b6c03e
7 changed files with 343 additions and 101 deletions
+4 -4
View File
@@ -37,7 +37,7 @@ public abstract class MemoryBase : IDisposable
if (isRelative)
address = GetAbsolute(address);
int size = MarshalCache<T>.Size;
int size = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
byte[] raw = ReadBytes(address, size);
if (raw.Length < size)
@@ -56,7 +56,7 @@ public abstract class MemoryBase : IDisposable
if (isRelative)
address = GetAbsolute(address);
int size = MarshalCache<T>.Size;
int size = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
byte[] raw;
if (MarshalCache<T>.TypeRequiresMarshal)
@@ -81,7 +81,7 @@ public abstract class MemoryBase : IDisposable
if (isRelative)
address = GetAbsolute(address);
int elementSize = MarshalCache<T>.Size;
int elementSize = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
long totalSize = (long)elementSize * count;
ArgumentOutOfRangeException.ThrowIfGreaterThan(totalSize, int.MaxValue, nameof(count));
@@ -127,7 +127,7 @@ public abstract class MemoryBase : IDisposable
if (values is null || values.Length == 0)
return true;
int elementSize = MarshalCache<T>.Size;
int elementSize = MarshalCache<T>.TypeRequiresMarshal ? MarshalCache<T>.MarshalSize : MarshalCache<T>.Size;
long total = (long)elementSize * values.Length;
ArgumentOutOfRangeException.ThrowIfGreaterThan(total, int.MaxValue, nameof(values));
int totalSize = (int)total;