kbe/whitemagic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix x64 stub ABI and marshal-path sizing; dedupe memory readers

Browse Source 
x64 call stub was ABI-broken: fixed 0x20 frame left rsp misaligned at the
inner call (callee entry rsp ≡ 0, ABI requires ≡ 8) and, for 5+ args, wrote
stack args over the return address. Compute frame K ≡ 8 (mod 16), K ≥
0x20 + 8*stackArgs, so the callee sees a 16-aligned stack and stack args land
above the shadow window. Load register args as full 64-bit imm64 (was imm32,
which truncated pointers > 4 GiB). BuildCallStub now takes nuint[]; x86 range-
checks each arg against uint.MaxValue instead of silently truncating.

MarshalCache conflated managed and unmanaged width in one Size field: the
blittable path needs Unsafe.SizeOf<T> (bool = 1) while the marshal path needs
Marshal.SizeOf<T> (inline ByValTStr/ByValArray expand past the managed
pointer). Add MarshalSize; MemoryBase picks per TypeRequiresMarshal at all four
IO sites. Prevents PtrToStructure/StructureToPtr from over-reading/overwriting
the pinned scratch buffer (heap corruption on write).

Extract shared RPM/WPM into RpmHelper: honor partial reads (dead Array.Resize
removed), consistent write-return semantics; InProcessReader now guards
MainModule like ExternalReader.

Tests: x64 frame-alignment property + inline-marshal round-trip added (both
fail against the pre-fix code); existing x64 byte-expectation tests updated to
the new frame. Build clean, 100/100 pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
kbe
2026-07-21 22:30:10 +02:00
co-authored by Claude Opus 4.8
parent 184dec86ca
commit 12b9b6c03e
7 changed files with 343 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
WhiteMagicTest/StubAssemblerTests.cs
+88 -23
View File
@@ -136,19 +136,19 @@ public class StubAssemblerTests
// ── x64 (Microsoft x64 ABI — shadow space + 16-byte alignment + 64-bit loads) ──
//
// Stub frame layout:
// bytes 0..6 sub rsp, 0x20 (7 bytes — shadow space + realignment)
// bytes 0..6 sub rsp, K (7 bytes — K = 32 + 8·stackArgs, rounded so K ≡ 8 mod 16)
// bytes 7..N mov r64, imm64 ... (10 bytes per reg move: 2-byte prefix + 8-byte imm)
// mov rax, imm64 / mov [rsp+0x20+8*(i-4)], rax for stack args (15 bytes each)
// E8 rel32 call target (5 bytes)
// 48 81 C4 20 00... add rsp, 0x20 (7 bytes)
// 48 81 C4 K 00... add rsp, K (7 bytes)
// C3 ret (1 byte)
//
// Each `mov rNN, imm64` is 10 bytes regardless of the target register:
// RCX REX.W 0xB9 + 8 imm (0x48 0xB9)
// RDX REX.W 0xBA + 8 imm (0x48 0xBA)
// R8 REX.WB 0xB8 + 8 imm (0x49 0xB8 — REX.W|R = 0x49)
// R9 REX.WB 0xB9 + 8 imm (0x49 0xB9)
// RAX REX.W 0xB8 + 8 imm (0x48 0xB8)
// RCX REX.W+opcode B9 (0x48 0xB9)
// RDX REX.W+opcode BA (0x48 0xBA)
// R8 REX.WB+opcode B8 (0x49 0xB8, REX.R needed for r8)
// R9 REX.WB+opcode B9 (0x49 0xB9, REX.R needed for r9)
// RAX REX.W+opcode B8 (0x48 0xB8)
[Fact]
public void X64_0args_allocates_shadow_space_and_aligns()
@@ -158,15 +158,16 @@ public class StubAssemblerTests
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, [], 8, CallConvention.Cdecl);
// sub (7) + call (5) + add (7) + ret (1) = 20
// K = 0x20 + 0·8 = 0x20; round up to ≡ 8 mod 16 → K = 0x28.
// Frame = sub(7) + call(5) + add(7) + ret(1) = 20
Assert.Equal(20, stub.Length);
uint rel = (uint)(t - (a + 7 + 5)); // = t - a - 12
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x20
Assert.Equal([0x48, 0x81, 0xEC, 0x28, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x28 (K ≡ 8 mod 16)
Assert.Equal(0xE8, stub[7]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 8));
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[12..19]); // add rsp, 0x20
Assert.Equal([0x48, 0x81, 0xC4, 0x28, 0x00, 0x00, 0x00], stub[12..19]); // add rsp, 0x28
Assert.Equal(0xC3, stub[19]); // ret
}
@@ -178,12 +179,12 @@ public class StubAssemblerTests
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, [0xAABBCCDDu], 8, CallConvention.Cdecl);
// sub (7) + mov rcx, imm64 (10) + call (5) + add (7) + ret (1) = 30
// K = 0x28. Total = sub(7) + mov(10) + call(5) + add(7) + ret(1) = 30
Assert.Equal(30, stub.Length);
uint rel = (uint)(t - (a + 7 + 10 + 5)); // = t - a - 22
// stub[0..6] = sub rsp, 0x20
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]);
// stub[0..6] = sub rsp, 0x28 (K ≡ 8 mod 16, 16-aligned call-site for SSE safety)
Assert.Equal([0x48, 0x81, 0xEC, 0x28, 0x00, 0x00, 0x00], stub[..7]);
// stub[7..16] = mov rcx, 0x00000000_AABBCCDD (zero-extended)
Assert.Equal(0x48, stub[7]); Assert.Equal(0xB9, stub[8]);
Assert.Equal(0xDD, stub[9]); Assert.Equal(0xCC, stub[10]);
@@ -193,8 +194,8 @@ public class StubAssemblerTests
// stub[17..21] = call rel32
Assert.Equal(0xE8, stub[17]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 18));
// stub[22..28] = add rsp, 0x20
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[22..29]);
// stub[22..28] = add rsp, 0x28
Assert.Equal([0x48, 0x81, 0xC4, 0x28, 0x00, 0x00, 0x00], stub[22..29]);
Assert.Equal(0xC3, stub[29]); // ret
}
@@ -207,12 +208,12 @@ public class StubAssemblerTests
(IntPtr)(nint)a, (IntPtr)(nint)t,
[0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u], 8, CallConvention.Cdecl);
// sub (7) + 4 x mov (4*10=40) + call (5) + add (7) + ret (1) = 60
// K = 0x28. Total = sub(7) + 4×mov(40) + call(5) + add(7) + ret(1) = 60
Assert.Equal(60, stub.Length);
uint rel = (uint)(t - (a + 7 + 40 + 5)); // = t - a - 52
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x20
Assert.Equal([0x48, 0x81, 0xEC, 0x28, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x28
// mov rcx, 0x11111111 (48 B9 + 8 imm) at [7..16]
Assert.Equal(0x48, stub[7]); Assert.Equal(0xB9, stub[8]);
@@ -246,8 +247,8 @@ public class StubAssemblerTests
Assert.Equal(0xE8, stub[47]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 48));
// add rsp, 0x20 at [52..58]
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[52..59]);
// add rsp, 0x28 at [52..58]
Assert.Equal([0x48, 0x81, 0xC4, 0x28, 0x00, 0x00, 0x00], stub[52..59]);
// ret at [59]
Assert.Equal(0xC3, stub[59]);
}
@@ -261,11 +262,12 @@ public class StubAssemblerTests
(IntPtr)(nint)a, (IntPtr)(nint)t,
[(nuint)1, (nuint)2, (nuint)3, (nuint)4, (nuint)5], 8, CallConvention.Cdecl);
// sub (7) + 4 reg moves (40) + stack arg (mov rax 10 + mov [rsp+0x20],rax 5 = 15)
// K = 0x20 + 1·8 = 0x28. Round-up rule: 0x28 % 16 = 8 → no extra padding.
// sub(7) + 4 reg moves (40) + stack arg (mov rax 10 + mov [rsp+0x20],rax 5 = 15)
// + call (5) + add (7) + ret (1) = 75
Assert.Equal(75, stub.Length);
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]);
Assert.Equal([0x48, 0x81, 0xEC, 0x28, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x28 (K=0x20+8=0x28, 0x28 % 16 = 8 ✓)
// mov rcx, 1 at [7..16]
Assert.Equal(0x48, stub[7]); Assert.Equal(0xB9, stub[8]);
@@ -296,12 +298,75 @@ public class StubAssemblerTests
Assert.Equal(0xE8, stub[62]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 63));
// add rsp, 0x20 at [67..73]
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[67..74]);
// add rsp, 0x28 at [67..73]
Assert.Equal([0x48, 0x81, 0xC4, 0x28, 0x00, 0x00, 0x00], stub[67..74]);
// ret at [74]
Assert.Equal(0xC3, stub[74]);
}
[Fact]
public void X64_frame_alignment_property_for_arg_counts()
{
// ABI invariant: for every arg count the sub operand K must satisfy
// K ≡ 8 (mod 16), and the same K must appear in the matching 'add rsp, K'
// just before the ret. Violating this misaligns the inner call, which
// #GP-faults the next time an SSE-using callee executes movaps/movdqa.
var s = Create();
// Keep stub/target within E8 rel32 range (< 2 GiB) so the property check
// exercises the frame math, not the distance guard.
ulong a = 0x140000000, t = 0x140100000;
for (int argc = 0; argc <= 12; argc++)
{
nuint[] args = new nuint[argc];
for (int i = 0; i < argc; i++) args[i] = (nuint)(i + 1);
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, args, 8, CallConvention.Cdecl);
// sub rsp, imm32: 48 81 EC K0 K1 K2 K3
Assert.Equal(0x48, stub[0]);
Assert.Equal(0x81, stub[1]);
Assert.Equal(0xEC, stub[2]);
uint subK = BitConverter.ToUInt32(stub, 3);
Assert.True(subK % 16 == 8,
$"argc={argc}: sub K=0x{subK:X} must satisfy K % 16 == 8");
// add rsp, imm32 is 7 bytes immediately before the trailing C3
int last = stub.Length - 1;
Assert.Equal(0xC3, stub[last]);
int addIdx = last - 7;
Assert.Equal(0x48, stub[addIdx]);
Assert.Equal(0x81, stub[addIdx + 1]);
Assert.Equal(0xC4, stub[addIdx + 2]);
uint addK = BitConverter.ToUInt32(stub, addIdx + 3);
Assert.True(subK == addK,
$"argc={argc}: add K=0x{addK:X} must match sub K=0x{subK:X}");
}
}
[Fact]
public void X64_6args_frame_grows_to_0x38()
{
// Six args: frameBytes = 0x20 + 2·8 = 0x30. 0x30 % 16 = 0, so the
// pad-to-≡8 rule adds 8 more bytes → K = 0x38. Args 5 and 6 still live
// at [rsp+0x20] and [rsp+0x28]; the extra 8 bytes of padding at [rsp+0x30]
// are unused but necessary for alignment.
var s = Create();
ulong a = 0x100000000, t = 0x123456788;
nuint[] args = [(nuint)1, (nuint)2, (nuint)3, (nuint)4, (nuint)5, (nuint)6];
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, args, 8, CallConvention.Cdecl);
// sub rsp, 0x38 (7 bytes)
Assert.Equal([0x48, 0x81, 0xEC, 0x38, 0x00, 0x00, 0x00], stub[..7]);
// add rsp, 0x38 occupies the 7 bytes immediately before ret
int addIdx = stub.Length - 8;
Assert.Equal([0x48, 0x81, 0xC4, 0x38, 0x00, 0x00, 0x00], stub[addIdx..(addIdx + 7)]);
Assert.Equal(0xC3, stub[stub.Length - 1]);
}
[Fact]
public void X64_full_64bit_args_are_preserved_not_truncated()
{