Fix x64 stub ABI and marshal-path sizing; dedupe memory readers

x64 call stub was ABI-broken: fixed 0x20 frame left rsp misaligned at the
inner call (callee entry rsp ≡ 0, ABI requires ≡ 8) and, for 5+ args, wrote
stack args over the return address. Compute frame K ≡ 8 (mod 16), K ≥
0x20 + 8*stackArgs, so the callee sees a 16-aligned stack and stack args land
above the shadow window. Load register args as full 64-bit imm64 (was imm32,
which truncated pointers > 4 GiB). BuildCallStub now takes nuint[]; x86 range-
checks each arg against uint.MaxValue instead of silently truncating.

MarshalCache conflated managed and unmanaged width in one Size field: the
blittable path needs Unsafe.SizeOf<T> (bool = 1) while the marshal path needs
Marshal.SizeOf<T> (inline ByValTStr/ByValArray expand past the managed
pointer). Add MarshalSize; MemoryBase picks per TypeRequiresMarshal at all four
IO sites. Prevents PtrToStructure/StructureToPtr from over-reading/overwriting
the pinned scratch buffer (heap corruption on write).

Extract shared RPM/WPM into RpmHelper: honor partial reads (dead Array.Resize
removed), consistent write-return semantics; InProcessReader now guards
MainModule like ExternalReader.

Tests: x64 frame-alignment property + inline-marshal round-trip added (both
fail against the pre-fix code); existing x64 byte-expectation tests updated to
the new frame. Build clean, 100/100 pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
kbe
2026-07-21 22:30:10 +02:00
co-authored by Claude Opus 4.8
parent 184dec86ca
commit 12b9b6c03e
7 changed files with 343 additions and 101 deletions
+46 -12
View File
@@ -6,7 +6,7 @@ namespace WhiteMagicTest;
/// <summary>
/// Tests for <see cref="MarshalCache{T}"/>: blittable size, marshal-required flag,
/// and computed-once behavior.
/// the separate MarshalSize field, and computed-once behavior.
/// </summary>
public class MarshalCacheTests
{
@@ -49,10 +49,10 @@ public class MarshalCacheTests
[Fact]
public void Size_for_struct_with_bool_is_managed_layout_width()
{
// Regression for the Marshal.SizeOf / Unsafe.SizeOf disagreement on a struct
// whose only field is `bool`: Marshal reports 4 bytes (Win32 BOOL default marshaling),
// but the blittable path (MemoryMarshal.Read<T>) actually lays out a bool as 1 byte.
// MarshalCache.Size must match what the reader actually touches.
// Regression for the Marshal vs. Unsafe size disagreement on a struct
// whose only field is `bool`: Marshal reports 4 (Win32 BOOL default
// marshaling), but the blittable path (MemoryMarshal.Read<T>) actually
// lays out a bool as 1 byte. MarshalCache.Size must match managed width.
Assert.Equal(Unsafe.SizeOf<SingleBoolStruct>(), MarshalCache<SingleBoolStruct>.Size);
Assert.Equal(1, MarshalCache<SingleBoolStruct>.Size);
}
@@ -60,8 +60,9 @@ public class MarshalCacheTests
[Fact]
public void Size_for_struct_with_bools_in_sequence_matches_managed_layout()
{
// Sequential struct { bool, bool } — managed width is 2, Marshal width is 8 (two BOOLs).
// The blittable path uses 1 byte per bool, so Size must equal the managed width.
// Sequential struct { bool, bool } — managed width is 2, Marshal width is 8
// (two BOOLs). The blittable path uses 1 byte per bool, so Size must equal
// the managed width.
Assert.Equal(Unsafe.SizeOf<SequentialBoolStruct>(), MarshalCache<SequentialBoolStruct>.Size);
Assert.Equal(2, MarshalCache<SequentialBoolStruct>.Size);
}
@@ -72,7 +73,6 @@ public class MarshalCacheTests
Assert.False(MarshalCache<int>.TypeRequiresMarshal);
Assert.False(MarshalCache<byte>.TypeRequiresMarshal);
Assert.False(MarshalCache<IntPtr>.TypeRequiresMarshal);
Assert.False(MarshalCache<IntPtr>.TypeRequiresMarshal);
}
[Fact]
@@ -84,8 +84,35 @@ public class MarshalCacheTests
[Fact]
public void TypeRequiresMarshal_is_true_for_reference_containing_types()
{
Assert.True(MarshalCache<string>.TypeRequiresMarshal);
Assert.True(MarshalCache<ClassWithInt>.TypeRequiresMarshal);
Assert.True(MarshalCache<InlineStrStruct>.TypeRequiresMarshal);
}
[Fact]
public void Inline_struct_with_MarshalAs_has_separate_MarshalSize()
{
// Regression for the marshal-path size bug. A struct with an inline
// ByValTStr field has mismatched managed and unmanaged widths: the managed
// width is just the pointer reference (8 bytes); the marshal unroller
// expands it into a 16-WCHAR inline buffer (32 bytes). MarshalCache.Size
// must match what the blittable path uses; MarshalCache.MarshalSize must
// match what the marshal path uses.
Assert.True(MarshalCache<InlineStrStruct>.TypeRequiresMarshal);
int expectedManaged = Unsafe.SizeOf<InlineStrStruct>(); // 8 (ptr)
int expectedMarshal = Marshal.SizeOf<InlineStrStruct>(); // 32 (16 WCHAR)
Assert.Equal(expectedManaged, MarshalCache<InlineStrStruct>.Size);
Assert.Equal(expectedMarshal, MarshalCache<InlineStrStruct>.MarshalSize);
Assert.NotEqual(MarshalCache<InlineStrStruct>.Size,
MarshalCache<InlineStrStruct>.MarshalSize);
}
[Fact]
public void MarshalSize_equals_Size_for_blittable_types()
{
// No interop expansion is needed when the type is blittable; both widths
// coincide.
Assert.Equal(MarshalCache<int>.Size, MarshalCache<int>.MarshalSize);
Assert.Equal(MarshalCache<IntPtr>.Size, MarshalCache<IntPtr>.MarshalSize);
Assert.Equal(MarshalCache<BlittableStruct>.Size, MarshalCache<BlittableStruct>.MarshalSize);
}
[Fact]
@@ -128,8 +155,15 @@ public class MarshalCacheTests
public bool B;
}
private class ClassWithInt
/// <summary>
/// A struct whose marshal layout carries an inline character buffer but
/// whose CLR managed layout is just a reference pointer. The canonical way
/// to exercise the marshal-vs-managed width split.
/// </summary>
[StructLayout(LayoutKind.Sequential)]
public struct InlineStrStruct
{
public int Value = 0;
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 16)]
public string Name;
}
}