Files
whitemagic/WhiteMagicTest/StubAssemblerTests.cs
T
2026-07-21 22:30:10 +02:00

394 lines
17 KiB
C#

using WhiteMagic.Assembly;
namespace WhiteMagicTest;
public class StubAssemblerTests
{
private static StubAssembler Create() => new();
// ── Emit primitives ────────────────────────────────────────────────────
[Fact] public void EmitU8_appends_a_single_byte() { var s=Create(); var b=new List<byte>(); s.EmitU8(b,0xAB); Assert.Equal([0xAB],b); }
[Fact] public void EmitU32_appends_little_endian() { var s=Create(); var b=new List<byte>(); s.EmitU32(b,0x11223344); Assert.Equal([0x44,0x33,0x22,0x11],b); }
[Fact] public void EmitU32_appends_zero() { var s=Create(); var b=new List<byte>(); s.EmitU32(b,0); Assert.Equal([0,0,0,0],b); }
[Fact] public void EmitU64_appends_little_endian() { var s=Create(); var b=new List<byte>(); s.EmitU64(b,0x1122334455667788); Assert.Equal([0x88,0x77,0x66,0x55,0x44,0x33,0x22,0x11],b); }
[Fact] public void EmitU64_appends_high_bits() { var s=Create(); var b=new List<byte>(); s.EmitU64(b,0xDEADBEEF_CAFEBABE); Assert.Equal([0xBE,0xBA,0xFE,0xCA,0xEF,0xBE,0xAD,0xDE],b); }
[Fact] public void StubAssembler_is_IAssembler() { Assert.IsAssignableFrom<IAssembler>(Create()); }
[Fact] public void Assemble_throws() { Assert.Throws<NotSupportedException>(()=>Create().Assemble("nop",0)); }
// ── x86 cdecl ──────────────────────────────────────────────────────────
[Fact]
public void Cdecl_0args()
{
uint r = 0x12345678u-(0x10000000u+5);
Assert.Equal([0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[],4,CallConvention.Cdecl));
}
[Fact]
public void Cdecl_1arg()
{
uint ca=0x10000000u+5,r=0x12345678u-(ca+5);
Assert.Equal([
0x68,0xDD,0xCC,0xBB,0xAA,
0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),
0x83,0xC4,0x04,0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0xAABBCCDD],4,CallConvention.Cdecl));
}
[Fact]
public void Cdecl_2args()
{
uint ca=0x10000000u+10,r=0x12345678u-(ca+5);
Assert.Equal([
0x68,0x22,0x22,0x22,0x22,
0x68,0x11,0x11,0x11,0x11,
0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),
0x83,0xC4,0x08,0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0x11111111,0x22222222],4,CallConvention.Cdecl));
}
// ── x86 stdcall ──────────────────────────────────────────────────────
[Fact]
public void Stdcall_1arg()
{
uint ca=0x10000000u+5,r=0x12345678u-(ca+5);
Assert.Equal([
0x68,0xDD,0xCC,0xBB,0xAA,
0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0xAABBCCDD],4,CallConvention.Stdcall));
}
[Fact]
public void Stdcall_2args()
{
uint ca=0x10000000u+10,r=0x12345678u-(ca+5);
Assert.Equal([
0x68,0x22,0x22,0x22,0x22,
0x68,0x11,0x11,0x11,0x11,
0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0x11111111,0x22222222],4,CallConvention.Stdcall));
}
// ── x86 thiscall ─────────────────────────────────────────────────────
[Fact]
public void Thiscall_ecx_then_stack()
{
uint ca=0x10000000u+10,r=0x12345678u-(ca+5);
Assert.Equal([
0xB9,0x55,0x55,0xAA,0xAA,
0x68,0x66,0x66,0xBB,0xBB,
0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0xAAAA5555,0xBBBB6666],4,CallConvention.Thiscall));
}
[Fact]
public void Thiscall_1arg_ecx_only()
{
uint ca=0x10000000u+5,r=0x12345678u-(ca+5);
byte[] s=Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0xCAFEBABE],4,CallConvention.Thiscall);
Assert.Equal(11,s.Length); Assert.Equal(0xB9,s[0]); Assert.Equal(0xCAFEBABEu,BitConverter.ToUInt32(s,1));
Assert.Equal(0xE8,s[5]); Assert.Equal(r,BitConverter.ToUInt32(s,6)); Assert.Equal(0xC3,s[10]);
}
[Fact]
public void Thiscall_0args_throws()
{
Assert.Throws<ArgumentOutOfRangeException>(() =>
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[],4,CallConvention.Thiscall));
}
// ── x86 fastcall ────────────────────────────────────────────────────
[Fact]
public void Fastcall_ecx_edx_stack()
{
uint ca=0x10000000u+15,r=0x12345678u-(ca+5);
Assert.Equal([
0xB9,0x11,0x11,0x11,0x11,
0xBA,0x22,0x22,0x22,0x22,
0x68,0x33,0x33,0x33,0x33,
0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0x11111111,0x22222222,0x33333333],4,CallConvention.Fastcall));
}
[Fact]
public void Fastcall_2args_registers_only()
{
uint ca=0x10000000u+10,r=0x12345678u-(ca+5);
byte[] s=Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[0xAAAAAAAA,0xBBBBBBBB],4,CallConvention.Fastcall);
Assert.Equal(16,s.Length); Assert.Equal(0xB9,s[0]); Assert.Equal(0xAAAAAAAAu,BitConverter.ToUInt32(s,1));
Assert.Equal(0xBA,s[5]); Assert.Equal(0xBBBBBBBBu,BitConverter.ToUInt32(s,6));
Assert.Equal(0xE8,s[10]); Assert.Equal(r,BitConverter.ToUInt32(s,11)); Assert.Equal(0xC3,s[15]);
}
[Fact]
public void Fastcall_0args_is_valid()
{
uint r=0x12345678u-(0x10000000u+5);
Assert.Equal([0xE8,(byte)r,(byte)(r>>8),(byte)(r>>16),(byte)(r>>24),0xC3],
Create().BuildCallStub((IntPtr)0x10000000,(IntPtr)0x12345678,[],4,CallConvention.Fastcall));
}
// ── x64 (Microsoft x64 ABI — shadow space + 16-byte alignment + 64-bit loads) ──
//
// Stub frame layout:
// bytes 0..6 sub rsp, 0x20 (7 bytes — shadow space + realignment)
// bytes 7..N mov r64, imm64 ... (10 bytes per reg move: 2-byte prefix + 8-byte imm)
// mov rax, imm64 / mov [rsp+0x20+8*(i-4)], rax for stack args (15 bytes each)
// E8 rel32 call target (5 bytes)
// 48 81 C4 20 00... add rsp, 0x20 (7 bytes)
// C3 ret (1 byte)
//
// Each `mov rNN, imm64` is 10 bytes regardless of the target register:
// RCX REX.W 0xB9 + 8 imm (0x48 0xB9)
// RDX REX.W 0xBA + 8 imm (0x48 0xBA)
// R8 REX.WB 0xB8 + 8 imm (0x49 0xB8 — REX.W|R = 0x49)
// R9 REX.WB 0xB9 + 8 imm (0x49 0xB9)
// RAX REX.W 0xB8 + 8 imm (0x48 0xB8)
[Fact]
public void X64_0args_allocates_shadow_space_and_aligns()
{
var s = Create();
ulong a = 0x100000000, t = 0x123456788;
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, [], 8, CallConvention.Cdecl);
// sub (7) + call (5) + add (7) + ret (1) = 20
Assert.Equal(20, stub.Length);
uint rel = (uint)(t - (a + 7 + 5)); // = t - a - 12
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x20
Assert.Equal(0xE8, stub[7]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 8));
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[12..19]); // add rsp, 0x20
Assert.Equal(0xC3, stub[19]); // ret
}
[Fact]
public void X64_1arg_loads_rcx_as_64bit()
{
var s = Create();
ulong a = 0x100000000, t = 0x123456788;
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, [0xAABBCCDDu], 8, CallConvention.Cdecl);
// sub (7) + mov rcx, imm64 (10) + call (5) + add (7) + ret (1) = 30
Assert.Equal(30, stub.Length);
uint rel = (uint)(t - (a + 7 + 10 + 5)); // = t - a - 22
// stub[0..6] = sub rsp, 0x20
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]);
// stub[7..16] = mov rcx, 0x00000000_AABBCCDD (zero-extended)
Assert.Equal(0x48, stub[7]); Assert.Equal(0xB9, stub[8]);
Assert.Equal(0xDD, stub[9]); Assert.Equal(0xCC, stub[10]);
Assert.Equal(0xBB, stub[11]); Assert.Equal(0xAA, stub[12]);
Assert.Equal(0x00, stub[13]); Assert.Equal(0x00, stub[14]);
Assert.Equal(0x00, stub[15]); Assert.Equal(0x00, stub[16]);
// stub[17..21] = call rel32
Assert.Equal(0xE8, stub[17]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 18));
// stub[22..28] = add rsp, 0x20
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[22..29]);
Assert.Equal(0xC3, stub[29]); // ret
}
[Fact]
public void X64_4args_loads_rcx_rdx_r8_r9_as_64bit()
{
var s = Create();
ulong a = 0x100000000, t = 0x123456788;
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t,
[0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u], 8, CallConvention.Cdecl);
// sub (7) + 4 x mov (4*10=40) + call (5) + add (7) + ret (1) = 60
Assert.Equal(60, stub.Length);
uint rel = (uint)(t - (a + 7 + 40 + 5)); // = t - a - 52
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]); // sub rsp, 0x20
// mov rcx, 0x11111111 (48 B9 + 8 imm) at [7..16]
Assert.Equal(0x48, stub[7]); Assert.Equal(0xB9, stub[8]);
Assert.Equal(0x11, stub[9]); Assert.Equal(0x11, stub[10]);
Assert.Equal(0x11, stub[11]); Assert.Equal(0x11, stub[12]);
Assert.Equal(0x00, stub[13]); Assert.Equal(0x00, stub[14]);
Assert.Equal(0x00, stub[15]); Assert.Equal(0x00, stub[16]);
// mov rdx, 0x22222222 (48 BA + 8 imm) at [17..26]
Assert.Equal(0x48, stub[17]); Assert.Equal(0xBA, stub[18]);
Assert.Equal(0x22, stub[19]); Assert.Equal(0x22, stub[20]);
Assert.Equal(0x22, stub[21]); Assert.Equal(0x22, stub[22]);
Assert.Equal(0x00, stub[23]); Assert.Equal(0x00, stub[24]);
Assert.Equal(0x00, stub[25]); Assert.Equal(0x00, stub[26]);
// mov r8, 0x33333333 (49 B8 + 8 imm) at [27..36]
Assert.Equal(0x49, stub[27]); Assert.Equal(0xB8, stub[28]);
Assert.Equal(0x33, stub[29]); Assert.Equal(0x33, stub[30]);
Assert.Equal(0x33, stub[31]); Assert.Equal(0x33, stub[32]);
Assert.Equal(0x00, stub[33]); Assert.Equal(0x00, stub[34]);
Assert.Equal(0x00, stub[35]); Assert.Equal(0x00, stub[36]);
// mov r9, 0x44444444 (49 B9 + 8 imm) at [37..46]
Assert.Equal(0x49, stub[37]); Assert.Equal(0xB9, stub[38]);
Assert.Equal(0x44, stub[39]); Assert.Equal(0x44, stub[40]);
Assert.Equal(0x44, stub[41]); Assert.Equal(0x44, stub[42]);
Assert.Equal(0x00, stub[43]); Assert.Equal(0x00, stub[44]);
Assert.Equal(0x00, stub[45]); Assert.Equal(0x00, stub[46]);
// call rel32 at [47..51]
Assert.Equal(0xE8, stub[47]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 48));
// add rsp, 0x20 at [52..58]
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[52..59]);
// ret at [59]
Assert.Equal(0xC3, stub[59]);
}
[Fact]
public void X64_5args_places_first_stack_arg_in_shadow_plus_0x20()
{
var s = Create();
ulong a = 0x100000000, t = 0x123456788;
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t,
[(nuint)1, (nuint)2, (nuint)3, (nuint)4, (nuint)5], 8, CallConvention.Cdecl);
// sub (7) + 4 reg moves (40) + stack arg (mov rax 10 + mov [rsp+0x20],rax 5 = 15)
// + call (5) + add (7) + ret (1) = 75
Assert.Equal(75, stub.Length);
Assert.Equal([0x48, 0x81, 0xEC, 0x20, 0x00, 0x00, 0x00], stub[..7]);
// mov rcx, 1 at [7..16]
Assert.Equal(0x48, stub[7]); Assert.Equal(0xB9, stub[8]);
Assert.Equal(0x01, stub[9]); Assert.Equal(0x00, stub[10]);
Assert.Equal(0x00, stub[11]); Assert.Equal(0x00, stub[12]);
// mov rdx, 2 at [17..26]
Assert.Equal(0x48, stub[17]); Assert.Equal(0xBA, stub[18]);
Assert.Equal(0x02, stub[19]);
// mov r8, 3 at [27..36]
Assert.Equal(0x49, stub[27]); Assert.Equal(0xB8, stub[28]);
Assert.Equal(0x03, stub[29]);
// mov r9, 4 at [37..46]
Assert.Equal(0x49, stub[37]); Assert.Equal(0xB9, stub[38]);
Assert.Equal(0x04, stub[39]);
// mov rax, 5 (48 B8 + 8-byte imm) at [47..56]
Assert.Equal(0x48, stub[47]); Assert.Equal(0xB8, stub[48]);
Assert.Equal(0x05, stub[49]);
for (int k = 50; k <= 56; k++) Assert.Equal(0x00, stub[k]);
// mov [rsp + 0x20], rax (48 89 44 24 20) at [57..61]
Assert.Equal(0x48, stub[57]); Assert.Equal(0x89, stub[58]);
Assert.Equal(0x44, stub[59]); Assert.Equal(0x24, stub[60]);
Assert.Equal(0x20, stub[61]);
// call rel32 at [62..66]; distance = t - (a + 62 + 5) = t - a - 67
uint rel = (uint)(t - (a + 62 + 5));
Assert.Equal(0xE8, stub[62]);
Assert.Equal(rel, BitConverter.ToUInt32(stub, 63));
// add rsp, 0x20 at [67..73]
Assert.Equal([0x48, 0x81, 0xC4, 0x20, 0x00, 0x00, 0x00], stub[67..74]);
// ret at [74]
Assert.Equal(0xC3, stub[74]);
}
[Fact]
public void X64_full_64bit_args_are_preserved_not_truncated()
{
// The bug this catches: an earlier stub emitted "mov r32d, imm32" which zero-extended
// a 32-bit immediate into the lower half of the 64-bit register, silently dropping
// the high bits of any pointer-sized argument above 4 GiB.
var s = Create();
ulong a = 0x100000000, t = 0x123456788;
nuint wideArg = unchecked((nuint)0xDEADBEEF_CAFEBABEUL);
byte[] stub = s.BuildCallStub(
(IntPtr)(nint)a, (IntPtr)(nint)t, [wideArg], 8, CallConvention.Cdecl);
// The 8-byte immediate for arg0 lives inside `mov rcx, imm64` at bytes [9..16].
ulong read = BitConverter.ToUInt64(stub, 9);
Assert.Equal(0xDEADBEEF_CAFEBABEul, read);
}
[Fact]
public void X86_target_rejects_arg_value_larger_than_32_bits()
{
if (!Environment.Is64BitProcess)
{
// On a 32-bit host, nuint cannot exceed uint.MaxValue — the precondition
// cannot be exercised. Mark the test as an intentional no-op.
Assert.True(true);
return;
}
var s = Create();
nuint tooBig = unchecked((nuint)0x1_00000000UL);
Assert.Throws<ArgumentOutOfRangeException>(() =>
s.BuildCallStub((IntPtr)0x10000000, (IntPtr)0x12345678, [tooBig], 4, CallConvention.Cdecl));
}
// ── Edge cases ────────────────────────────────────────────────────────
[Fact]
public void Far_target_throws()
{
Assert.Throws<ArgumentOutOfRangeException>(() =>
Create().BuildCallStub(IntPtr.Zero, unchecked((IntPtr)(nint)0xC0000000), [], 4, CallConvention.Cdecl));
}
[Fact]
public void Many_args_cleanup_uses_imm32_form()
{
// x86 path: 33 args, stack cleanup > 127 bytes → must emit add esp, imm32 (81 C4)
var args = new nuint[33];
for (int i = 0; i < 33; i++) args[i] = (nuint)(uint)(i * 0x10000 + i);
byte[] stub = Create().BuildCallStub(
(IntPtr)0x10000000, (IntPtr)0x12345678, args, 4, CallConvention.Cdecl);
for (int i = 0; i < stub.Length - 5; i++)
{
if (stub[i] == 0x81 && stub[i + 1] == 0xC4)
{
Assert.Equal(132, BitConverter.ToInt32(stub, i + 2));
return;
}
}
Assert.Fail("Expected 0x81 0xC4 (add esp, imm32) not found");
}
[Fact]
public void Invalid_pointerSize_throws()
{
Assert.Throws<ArgumentOutOfRangeException>(() =>
Create().BuildCallStub((IntPtr)0x10000000, (IntPtr)0x12345678, [], 2, CallConvention.Cdecl));
}
[Fact]
public void Invalid_calling_convention_throws()
{
Assert.Throws<ArgumentOutOfRangeException>(() =>
Create().BuildCallStub((IntPtr)0x10000000, (IntPtr)0x12345678, [], 4, (CallConvention)99));
}
// ── No-FASM ─────────────────────────────────────────────────────────
[Fact]
public void No_fasm_reference_in_output()
{
var asm = typeof(StubAssembler).Assembly;
var refs = asm.GetReferencedAssemblies();
Assert.DoesNotContain(refs, r =>
r.Name!.Contains("Fasm", StringComparison.OrdinalIgnoreCase) ||
r.Name!.Contains("ManagedFasm", StringComparison.OrdinalIgnoreCase));
}
}