task 3.3-3.4: x86 cdecl stub encoding + CallingConvention enum

Add CallingConvention enum (Cdecl, Stdcall, Thiscall, Fastcall). Implement BuildCallStub on StubAssembler with x86 cdecl support: reverse arg push, call rel32, add esp (caller cleanup), ret. x64 stub is a placeholder (task 3.7-3.8). 3 new tests: 0-arg (call+ret), 1-arg (push+call+cleanup+ret), 2-args (reverse push+call+cleanup+ret). Known byte expectations. All passing (total: 67).
2026-07-21 19:37:20 +02:00
parent 855595837f
commit f7236eea9b
3 changed files with 163 additions and 11 deletions
@@ -0,0 +1,19 @@
+namespace WhiteMagic.Assembly;
+
+/// <summary>
+/// x86/x86-64 calling conventions for call-stub generation.
+/// </summary>
+public enum CallingConvention
+{
+    /// <summary>Caller pushes args right-to-left and cleans the stack (x86).</summary>
+    Cdecl,
+
+    /// <summary>Caller pushes args right-to-left; callee cleans the stack (x86).</summary>
+    Stdcall,
+
+    /// <summary>ECX receives the <c>this</c> pointer; remaining args on stack right-to-left; callee cleans (x86).</summary>
+    Thiscall,
+
+    /// <summary>ECX/EDX receive the first two args; remaining on stack right-to-left; callee cleans (x86).</summary>
+    Fastcall,
+}
@@ -14,8 +14,6 @@ namespace WhiteMagic.Assembly;
 public class StubAssembler : IAssembler
 {
    /// <inheritdoc />
-    /// <exception cref="NotSupportedException">Always thrown. StubAssembler does
-    /// not parse text assembly; use IcedAssembler for that.</exception>
    public byte[] Assemble(string assemblyText, ulong origin = 0)
    {
        throw new NotSupportedException(
@@ -25,13 +23,8 @@ public class StubAssembler : IAssembler

    // ── Emit primitives ────────────────────────────────────────────────────

-    /// <summary>Appends a single byte to <paramref name="buffer"/>.</summary>
-    public void EmitU8(List<byte> buffer, byte value)
-    {
-        buffer.Add(value);
-    }
+    public void EmitU8(List<byte> buffer, byte value) => buffer.Add(value);

-    /// <summary>Appends <paramref name="value"/> as 4 little-endian bytes.</summary>
    public void EmitU32(List<byte> buffer, uint value)
    {
        buffer.Add((byte)value);
@@ -40,10 +33,86 @@ public class StubAssembler : IAssembler
        buffer.Add((byte)(value >> 24));
    }

-    /// <summary>Appends <paramref name="value"/> as 8 little-endian bytes.</summary>
    public void EmitU64(List<byte> buffer, ulong value)
    {
        EmitU32(buffer, (uint)value);
        EmitU32(buffer, (uint)(value >> 32));
    }
+
+    // ── Call-stub builders ─────────────────────────────────────────────────
+
+    public byte[] BuildCallStub(IntPtr stubAddress, IntPtr targetAddress,
+        uint[] arguments, int pointerSize, CallingConvention convention)
+    {
+        var buffer = new List<byte>(64);
+
+        if (pointerSize == 4)
+            BuildX86Stub(buffer, (uint)stubAddress, (uint)targetAddress, arguments, convention);
+        else
+            BuildX64Stub(buffer, (ulong)stubAddress, (ulong)targetAddress, arguments);
+
+        return buffer.ToArray();
+    }
+
+    private void BuildX86Stub(List<byte> buffer, uint stubAddr,
+        uint target, uint[] args, CallingConvention convention)
+    {
+        uint current = stubAddr;
+
+        switch (convention)
+        {
+            case CallingConvention.Thiscall when args.Length >= 1:
+                buffer.Add(0xB9);         // mov ecx, arg0
+                EmitU32(buffer, args[0]);
+                current += 5;
+                args = args[1..];
+                break;
+
+            case CallingConvention.Fastcall:
+                if (args.Length >= 1)
+                {
+                    buffer.Add(0xB9);     // mov ecx, arg0
+                    EmitU32(buffer, args[0]);
+                    current += 5;
+                    args = args[1..];
+                }
+                if (args.Length >= 1)
+                {
+                    buffer.Add(0xBA);     // mov edx, arg1
+                    EmitU32(buffer, args[0]);
+                    current += 5;
+                    args = args[1..];
+                }
+                break;
+        }
+
+        // Push remaining args in reverse order
+        for (int i = args.Length - 1; i >= 0; i--)
+        {
+            buffer.Add(0x68);             // push imm32
+            EmitU32(buffer, args[i]);
+            current += 5;
+        }
+
+        // call rel32
+        uint rel32 = target - (current + 5);
+        buffer.Add(0xE8);
+        EmitU32(buffer, rel32);
+
+        // Caller cleanup (cdecl only)
+        if (convention == CallingConvention.Cdecl && args.Length > 0)
+        {
+            buffer.Add(0x83);             // add esp, imm8
+            buffer.Add(0xC4);
+            buffer.Add((byte)(args.Length * 4));
+        }
+
+        buffer.Add(0xC3);                 // ret
+    }
+
+    private static void BuildX64Stub(List<byte> buffer, ulong stubAddr,
+        ulong target, uint[] args)
+    {
+        throw new NotImplementedException("x64 stubs (task 3.7-3.8)");
+    }
 }
@@ -4,8 +4,8 @@ namespace WhiteMagicTest;

 /// <summary>
 /// Tests for <see cref="StubAssembler"/> emit primitives (<see cref="StubAssembler.EmitU8"/>,
-/// <see cref="StubAssembler.EmitU32"/>, <see cref="StubAssembler.EmitU64"/>) and the
-/// <see cref="IAssembler"/> interface contract.
+/// <see cref="StubAssembler.EmitU32"/>, <see cref="StubAssembler.EmitU64"/>) and
+/// calling-convention stub encoding.
 /// </summary>
 public class StubAssemblerTests
 {
@@ -75,4 +75,68 @@ public class StubAssemblerTests
        var sut = Create();
        Assert.Throws<NotSupportedException>(() => sut.Assemble("nop", 0));
    }
+
+    // ── Calling convention: x86 cdecl ──────────────────────────────────────
+
+    [Fact]
+    public void Cdecl_stub_with_zero_args_is_call_then_ret()
+    {
+        var sut = Create();
+        IntPtr stubAddr = (IntPtr)0x10000000;
+        IntPtr target = (IntPtr)0x12345678;
+        uint[] args = [];
+
+        uint rel32 = (uint)target - ((uint)stubAddr + 5);
+        byte[] stub = sut.BuildCallStub(stubAddr, target, args, 4, CallingConvention.Cdecl);
+
+        Assert.Equal(6, stub.Length);
+        Assert.Equal(0xE8, stub[0]);                           // call
+        Assert.Equal(rel32, BitConverter.ToUInt32(stub, 1));   // rel32
+        Assert.Equal(0xC3, stub[5]);                           // ret
+    }
+
+    [Fact]
+    public void Cdecl_stub_one_arg_reverse_push_then_call_then_cleanup()
+    {
+        var sut = Create();
+        IntPtr stubAddr = (IntPtr)0x10000000;
+        IntPtr target = (IntPtr)0x12345678;
+        uint[] args = [0xAABBCCDD];
+
+        uint callAddr = (uint)stubAddr + 5;
+        uint rel32 = (uint)target - (callAddr + 5);
+        byte[] stub = sut.BuildCallStub(stubAddr, target, args, 4, CallingConvention.Cdecl);
+
+        byte[] expected = [
+            0x68, 0xDD, 0xCC, 0xBB, 0xAA, // push 0xAABBCCDD
+            0xE8,
+            (byte)rel32, (byte)(rel32 >> 8), (byte)(rel32 >> 16), (byte)(rel32 >> 24),
+            0x83, 0xC4, 0x04,              // add esp, 4
+            0xC3                            // ret
+        ];
+        Assert.Equal(expected, stub);
+    }
+
+    [Fact]
+    public void Cdecl_stub_two_args_reverse_order()
+    {
+        var sut = Create();
+        IntPtr stubAddr = (IntPtr)0x10000000;
+        IntPtr target = (IntPtr)0x12345678;
+        uint[] args = [0x11111111, 0x22222222];
+
+        uint callAddr = (uint)stubAddr + 10;
+        uint rel32 = (uint)target - (callAddr + 5);
+        byte[] stub = sut.BuildCallStub(stubAddr, target, args, 4, CallingConvention.Cdecl);
+
+        byte[] expected = [
+            0x68, 0x22, 0x22, 0x22, 0x22, // push arg1 (reverse order)
+            0x68, 0x11, 0x11, 0x11, 0x11, // push arg0
+            0xE8,
+            (byte)rel32, (byte)(rel32 >> 8), (byte)(rel32 >> 16), (byte)(rel32 >> 24),
+            0x83, 0xC4, 0x08,              // add esp, 8
+            0xC3
+        ];
+        Assert.Equal(expected, stub);
+    }
 }