global mymul
mymul:
    mov rax, rdi
    mul rsi
    ret

#include <stdio.h>

typedef struct {
        unsigned long long high;
            unsigned long long low;
} resmul;

void mymul(unsigned long long, unsigned long long, resmul *res);

int main() {
    resmul res;

    mymul(3, 6, &res);
    printf("mymul(3, 6); res.high=0x%llx, res.low=0x%llx\n", res.high, res.low);
    //mymul(3, 6); res.high=0x0, res.low=0x12
    return 0;
}

the goal is to multiply first arg with the second and send to result to the last arg first arg = RDI / second arg = RSI goal to send result high/low to typestruct

I dont understand why it gives 0 to both results RAX and RDX should be returned but i doesnt

CodePudding user response：

Your mymul function is declared from C as taking a pointer argument, where it should be storing the results in memory. But in fact it is leaving them in the rdx:rax registers and never storing anything in memory at all, ignoring the pointer completely.

The third argument would be passed in the rdx register, which complicates things a little because mul overwrites it. So you have to do something like

global mymul
mymul:
    mov rcx, rdx  ; save argument
    mov rax, rdi
    mul rsi
    mov [rcx], rdx
    mov [rcx 8], rax
    ret

CodePudding user response：

You told the compiler your function returns void, so the caller isn't looking at RDX:RAX.

Use a debugger and/or look at the asm in the caller; it will be loading its uninitialized local variable res from the stack, and that unwritten stack memory just happens to be zero.

To properly declare a function returning in RDX:RAX for the x86-64 SysV calling convention, either return a struct, or return unsigned __int128 if you're using GNU extensions.

#include <stdio.h>
#include <stdint.h>

typedef struct {
        uint64_t low;         // low first, the low half of RDX:RAX
        uint64_t high;
} resmul;              // x86-64 System V will return this in RDX:RAX, like __int128

resmul mymul(uint64_t, uint64_t);  // your function doesn't look for a pointer in RDX

int main() {
    resmul res = mymul(3, 6);
    printf("mymul(3, 6); res.high=%#lx, res.low=%#lx\n", res.high, res.low);
    //mymul(3, 6); res.high=0x0, res.low=0x12
    return 0;
}

>64}; } #endif unsigned __int128 mulv2(uint64_t a, uint64_t b){ return a * (unsigned __int128)b; }'),l:'5',n:'0',o:'C++ source #1',t:'0')),header:(),k:35.99514583278259,l:'4',m:100,n:'0',o:'',s:0,t:'0'),(g:!((h:compiler,i:(compiler:g122,deviceViewOpen:'1',filters:(b:'0',binary:'1',commentOnly:'0',demangle:'0',directives:'0',execute:'1',intel:'0',libraryCode:'0',trim:'1'),flagsViewOpen:'1',fontScale:14,fontUsePx:'0',j:2,lang:c++,libs:!(),options:'-O3 -Wall',selection:(endColumn:1,endLineNumber:1,positionColumn:1,positionLineNumber:1,selectionStartColumn:1,selectionStartLineNumber:1,startColumn:1,startLineNumber:1),source:1),l:'5',n:'0',o:' x86-64 gcc 12.2 (Editor #1)',t:'0')),k:30.671520833884088,l:'4',m:100,n:'0',o:'',s:0,t:'0'),(g:!((h:output,i:(compilerName:'x86-64 gcc 12.2',editorid:1,fontScale:14,fontUsePx:'0',j:2,wrap:'1'),l:'5',n:'0',o:'Output of x86-64 gcc 12.2 (Compiler #2)',t:'0')),k:33.33333333333333,l:'4',n:'0',o:'',s:0,t:'0')),l:'2',n:'0',o:'',t:'0')),version:4" rel="nofollow noreferrer">Godbolt, also including GNU C implementations of your function that compile to the same asm:

// compiles to the same asm
resmul mymul(uint64_t a, uint64_t b){
    unsigned __int128 prod = a * (unsigned __int128)b;
    return (resmul){prod, prod>>64};
}

unsigned __int128 mulv2(uint64_t a, uint64_t b){
    return a * (unsigned __int128)b;
}

mymul(unsigned long, unsigned long):
        mov     rax, rdi
        mul     rsi
        ret
mulv2(unsigned long, unsigned long):
        mov     rax, rdi
        mul     rsi
        ret