Segmentation fault (core dumped) - C calling Assembly code-CodePudding

I have an assembly language code I'm calling from C but I keep getting error: Segmentation fault (core dumped) I don't know what's the cause.

; this is the assembly code
section .text

global isPrime

isPrime: mov ecx, 2
.test:   cmp ecx, [rsi]
         jle .doif
         jg .prime

.doif:   mov eax, [rdi]
         mov edx, 0
         mov ebx, [rsi]
         div ebx
         cmp edx, 0
         jle .notPrime
         jg .checkAgain
.notPrime:
         mov eax, 1
         ret

.checkAgain:
         inc ecx
         jmp .test

.prime: mov eax, 0
         ret

The C code:

// C code

#include <stdio.h>

extern int isPrime(int *number, int *mValue);

int main() {
    int limit, m, input = 0;
    printf("Enter the limit of the prime numbers:");
    input = scanf("%d", &limit);
    while (input != 1) {
        printf("Not a number!\n");
        scanf("%*[^\n]");
        printf("Enter the limit of the prime numbers:");
        input = scanf("%d", &limit);
    }
    for (int i = 2; i <= limit;   i) {
        m = i / 2;
        int flag = isPrime(&i, &m); //this is what I'm trying to implement
        // for (int j = 2; j <= m; j  ) {
        //     if (i % j == 0) {
        //         printf("Number %d is not prime\n", i);
        //         flag = 1;
        //         break;
        //     }
        // }
        printf("%d\n", flag);
        if (flag == 0)
            printf("Number %d is prime\n", i);
    }
  return 0;
}

Error:

Enter the limit of the prime numbers:10
0
0
Segmentation fault (core dumped)

the commented part in the C code is what I want to write in assembly but got the error I mentioned above. From my research, I'm trying to write a memory address I do not have access to. The error is from assembly code but I don't know where exactly, please any possible solutions?

CodePudding user response：

In System V x86_64 calling convention registers rbx, rsp, rbp, r12, r13, r14, r15 must be preserved. You modify ebx register as a part of rbx but don't preserve it. You can preserve it, for example, on stack:

BITS 64
section .text

global isPrime

isPrime: push rbx
         mov ecx, 2
.test:   cmp ecx, [rsi]
         jle .doif
         jg .prime

.doif:   mov eax, [rdi]
         mov edx, 0
         mov ebx, [rsi]
         div ebx
         cmp edx, 0
         jle .notPrime
         jg .checkAgain
.notPrime:
         pop rbx
         mov eax, 1
         ret

.checkAgain:
         inc ecx
         jmp .test

.prime:  pop rbx
         mov eax, 0
         ret

Note: there are two return points, so you have to restore rbx before each return.

EDIT: It is also possible to save rbx before modification and restore it only once after:

BITS 64
section .text

global isPrime

isPrime: mov ecx, 2
.test:   cmp ecx, [rsi]
         jle .doif
         jg .prime

.doif:   push rbx
         mov eax, [rdi]
         mov edx, 0
         mov ebx, [rsi]
         div ebx
         pop rbx
         cmp edx, 0
         jle .notPrime
         jg .checkAgain
.notPrime:
         mov eax, 1
         ret

.checkAgain:
         inc ecx
         jmp .test

.prime:  mov eax, 0
         ret

EDIT2: Or use one of scratch registers as a temporary storage:

...
.doif:   mov r11, rbx
         mov eax, [rdi]
         mov edx, 0
         mov ebx, [rsi]
         div ebx
         mov rbx, r11
...

CodePudding user response：

Just in case anyone is facing similar issue and needs help:

section .text
global isPrime

isPrime: mov ecx, 2
.test:   cmp ecx, [rsi]
         jle .doif
         jg .prime

.doif:   mov eax, [rdi]
         mov edx, 0
         div ecx
         cmp edx, 0
         jg .checkAgain
         mov eax, 1
         ret

.checkAgain:
         inc ecx
         jmp .test

.prime:  mov eax, 0
         ret