bmaxa@Branimirs-Air memtest % cat mem.c
#include <stdlib.h>
#include <stdio.h>

int main(void) {
  for (long int i=4096;;i*=2){
    void *p = malloc(i);
    if (p)free(p);
    else {
      printf("max %ld\n", i);
      break;
    }
  }
}
    

This most likely means that it was not able to get an allocation in 32-bit addressable space, because this is what it needs to function.

After reviewing your feedback, we have some additional information for you, or some additional information, or action is necessary for this issue:

We make no guarantee that returned addresses will fit in 32-bit space. If fasmg can't handle that, it needs to change.

valloc:
; in: ecx = requested minimum size
; out: eax - allocated block, ecx = allocated size, zero if failed
; preserves: ebx, esi, edi
        cmp     ecx,VALLOC_MINIMUM_SIZE
        jbe     valloc_size_minimum
        dec     ecx
        and     ecx,(-1) shl 12
        add     ecx,1 shl 12
        jmp     valloc_size_ready
    valloc_size_minimum:
        mov     ecx,VALLOC_MINIMUM_SIZE
    valloc_size_ready:
        push    rbx rsi rdi
        cmp     [local_heap_available],0
        je      valloc_mmap
        cmp     ecx,LOCAL_HEAP_SIZE
        ja      valloc_mmap
        and     [local_heap_available],0
        mov     eax,local_heap
        mov     ecx,LOCAL_HEAP_SIZE
        jmp     valloc_ok
    valloc_mmap:
        xor     r9d,r9d
        or      r8,-1
        mov     r10d,62h                ; MAP_PRIVATE + MAP_ANONYMOUS + MAP_32BIT
        mov     edx,3                   ; PROT_READ + PROT_WRITE
        mov     esi,ecx
        xor     edi,edi
        mov     eax,9                   ; sys_mmap
        syscall
        cmp     eax,-1
        je      valloc_mmap_with_hint
        mov     ecx,eax
        cmp     rcx,rax
        jne     valloc_mmap_unusable
        add     ecx,esi
        jnc     mmap_ok
    valloc_mmap_unusable:
        mov     rdi,rax
        mov     eax,11                  ; sys_munmap
        syscall
    valloc_mmap_with_hint:
        mov     r10d,22h                ; MAP_PRIVATE + MAP_ANONYMOUS
        mov     edx,3                   ; PROT_READ + PROT_WRITE
        mov     edi,[mmap_hint]
        mov     eax,9                   ; sys_mmap
        syscall
        cmp     eax,-1
        je      valloc_failed
        mov     ecx,eax
        cmp     rcx,rax
        jne     valloc_failed
        add     ecx,esi
        jc      valloc_failed
    mmap_ok:
        sub     ecx,eax
    valloc_ok:
        lea     edx,[eax+ecx]
        mov     [mmap_hint],edx
        pop     rdi rsi rbx
        retn
    valloc_failed:
        xor     ecx,ecx
        pop     rdi rsi rbx
        retn    

segment readable writeable

  align 1000h

  LOCAL_HEAP_SIZE = 1000000h

  local_heap rb LOCAL_HEAP_SIZE    

Since we wanted to actually solve the issue rather than work around it, I've taken a look at this topic and tried to adapt your linux changes with valloc/mmap to macOS, and I may be close, but the mmap (sys)call isn't something I've adapted yet so it obviously crashes for now.

system_init:
        ; ...
        or      [local_heap_available],1
        retn    

Apparently registers are not the same, plus there's this whole 0x2000000 offset thing to take care of, but anyway Apple says calling mmap directly is much safer than syscall'ing, so maybe a ccall here would be better (if this is even possible at this point).

    valloc_mmap:
        push    rcx
        ccall   mmap,0,rcx, \
                        3, \            ; PROT_READ + PROT_WRITE
                        62h, \          ; MAP_PRIVATE + MAP_ANONYMOUS + MAP_32BIT
                        -1,0
        pop     rsi
        cmp     eax,-1
        je      valloc_mmap_with_hint
        mov     ecx,eax
        cmp     rcx,rax
        jne     valloc_mmap_unusable
        add     ecx,esi
        jnc     mmap_ok
    valloc_mmap_unusable:
        ccall   munmap,rax,rsi
    valloc_mmap_with_hint:
        push    rcx
        mov     edi,[mmap_hint]
        ccall   mmap,rdi,rcx, \
                        3, \            ; PROT_READ + PROT_WRITE
                        22h, \          ; MAP_PRIVATE + MAP_ANONYMOUS
                        -1,0
        pop     rsi
        cmp     eax,-1
        je      valloc_failed
        mov     ecx,eax
        cmp     rcx,rax
        jne     valloc_failed
        add     ecx,esi
        jc      valloc_failed    

That said, I still get "Error: not enough memory to complete the assembly." on a bigger source, so I'm not sure how to investigate further now.

In addition, the GuarMalloc workaround doesn't work anymore with this, but I guess that's because it acted on malloc, not mmap.

I found that mmap may ignore MAP_32BIT, so maybe that's related? (well, see https://stackoverflow.com/a/69792033/378298)

60087/0x9cf7fa:  mmap(0x0, 0x2000, 0x1, 0x40001, 0x3, 0x0)               = 0x105306000 0
60088/0x9cf81c:  mmap(0x0, 0x2000, 0x1, 0x40001, 0x3, 0x0)               = 0x1017000 0
60088/0x9cf81c:  mmap(0x0, 0x100000, 0x3, 0x40062, 0xFFFFFFFFFFFFFFFF, 0x0)              = 0xFFFFFFFFFFFFFFFF 9
60088/0x9cf81c:  mmap(0x1014000, 0x7FF806E8D59E, 0x3, 0x40022, 0xFFFFFFFFFFFFFFFF, 0x0)          = 0xFFFFFFFFFFFFFFFF 9    

Code:

60087/0x9cf7fa:  mmap(0x0, 0x2000, 0x1, 0x40001, 0x3, 0x0)               = 0x105306000 0
60088/0x9cf81c:  mmap(0x0, 0x2000, 0x1, 0x40001, 0x3, 0x0)               = 0x1017000 0
60088/0x9cf81c:  mmap(0x0, 0x100000, 0x3, 0x40062, 0xFFFFFFFFFFFFFFFF, 0x0)              = 0xFFFFFFFFFFFFFFFF 9
60088/0x9cf81c:  mmap(0x1014000, 0x7FF806E8D59E, 0x3, 0x40022, 0xFFFFFFFFFFFFFFFF, 0x0)          = 0xFFFFFFFFFFFFFFFF 9    

    valloc_mmap:
        push    rcx
        ccall   mmap,0,rcx, \
                        3, \            ; PROT_READ + PROT_WRITE
                        62h, \          ; MAP_PRIVATE + MAP_ANONYMOUS + MAP_32BIT
                        -1,0
        pop     rsi
        cmp     eax,-1
        je      valloc_mmap_with_hint
        mov     ecx,eax
        cmp     rcx,rax
        jne     valloc_mmap_unusable
        add     ecx,esi
        jnc     mmap_ok
    valloc_mmap_unusable:
        ccall   munmap,rax,rsi
    valloc_mmap_with_hint:
        push    rsi
        mov     edi,[mmap_hint]
        ccall   mmap,rdi,rsi, \
                        3, \            ; PROT_READ + PROT_WRITE
                        22h, \          ; MAP_PRIVATE + MAP_ANONYMOUS
                        -1,0
        pop     rsi
        cmp     eax,-1
        je      valloc_failed
        mov     ecx,eax
        cmp     rcx,rax
        jne     valloc_failed
        add     ecx,esi
        jc      valloc_failed    

67683/0x9e5bc4:  mmap(0x0, 0x2000, 0x1, 0x40001, 0x3, 0x0)               = 0x10A488000 0
67684/0x9e5bcc:  mmap(0x0, 0x2000, 0x1, 0x40001, 0x3, 0x0)               = 0x1017000 0
67684/0x9e5bcc:  mmap(0x0, 0x100000, 0x3, 0x40062, 0xFFFFFFFFFFFFFFFF, 0x0)              = 0xFFFFFFFFFFFFFFFF 9
67684/0x9e5bcc:  mmap(0x1014000, 0x100000, 0x3, 0x40022, 0xFFFFFFFFFFFFFFFF, 0x0)                = 0xFFFFFFFFFFFFFFFF 9    

                        9002h, \          ; MAP_PRIVATE + MAP_ANONYMOUS + MAP_32BIT
...
                        1002h, \          ; MAP_PRIVATE + MAP_ANONYMOUS