Segmentation fault when replacing `malloc`, `calloc`, `realloc` and `free` via `LD_PRELOAD`

Question

Background

I am experimenting to replace malloc(3)/calloc(3)/realloc(3)/free(3) via LD_PRELOAD environment variable. I have tried to use the customized functions statically linked, they worked perfectly.

But, when I attached it as shared library to LD_PRELOAD, it always results in segfault.

---

Short technical explanation about functions

• I use Linux x86-64 mmap(2) and munmap(2) syscall for malloc(3) and free(3).
• The calloc(3) is just a call to malloc(3) with multiply overflow check.
• The realloc(3) calls malloc(3), then copy old data to new allocated memory and unmap the old memory.

---

Questions

• What is wrong with my approach so that it always result in segfault?
• How can I debug it (gdb and valgrind also segfault)?
• What did I miss here?

---

Note

I am fully aware that always using mmap for every malloc call is a bad idea, especially for performance. I just want to know why my approach doesn't work.

---

Output

ammarfaizi2@integral:~$ gcc -shared mem.c -O3 -o my_mem.so
ammarfaizi2@integral:~$ LD_PRELOAD=$(pwd)/my_mem.so ls
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ LD_PRELOAD=$(pwd)/my_mem.so cat
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ LD_PRELOAD=$(pwd)/my_mem.so w
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ LD_PRELOAD=$(pwd)/my_mem.so gdb ls
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ LD_PRELOAD=$(pwd)/my_mem.so valgrind ls
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ 

---

glibc version

ammarfaizi2@integral:~$ /lib/x86_64-linux-gnu/libc.so.6
GNU C Library (Ubuntu GLIBC 2.33-0ubuntu2) release release version 2.33.
Copyright (C) 2021 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.
There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE.
Compiled by GNU CC version 10.2.1 20210130.
libc ABIs: UNIQUE IFUNC ABSOLUTE
For bug reporting instructions, please see:
<https://bugs.launchpad.net/ubuntu/+source/glibc/+bugs>.
ammarfaizi2@integral:~$ 

---

Code mem.c

#include <errno.h>
#include <stdlib.h>
#include <stdint.h>
#include <sys/mman.h>
#include <string.h>


static inline void *my_mmap(void *addr, size_t length, int prot, int flags,
                            int fd, off_t offset)
{
    void *ret;
    register int _flags asm("r10") = flags;
    register int _fd asm("r8") = fd;
    register off_t _offset asm("r9") = offset;

    asm volatile(
        "syscall"
        : "=a"(ret)
        : "a"(9), "D"(addr), "S"(length), "d"(prot),
          "r"(_flags), "r"(_fd), "r"(_offset)
        : "memory", "r11", "rcx"
    );
    return ret;
}


static inline int my_munmap(void *addr, size_t length)
{
    int ret;

    asm volatile(
        "syscall"
        : "=a"(ret)
        : "a"(11), "D"(addr), "S"(length)
        : "memory", "r11", "rcx"
    );
    return ret;
}

#define unlikely(EXPR) __builtin_expect(EXPR, 0)

void * __attribute__((noinline)) malloc(size_t len)
{
    void *start_map;
    uintptr_t user_ptr, cmperr;
    size_t add_req = 0;

    add_req += sizeof(size_t);
    add_req += sizeof(uint8_t);
    add_req += 0x1full;

    start_map = my_mmap(NULL, add_req + len, PROT_READ | PROT_WRITE,
                        MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

    cmperr = 0xffffffffffffff00ull;
    if (unlikely(((uintptr_t)start_map & cmperr) == cmperr)) {
        errno = ENOMEM;
        return NULL;
    }

    /* Align 32-byte and take space to save the length and diff */
    user_ptr = ((uintptr_t)start_map + add_req) & ~0x1full;
    *(size_t  *)(user_ptr - 8) = len;
    *(uint8_t *)(user_ptr - 9) = (uint8_t)(user_ptr - (uintptr_t)start_map);

    return (void *)user_ptr;
}


void free(void *__user_ptr)
{
    size_t len;
    uint8_t diff;
    uintptr_t user_ptr = (uintptr_t)__user_ptr;

    len  = *(size_t  *)(user_ptr - 8);
    diff = *(uint8_t *)(user_ptr - 9);
    my_munmap((void *)(user_ptr - diff), len);
}


void *realloc(void *__user_ptr, size_t new_len)
{
    void *new_mem;
    size_t len;
    uint8_t diff;
    uintptr_t user_ptr = (uintptr_t)__user_ptr;

    len  = *(size_t  *)(user_ptr - 8);
    diff = *(uint8_t *)(user_ptr - 9);

    new_mem = malloc(new_len);
    if (unlikely(new_mem == NULL))
        return NULL;

    memcpy(new_mem, __user_ptr, (new_len < len) ? new_len : len);
    my_munmap((void *)(user_ptr - diff), len);
    return new_mem;
}


void *calloc(size_t nmemb, size_t len)
{
    size_t x = nmemb * len;
    if (unlikely(nmemb != 0 && x / nmemb != len)) {
        errno = EOVERFLOW;
        return NULL;
    }
    return malloc(x);
}

// #include <stdio.h>
// int main(void)
// {
//  char *test = malloc(1);

//  for (size_t i = 2; i <= (1024 * 1024); i++) {
//      test = realloc(test, i);
//      memset(test, 'q', i);
//  }

//  free(test);
// }

---

Recompile with -Wall -Wextra -ggdb3 and strace output

ammarfaizi2@integral:~$ 
ammarfaizi2@integral:~$ gcc -Wall -Wextra -ggdb3 -shared mem.c -O3 -o my_mem.so
ammarfaizi2@integral:~$ strace -tf /usr/bin/env LD_PRELOAD=$(pwd)/my_mem.so ls
12:59:15 execve("/usr/bin/env", ["/usr/bin/env", "LD_PRELOAD=/home/ammarfaizi2/my_"..., "ls"], 0x7ffd2d6ee188 /* 34 vars */) = 0
12:59:15 brk(NULL)                      = 0x565552193000
12:59:15 arch_prctl(0x3001 /* ARCH_??? */, 0x7ffd13017120) = -1 EINVAL (Invalid argument)
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=74118, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 74118, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7facba2ba000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\206\2\0\0\0\0\0"..., 832) = 832
12:59:15 pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
12:59:15 pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
12:59:15 pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\30\355\366\266\203\242\371v\214\300\356\234\306J\346\373"..., 68, 896) = 68
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=1983576, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7facba2b8000
12:59:15 pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
12:59:15 mmap(NULL, 2012056, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7facba0cc000
12:59:15 mmap(0x7facba0f2000, 1486848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x7facba0f2000
12:59:15 mmap(0x7facba25d000, 311296, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x191000) = 0x7facba25d000
12:59:15 mmap(0x7facba2a9000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1dc000) = 0x7facba2a9000
12:59:15 mmap(0x7facba2af000, 33688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7facba2af000
12:59:15 close(3)                       = 0
12:59:15 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7facba0ca000
12:59:15 arch_prctl(ARCH_SET_FS, 0x7facba2b95c0) = 0
12:59:15 mprotect(0x7facba2a9000, 12288, PROT_READ) = 0
12:59:15 mprotect(0x565550cb5000, 4096, PROT_READ) = 0
12:59:15 mprotect(0x7facba2ff000, 8192, PROT_READ) = 0
12:59:15 munmap(0x7facba2ba000, 74118)  = 0
12:59:15 brk(NULL)                      = 0x565552193000
12:59:15 brk(0x5655521b4000)            = 0x5655521b4000
12:59:15 openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=3041456, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 3041456, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7facb9de3000
12:59:15 close(3)                       = 0
12:59:15 execve("/home/ammarfaizi2/.local/bin/ls", ["ls"], 0x565552194550 /* 35 vars */) = -1 ENOENT (No such file or directory)
12:59:15 execve("/usr/local/sbin/ls", ["ls"], 0x565552194550 /* 35 vars */) = -1 ENOENT (No such file or directory)
12:59:15 execve("/usr/local/bin/ls", ["ls"], 0x565552194550 /* 35 vars */) = -1 ENOENT (No such file or directory)
12:59:15 execve("/usr/sbin/ls", ["ls"], 0x565552194550 /* 35 vars */) = -1 ENOENT (No such file or directory)
12:59:15 execve("/usr/bin/ls", ["ls"], 0x565552194550 /* 35 vars */) = 0
12:59:15 brk(NULL)                      = 0x557f8624b000
12:59:15 arch_prctl(0x3001 /* ARCH_??? */, 0x7fff39dc1a30) = -1 EINVAL (Invalid argument)
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/home/ammarfaizi2/my_mem.so", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\20\0\0\0\0\0\0"..., 832) = 832
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0700, st_size=58768, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa7daa68000
12:59:15 mmap(NULL, 16448, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa7daa63000
12:59:15 mmap(0x7fa7daa64000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7fa7daa64000
12:59:15 mmap(0x7fa7daa65000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fa7daa65000
12:59:15 mmap(0x7fa7daa66000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fa7daa66000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=74118, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 74118, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa7daa50000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 p\0\0\0\0\0\0"..., 832) = 832
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=167352, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 178664, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa7daa24000
12:59:15 mmap(0x7fa7daa2a000, 106496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fa7daa2a000
12:59:15 mmap(0x7fa7daa44000, 32768, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x7fa7daa44000
12:59:15 mmap(0x7fa7daa4c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x7fa7daa4c000
12:59:15 mmap(0x7fa7daa4e000, 6632, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa7daa4e000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\206\2\0\0\0\0\0"..., 832) = 832
12:59:15 pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
12:59:15 pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
12:59:15 pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\30\355\366\266\203\242\371v\214\300\356\234\306J\346\373"..., 68, 896) = 68
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=1983576, ...}, AT_EMPTY_PATH) = 0
12:59:15 pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
12:59:15 mmap(NULL, 2012056, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa7da838000
12:59:15 mmap(0x7fa7da85e000, 1486848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x7fa7da85e000
12:59:15 mmap(0x7fa7da9c9000, 311296, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x191000) = 0x7fa7da9c9000
12:59:15 mmap(0x7fa7daa15000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1dc000) = 0x7fa7daa15000
12:59:15 mmap(0x7fa7daa1b000, 33688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa7daa1b000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\"\0\0\0\0\0\0"..., 832) = 832
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=617160, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 619304, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa7da7a0000
12:59:15 mmap(0x7fa7da7a2000, 438272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fa7da7a2000
12:59:15 mmap(0x7fa7da80d000, 167936, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6d000) = 0x7fa7da80d000
12:59:15 mmap(0x7fa7da836000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x95000) = 0x7fa7da836000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \"\0\0\0\0\0\0"..., 832) = 832
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=22912, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 24848, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa7da799000
12:59:15 mmap(0x7fa7da79b000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fa7da79b000
12:59:15 mmap(0x7fa7da79d000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7fa7da79d000
12:59:15 mmap(0x7fa7da79e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7fa7da79e000
12:59:15 close(3)                       = 0
12:59:15 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
12:59:15 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
12:59:15 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\200\0\0\0\0\0\0"..., 832) = 832
12:59:15 pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 792) = 48
12:59:15 pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0a7\363\352;k|\2228\244\6\253\346\2569\312"..., 68, 840) = 68
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=150456, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 136208, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa7da777000
12:59:15 mmap(0x7fa7da77e000, 65536, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7fa7da77e000
12:59:15 mmap(0x7fa7da78e000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fa7da78e000
12:59:15 mmap(0x7fa7da793000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x7fa7da793000
12:59:15 mmap(0x7fa7da795000, 13328, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa7da795000
12:59:15 close(3)                       = 0
12:59:15 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa7da775000
12:59:15 mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa7da772000
12:59:15 arch_prctl(ARCH_SET_FS, 0x7fa7da772800) = 0
12:59:15 mprotect(0x7fa7daa15000, 12288, PROT_READ) = 0
12:59:15 mprotect(0x7fa7da793000, 4096, PROT_READ) = 0
12:59:15 mprotect(0x7fa7da79e000, 4096, PROT_READ) = 0
12:59:15 mprotect(0x7fa7da836000, 4096, PROT_READ) = 0
12:59:15 mprotect(0x7fa7daa4c000, 4096, PROT_READ) = 0
12:59:15 mprotect(0x7fa7daa66000, 4096, PROT_READ) = 0
12:59:15 mprotect(0x557f842f9000, 8192, PROT_READ) = 0
12:59:15 mprotect(0x7fa7daa9c000, 8192, PROT_READ) = 0
12:59:15 munmap(0x7fa7daa50000, 74118)  = 0
12:59:15 set_tid_address(0x7fa7da772ad0) = 1639769
12:59:15 set_robust_list(0x7fa7da772ae0, 24) = 0
12:59:15 rt_sigaction(SIGRTMIN, {sa_handler=0x7fa7da77eb70, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7fa7da78b160}, NULL, 8) = 0
12:59:15 rt_sigaction(SIGRT_1, {sa_handler=0x7fa7da77ec10, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7fa7da78b160}, NULL, 8) = 0
12:59:15 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
12:59:15 prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
12:59:15 statfs("/sys/fs/selinux", 0x7fff39dc1a00) = -1 ENOENT (No such file or directory)
12:59:15 statfs("/selinux", 0x7fff39dc1a00) = -1 ENOENT (No such file or directory)
12:59:15 mmap(NULL, 512, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa7daa9b000
12:59:15 openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
12:59:15 mmap(NULL, 160, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa7daa62000
12:59:15 newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0
12:59:15 mmap(NULL, 1064, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa7daa61000
12:59:15 read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 410
12:59:15 read(3, "", 1024)              = 0
12:59:15 munmap(0x7fa7daa62000, 120)    = 0
12:59:15 close(3)                       = 0
12:59:15 munmap(0x7fa7daa61000, 1024)   = 0
12:59:15 munmap(0x7fa7daa9b000, 472)    = 0
12:59:15 --- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0xfffffffffffffff7} ---
12:59:16 +++ killed by SIGSEGV (core dumped) +++
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ 

---

Valgrind output after using /usr/bin/env

ammarfaizi2@integral:~$ valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all /usr/bin/env LD_PRELOAD=$(pwd)/my_mem.so ls
==1640100== Memcheck, a memory error detector
==1640100== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.
==1640100== Using Valgrind-3.17.0 and LibVEX; rerun with -h for copyright info
==1640100== Command: /usr/bin/env LD_PRELOAD=/home/ammarfaizi2/my_mem.so ls
==1640100== 
Segmentation fault (core dumped)
ammarfaizi2@integral:~$ 

Answer 1

I have debugged the core file and fixed the crash, for the free function, you need to check if the argument is nullptr, for the realloc we need to hander __user_ptr is nullptr too.

void free(void *__user_ptr) {
  if (!__user_ptr) return;
  // ...
}

void *realloc(void *__user_ptr, size_t new_len) {
  void *new_mem;
  size_t len;
  uint8_t diff;
  uintptr_t user_ptr = (uintptr_t)__user_ptr;

  new_mem = malloc(new_len);
  if (!__user_ptr) return new_mem;
  // ....
}

I have some experience in writing memory allocator libraries. During debugging, I find that some old c program uses realloc as malloc with a nullptr arguments, it's weird but is totally valid, please reference the man page

The realloc() function changes the size of the memory block pointed to by ptr to size bytes. The contents will be unchanged in the range from the start of the region up to the minimum of the old and new sizes. If the new size is larger than the old size, the added memory will not be initialized. If ptr is NULL, then the call is equivalent to malloc(size), for all values of size; if size is equal to zero, and ptr is not NULL, then the call is equivalent to free(ptr). Unless ptr is NULL, it must have been returned by an earlier call to malloc(), calloc() or realloc(). If the area pointed to was moved, a free(ptr) is done.

By the way, I see that you try to wrap the syscall for mmap and munmap, I suggest that we replace them with https://github.com/linux-on-ibm-z/linux-syscall-support , which is a production-level wrapper library and widely used. I think we should write as little code as possible to reduce potential errors.

Answer 2

gcc -Wall -Wextra -ggdb3 -shared mem.c -O3 -o my_mem.so

is wrong, if you want to build a shared library. See dlopen(3) and elf(5) and ld.so(8).

You practically need a position-independent-code file, so use

gcc -Wall -Wextra -ggdb3 -fPIC -shared mem.c -O3 -o my_mem.so

Read Advanced Linux Programming, Drepper's paper How to write shared libraries and the Program Library HowTo and Linux Assembler Howto.

BTW, your my_mmap is naive, and don't handle failure cases of mmap(2). See errno(3) and syscalls(2).

You may want to study (for inspiration) the source code of GNU libc or of musl-libc. Both have a better mmap implementation than yours.

You should consider using the Clang static analyzer, write your own GCC plugin (or use Bismon), and use the GCC address sanitizer. Take more time to read about Invoking GCC.

PS. In 2021 see also the DECODER project.