test-glibc-override.cpp

#include <pthread.h>
#include <dlfcn.h>
#include <stdlib.h>

// force jemalloc
void __attribute__((used)) force_jemalloc()
{
    /// calloc() is used in older glibc in dlsym()
    /// force this symbol from the jemalloc to trigger endless loop
    (void)calloc(0, 0);
}

static int (*real_pthread_mutex_lock)(pthread_mutex_t * mutex) = NULL;
int pthread_mutex_lock(pthread_mutex_t * mutex)
{
    if (!real_pthread_mutex_lock)
        real_pthread_mutex_lock = reinterpret_cast<int (*)(pthread_mutex_t * mutex)>(dlsym(RTLD_NEXT, "pthread_mutex_lock"));
    return real_pthread_mutex_lock(mutex);
}

static int (*real_pthread_mutex_unlock)(pthread_mutex_t * mutex) = NULL;
int pthread_mutex_unlock(pthread_mutex_t * mutex)
{
    if (!real_pthread_mutex_unlock)
        real_pthread_mutex_unlock = reinterpret_cast<int (*)(pthread_mutex_t * mutex)>(dlsym(RTLD_NEXT, "pthread_mutex_unlock"));
    return real_pthread_mutex_unlock(mutex);
}

int main()
{
    // something broken in ASan in interceptor for __pthread_mutex_lock
    // and only since glibc 2.36, and for pthread_mutex_lock everything is OK
    pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
    pthread_mutex_lock(&mutex);
    pthread_mutex_unlock(&mutex);
    return 0;
}

test-glibc-override.sh

# such complex build that relies on ClickHouse is to build on a system with a newer glibc
# and run on a an older glibc
$ clang++ -g3 -o test-glibc-override test-glibc-override.cpp --target=x86_64-linux-gnu --sysroot=/src/ch/clickhouse/cmake/linux/../../contrib/sysroot/linux-x86_64/x86_64-linux-gnu/libc --gcc-toolchain=/src/ch/clickhouse/cmake/linux/../../contrib/sysroot/linux-x86_64 ../.cmake/contrib/jemalloc-cmake/lib_jemalloc.a contrib/libunwind-cmake/libunwind.a base/glibc-compatibility/memcpy/libmemcpy.a  contrib/libcxx-cmake/libcxx.a  contrib/libcxxabi-cmake/libcxxabi.a   -nodefaultlibs -lc -lm -lrt -lpthread -ldl -fno-pie -fPIC -no-pie -Wl,-no-pie

$ podman run --privileged -v $PWD:/root/bin:ro --rm -it -e PATH=/bin:/root/bin ubuntu-dev-v3 lldb test-glibc-override
(lldb) target create "test-glibc-override"
Current executable set to 'test-glibc-override' (x86_64).
(lldb) r
Process 7 launched: '/root/bin/test-glibc-override' (x86_64)
Process 7 stopped
* thread #1, name = 'test-glibc-over', stop reason = signal SIGSEGV: invalid address (fault address: 0x7fffff7feff8)
    frame #0: 0x00007ffff7eaf099 libc.so.6`__GI___clock_gettime(clock_id=6, tp=0x00007fffff7ff018) at clock_gettime.c:31:1
(lldb) bt 16
* thread #1, name = 'test-glibc-over', stop reason = signal SIGSEGV: invalid address (fault address: 0x7fffff7feff8)
  * frame #0: 0x00007ffff7eaf099 libc.so.6`__GI___clock_gettime(clock_id=6, tp=0x00007fffff7ff018) at clock_gettime.c:31:1
    frame #1: 0x000000000043572c test-glibc-override`nstime_init_update [inlined] nstime_get(time=0x00007fffff7ff058) at nstime.c:192:2
    frame #2: 0x0000000000435722 test-glibc-override`nstime_init_update [inlined] nstime_update_impl(time=0x00007fffff7ff058) at nstime.c:268
    frame #3: 0x0000000000435722 test-glibc-override`nstime_init_update(time=0x00007fffff7ff058) at nstime.c:280
    frame #4: 0x0000000000434ff3 test-glibc-override`malloc_mutex_lock_slow(mutex=0x00000000004e5690) at mutex.c:76:2
    frame #5: 0x00000000004323f7 test-glibc-override`malloc_init_hard [inlined] malloc_mutex_lock(tsdn=<unavailable>) at mutex.h:217:4
    frame #6: 0x00000000004323df test-glibc-override`malloc_init_hard at jemalloc.c:2118
    frame #7: 0x000000000041af30 test-glibc-override`calloc [inlined] malloc_init at jemalloc.c:298:41
    frame #8: 0x000000000041af2b test-glibc-override`calloc [inlined] imalloc_init_check at jemalloc.c:2658
    frame #9: 0x000000000041af2b test-glibc-override`calloc [inlined] imalloc at jemalloc.c:2689
    frame #10: 0x000000000041af2b test-glibc-override`calloc(num=1, size=32) at jemalloc.c:2852
    frame #11: 0x00007ffff7c51c05 libdl.so.2`_dlerror_run(operate=(libdl.so.2`dlsym_doit at dlsym.c:47:1), args=0x00007fffff7ff220) at dlerror.c:148:41
    frame #12: 0x00007ffff7c51525 libdl.so.2`__dlsym(handle=<unavailable>, name="pthread_mutex_lock") at dlsym.c:70:19
    frame #13: 0x000000000041557d test-glibc-override`::pthread_mutex_lock(mutex=0x00000000004e56d0) at test-glibc-override.cpp:17:86
    frame #14: 0x0000000000435029 test-glibc-override`malloc_mutex_lock_slow [inlined] malloc_mutex_lock_final(mutex=0x00000000004e5690) at mutex.h:151:2
    frame #15: 0x0000000000435021 test-glibc-override`malloc_mutex_lock_slow(mutex=0x00000000004e5690) at mutex.c:90
...
    frame #151525: 0x00007ffff7c51c05 libdl.so.2`_dlerror_run(operate=(libdl.so.2`dlsym_doit at dlsym.c:47:1), args=0x00007fffffffbb60) at dlerror.c:148:41
    frame #151526: 0x00007ffff7c51525 libdl.so.2`__dlsym(handle=<unavailable>, name="pthread_mutex_lock") at dlsym.c:70:19
    frame #151527: 0x000000000041557d test-glibc-override`::pthread_mutex_lock(mutex=0x00000000004e56d0) at test-glibc-override.cpp:17:86
    frame #151528: 0x0000000000435029 test-glibc-override`malloc_mutex_lock_slow [inlined] malloc_mutex_lock_final(mutex=0x00000000004e5690) at mutex.h:151:2
    frame #151529: 0x0000000000435021 test-glibc-override`malloc_mutex_lock_slow(mutex=0x00000000004e5690) at mutex.c:90
    frame #151530: 0x00000000004323f7 test-glibc-override`malloc_init_hard [inlined] malloc_mutex_lock(tsdn=<unavailable>) at mutex.h:217:4
    frame #151531: 0x00000000004323df test-glibc-override`malloc_init_hard at jemalloc.c:2118
    frame #151532: 0x000000000041af30 test-glibc-override`calloc [inlined] malloc_init at jemalloc.c:298:41
    frame #151533: 0x000000000041af2b test-glibc-override`calloc [inlined] imalloc_init_check at jemalloc.c:2658
    frame #151534: 0x000000000041af2b test-glibc-override`calloc [inlined] imalloc at jemalloc.c:2689
    frame #151535: 0x000000000041af2b test-glibc-override`calloc(num=1, size=32) at jemalloc.c:2852
    frame #151536: 0x00007ffff7c51c05 libdl.so.2`_dlerror_run(operate=(libdl.so.2`dlsym_doit at dlsym.c:47:1), args=0x00007fffffffbdc0) at dlerror.c:148:41
    frame #151537: 0x00007ffff7c51525 libdl.so.2`__dlsym(handle=<unavailable>, name="pthread_mutex_lock") at dlsym.c:70:19
    frame #151538: 0x000000000041557d test-glibc-override`::pthread_mutex_lock(mutex=0x00000000004e56d0) at test-glibc-override.cpp:17:86
    frame #151539: 0x0000000000435029 test-glibc-override`malloc_mutex_lock_slow [inlined] malloc_mutex_lock_final(mutex=0x00000000004e5690) at mutex.h:151:2
    frame #151540: 0x0000000000435021 test-glibc-override`malloc_mutex_lock_slow(mutex=0x00000000004e5690) at mutex.c:90
    frame #151541: 0x00000000004323f7 test-glibc-override`malloc_init_hard [inlined] malloc_mutex_lock(tsdn=<unavailable>) at mutex.h:217:4
    frame #151542: 0x00000000004323df test-glibc-override`malloc_init_hard at jemalloc.c:2118
    frame #151543: 0x000000000041af30 test-glibc-override`calloc [inlined] malloc_init at jemalloc.c:298:41
    frame #151544: 0x000000000041af2b test-glibc-override`calloc [inlined] imalloc_init_check at jemalloc.c:2658
    frame #151545: 0x000000000041af2b test-glibc-override`calloc [inlined] imalloc at jemalloc.c:2689
    frame #151546: 0x000000000041af2b test-glibc-override`calloc(num=1, size=32) at jemalloc.c:2852
    frame #151547: 0x00007ffff7c51c05 libdl.so.2`_dlerror_run(operate=(libdl.so.2`dlsym_doit at dlsym.c:47:1), args=0x00007fffffffc020) at dlerror.c:148:41
    frame #151548: 0x00007ffff7c51525 libdl.so.2`__dlsym(handle=<unavailable>, name="pthread_mutex_unlock") at dlsym.c:70:19
    frame #151549: 0x00000000004155ed test-glibc-override`::pthread_mutex_unlock(mutex=0x00007ffff7a00120) at test-glibc-override.cpp:25:88
    frame #151550: 0x000000000045b28f test-glibc-override`base_alloc_impl [inlined] malloc_mutex_unlock(mutex=0x00007ffff7a000e0) at mutex.h:230:3
    frame #151551: 0x000000000045b27e test-glibc-override`base_alloc_impl(tsdn=0x0000000000000000, base=0x00007ffff7a000c0, size=<unavailable>, alignment=<unavailable>, esn=0x0000000000000000) at base.c:465
    frame #151552: 0x000000000044fe14 test-glibc-override`tcache_boot(tsdn=0x0000000000000000, base=0x00007ffff7a000c0) at tcache.c:1062:40
    frame #151553: 0x000000000042d36c test-glibc-override`malloc_init_hard_a0_locked at jemalloc.c:1866:6
    frame #151554: 0x000000000043256f test-glibc-override`malloc_init_hard at jemalloc.c:2129:6
    frame #151555: 0x00000000004b2e5d test-glibc-override`__libc_csu_init + 77
    frame #151556: 0x00007ffff7df6010 libc.so.6`__libc_start_main(main=(test-glibc-override`main at test-glibc-override.cpp:38), argc=1, argv=0x00007fffffffedb8, init=(test-glibc-override`__libc_csu_init), fini=<unavailable>, rtld_fini=<unavailable>, stack_end=0x00007fffffffeda8) at libc-start.c:264:6
    frame #151557: 0x000000000041546e test-glibc-override`_start + 46