Import mutex, semaphore, and condition variable from LWSP

Userland/Libraries/LibC/bits/pthread_integration.h

@@ -18,10 +18,11 @@ void __pthread_fork_atfork_register_prepare(void (*)(void));
 void __pthread_fork_atfork_register_parent(void (*)(void));
 void __pthread_fork_atfork_register_child(void (*)(void));
 
+int __pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*);
 int __pthread_mutex_lock(pthread_mutex_t*);
 int __pthread_mutex_trylock(pthread_mutex_t*);
+int __pthread_mutex_lock_pessimistic_np(pthread_mutex_t*);
 int __pthread_mutex_unlock(pthread_mutex_t*);
-int __pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*);
 
 typedef void (*KeyDestructor)(void*);
 

Userland/Libraries/LibC/pthread_integration.cpp

@@ -6,10 +6,12 @@
 
 #include <AK/Atomic.h>
 #include <AK/NeverDestroyed.h>
+#include <AK/Types.h>
 #include <AK/Vector.h>
 #include <bits/pthread_integration.h>
 #include <errno.h>
 #include <sched.h>
+#include <serenity.h>
 #include <unistd.h>
 
 namespace {
@@ -91,65 +93,111 @@ int __pthread_self()
 
 int pthread_self() __attribute__((weak, alias("__pthread_self")));
 
-int __pthread_mutex_lock(pthread_mutex_t* mutex)
+static constexpr u32 MUTEX_UNLOCKED = 0;
+static constexpr u32 MUTEX_LOCKED_NO_NEED_TO_WAKE = 1;
+static constexpr u32 MUTEX_LOCKED_NEED_TO_WAKE = 2;
+
+int __pthread_mutex_init(pthread_mutex_t* mutex, const pthread_mutexattr_t* attributes)
 {
-    pthread_t this_thread = __pthread_self();
-    for (;;) {
-        u32 expected = 0;
-        if (!AK::atomic_compare_exchange_strong(&mutex->lock, expected, 1u, AK::memory_order_acquire)) {
-            if (mutex->type == __PTHREAD_MUTEX_RECURSIVE && mutex->owner == this_thread) {
-                mutex->level++;
-                return 0;
-            }
-            sched_yield();
-            continue;
-        }
-        mutex->owner = this_thread;
-        mutex->level = 0;
-        return 0;
-    }
+    mutex->lock = 0;
+    mutex->owner = 0;
+    mutex->level = 0;
+    mutex->type = attributes ? attributes->type : __PTHREAD_MUTEX_NORMAL;
+    return 0;
 }
 
-int pthread_mutex_lock(pthread_mutex_t*) __attribute__((weak, alias("__pthread_mutex_lock")));
+int pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*) __attribute__((weak, alias("__pthread_mutex_init")));
 
-int __pthread_mutex_unlock(pthread_mutex_t* mutex)
+int __pthread_mutex_trylock(pthread_mutex_t* mutex)
 {
-    if (mutex->type == __PTHREAD_MUTEX_RECURSIVE && mutex->level > 0) {
-        mutex->level--;
+    u32 expected = MUTEX_UNLOCKED;
+    bool exchanged = AK::atomic_compare_exchange_strong(&mutex->lock, expected, MUTEX_LOCKED_NO_NEED_TO_WAKE, AK::memory_order_acquire);
+
+    if (exchanged) [[likely]] {
+        AK::atomic_store(&mutex->owner, __pthread_self(), AK::memory_order_relaxed);
+        mutex->level = 0;
         return 0;
+    } else if (mutex->type == __PTHREAD_MUTEX_RECURSIVE) {
+        pthread_t owner = AK::atomic_load(&mutex->owner, AK::memory_order_relaxed);
+        if (owner == __pthread_self()) {
+            // We already own the mutex!
+            mutex->level++;
+            return 0;
+        }
     }
-    mutex->owner = 0;
-    AK::atomic_store(&mutex->lock, 0u, AK::memory_order_release);
-    return 0;
+    return EBUSY;
 }
 
-int pthread_mutex_unlock(pthread_mutex_t*) __attribute__((weak, alias("__pthread_mutex_unlock")));
+int pthread_mutex_trylock(pthread_mutex_t* mutex) __attribute__((weak, alias("__pthread_mutex_trylock")));
 
-int __pthread_mutex_trylock(pthread_mutex_t* mutex)
+int __pthread_mutex_lock(pthread_mutex_t* mutex)
 {
-    u32 expected = 0;
-    if (!AK::atomic_compare_exchange_strong(&mutex->lock, expected, 1u, AK::memory_order_acquire)) {
-        if (mutex->type == __PTHREAD_MUTEX_RECURSIVE && mutex->owner == pthread_self()) {
+    pthread_t this_thread = __pthread_self();
+
+    // Fast path: attempt to claim the mutex without waiting.
+    u32 value = MUTEX_UNLOCKED;
+    bool exchanged = AK::atomic_compare_exchange_strong(&mutex->lock, value, MUTEX_LOCKED_NO_NEED_TO_WAKE, AK::memory_order_acquire);
+    if (exchanged) [[likely]] {
+        AK::atomic_store(&mutex->owner, this_thread, AK::memory_order_relaxed);
+        mutex->level = 0;
+        return 0;
+    } else if (mutex->type == __PTHREAD_MUTEX_RECURSIVE) {
+        pthread_t owner = AK::atomic_load(&mutex->owner, AK::memory_order_relaxed);
+        if (owner == this_thread) {
+            // We already own the mutex!
             mutex->level++;
             return 0;
         }
-        return EBUSY;
     }
-    mutex->owner = pthread_self();
+
+    // Slow path: wait, record the fact that we're going to wait, and always
+    // remember to wake the next thread up once we release the mutex.
+    if (value != MUTEX_LOCKED_NEED_TO_WAKE)
+        value = AK::atomic_exchange(&mutex->lock, MUTEX_LOCKED_NEED_TO_WAKE, AK::memory_order_acquire);
+
+    while (value != MUTEX_UNLOCKED) {
+        futex_wait(&mutex->lock, value, nullptr, 0);
+        value = AK::atomic_exchange(&mutex->lock, MUTEX_LOCKED_NEED_TO_WAKE, AK::memory_order_acquire);
+    }
+
+    AK::atomic_store(&mutex->owner, this_thread, AK::memory_order_relaxed);
     mutex->level = 0;
     return 0;
 }
 
-int pthread_mutex_trylock(pthread_mutex_t* mutex) __attribute__((weak, alias("__pthread_mutex_trylock")));
+int pthread_mutex_lock(pthread_mutex_t*) __attribute__((weak, alias("__pthread_mutex_lock")));
 
-int __pthread_mutex_init(pthread_mutex_t* mutex, const pthread_mutexattr_t* attributes)
+int __pthread_mutex_lock_pessimistic_np(pthread_mutex_t* mutex)
 {
-    mutex->lock = 0;
-    mutex->owner = 0;
+    // Same as pthread_mutex_lock(), but always set MUTEX_LOCKED_NEED_TO_WAKE,
+    // and also don't bother checking for already owning the mutex recursively,
+    // because we know we don't. Used in the condition variable implementation.
+    u32 value = AK::atomic_exchange(&mutex->lock, MUTEX_LOCKED_NEED_TO_WAKE, AK::memory_order_acquire);
+    while (value != MUTEX_UNLOCKED) {
+        futex_wait(&mutex->lock, value, nullptr, 0);
+        value = AK::atomic_exchange(&mutex->lock, MUTEX_LOCKED_NEED_TO_WAKE, AK::memory_order_acquire);
+    }
+
+    AK::atomic_store(&mutex->owner, __pthread_self(), AK::memory_order_relaxed);
     mutex->level = 0;
-    mutex->type = attributes ? attributes->type : __PTHREAD_MUTEX_NORMAL;
     return 0;
 }
 
-int pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*) __attribute__((weak, alias("__pthread_mutex_init")));
+int __pthread_mutex_unlock(pthread_mutex_t* mutex)
+{
+    if (mutex->type == __PTHREAD_MUTEX_RECURSIVE && mutex->level > 0) {
+        mutex->level--;
+        return 0;
+    }
+
+    AK::atomic_store(&mutex->owner, 0, AK::memory_order_relaxed);
+
+    u32 value = AK::atomic_exchange(&mutex->lock, MUTEX_UNLOCKED, AK::memory_order_release);
+    if (value == MUTEX_LOCKED_NEED_TO_WAKE) [[unlikely]]
+        futex_wake(&mutex->lock, 1);
+
+    return 0;
+}
+
+int pthread_mutex_unlock(pthread_mutex_t*) __attribute__((weak, alias("__pthread_mutex_unlock")));
 }

Userland/Libraries/LibC/serenity.h

@@ -7,7 +7,9 @@
 #pragma once
 
 #include <stdio.h>
+#include <sys/cdefs.h>
 #include <sys/types.h>
+#include <time.h>
 #include <unistd.h>
 
 __BEGIN_DECLS
@@ -70,6 +72,26 @@ int profiling_free_buffer(pid_t);
 
 int futex(uint32_t* userspace_address, int futex_op, uint32_t value, const struct timespec* timeout, uint32_t* userspace_address2, uint32_t value3);
 
+static ALWAYS_INLINE int futex_wait(uint32_t* userspace_address, uint32_t value, const struct timespec* abstime, int clockid)
+{
+    int op;
+
+    if (abstime) {
+        // NOTE: FUTEX_WAIT takes a relative timeout, so use FUTEX_WAIT_BITSET instead!
+        op = FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG;
+        if (clockid == CLOCK_REALTIME || clockid == CLOCK_REALTIME_COARSE)
+            op |= FUTEX_CLOCK_REALTIME;
+    } else {
+        op = FUTEX_WAIT;
+    }
+    return futex(userspace_address, op, value, abstime, nullptr, FUTEX_BITSET_MATCH_ANY);
+}
+
+static ALWAYS_INLINE int futex_wake(uint32_t* userspace_address, uint32_t count)
+{
+    return futex(userspace_address, FUTEX_WAKE, count, NULL, NULL, 0);
+}
+
 #define PURGE_ALL_VOLATILE 0x1
 #define PURGE_ALL_CLEAN_INODE 0x2
 

Userland/Libraries/LibC/sys/types.h

@@ -75,8 +75,8 @@ typedef struct __pthread_mutexattr_t {
 } pthread_mutexattr_t;
 
 typedef struct __pthread_cond_t {
+    pthread_mutex_t* mutex;
     uint32_t value;
-    uint32_t previous;
     int clockid; // clockid_t
 } pthread_cond_t;
 

Userland/Libraries/LibPthread/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(SOURCES
     forward.cpp
     pthread.cpp
+    pthread_cond.cpp
     pthread_once.cpp
     semaphore.cpp
 )

Userland/Libraries/LibPthread/pthread.cpp

@@ -445,89 +445,6 @@ int pthread_setschedparam([[maybe_unused]] pthread_t thread, [[maybe_unused]] in
     return 0;
 }
 
-int pthread_cond_init(pthread_cond_t* cond, const pthread_condattr_t* attr)
-{
-    cond->value = 0;
-    cond->previous = 0;
-    cond->clockid = attr ? attr->clockid : CLOCK_MONOTONIC_COARSE;
-    return 0;
-}
-
-int pthread_cond_destroy(pthread_cond_t*)
-{
-    return 0;
-}
-
-static int futex_wait(uint32_t& futex_addr, uint32_t value, const struct timespec* abstime)
-{
-    int saved_errno = errno;
-    // NOTE: FUTEX_WAIT takes a relative timeout, so use FUTEX_WAIT_BITSET instead!
-    int rc = futex(&futex_addr, FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, value, abstime, nullptr, FUTEX_BITSET_MATCH_ANY);
-    if (rc < 0 && errno == EAGAIN) {
-        // If we didn't wait, that's not an error
-        errno = saved_errno;
-        rc = 0;
-    }
-    return rc;
-}
-
-static int cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex, const struct timespec* abstime)
-{
-    u32 value = cond->value;
-    cond->previous = value;
-    pthread_mutex_unlock(mutex);
-    int rc = futex_wait(cond->value, value, abstime);
-    pthread_mutex_lock(mutex);
-    return rc;
-}
-
-int pthread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex)
-{
-    int rc = cond_wait(cond, mutex, nullptr);
-    VERIFY(rc == 0);
-    return 0;
-}
-
-int pthread_condattr_init(pthread_condattr_t* attr)
-{
-    attr->clockid = CLOCK_MONOTONIC_COARSE;
-    return 0;
-}
-
-int pthread_condattr_destroy(pthread_condattr_t*)
-{
-    return 0;
-}
-
-int pthread_condattr_setclock(pthread_condattr_t* attr, clockid_t clock)
-{
-    attr->clockid = clock;
-    return 0;
-}
-
-int pthread_cond_timedwait(pthread_cond_t* cond, pthread_mutex_t* mutex, const struct timespec* abstime)
-{
-    return cond_wait(cond, mutex, abstime);
-}
-
-int pthread_cond_signal(pthread_cond_t* cond)
-{
-    u32 value = cond->previous + 1;
-    cond->value = value;
-    int rc = futex(&cond->value, FUTEX_WAKE, 1, nullptr, nullptr, 0);
-    VERIFY(rc >= 0);
-    return 0;
-}
-
-int pthread_cond_broadcast(pthread_cond_t* cond)
-{
-    u32 value = cond->previous + 1;
-    cond->value = value;
-    int rc = futex(&cond->value, FUTEX_WAKE, INT32_MAX, nullptr, nullptr, 0);
-    VERIFY(rc >= 0);
-    return 0;
-}
-
 // libgcc expects this function to exist in libpthread, even
 // if it is not implemented.
 int pthread_cancel(pthread_t)