From 8621f25065bceb47a97cffe9698a03be0aa900bf Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 10 Dec 2024 14:05:20 -0800 Subject: [PATCH] GdbServer: Changes how thread addition and pausing works This is enough to get GdbServer working for my simple test case again. Some things of note: - There is now a callback which signals to GdbServer when a thread is created. - Thread sleeping is now a single uint32_t futex word for knowing when a thread is sleeping and if we need to signal it to wake up. - GdbServer thread sleeping versus ThreadManager thread sleeping distinction has been removed. GdbServer now uses ThreadManager to put a thread to sleep. This still only gets the attach at startup scheme of gdbserver working. I haven't yet dived back in to getting arbitrary attach working. Primarily the work necessary here is actually getting thread creation at the start to stop and wait for gdbserver to attach. Then actually sleeping correctly while gdbserver is communicating with gdb. --- .../LinuxSyscalls/GdbServer.cpp | 23 +++-- .../LinuxEmulation/LinuxSyscalls/GdbServer.h | 5 +- .../LinuxSyscalls/SignalDelegator.cpp | 2 +- .../LinuxEmulation/LinuxSyscalls/Syscalls.h | 2 +- .../LinuxSyscalls/ThreadManager.cpp | 31 ++++-- .../LinuxSyscalls/ThreadManager.h | 96 +++++++++++++++++-- 6 files changed, 130 insertions(+), 29 deletions(-) diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp index 03039ae5ef..0c83cda56e 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp @@ -10,6 +10,7 @@ desc: Provides a gdb interface to the guest state #include "GdbServer/Info.h" #include "LinuxSyscalls/NetStream.h" +#include "LinuxSyscalls/SignalDelegator.h" #include #include @@ -75,11 +76,6 @@ void GdbServer::Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal) { SendPacket(*CommsStream, str); } -void GdbServer::WaitForThreadWakeup() { - // Wait for gdbserver to tell us to wake up - ThreadBreakEvent.Wait(); -} - GdbServer::~GdbServer() { CloseListenSocket(); CoreShuttingDown = true; @@ -120,7 +116,7 @@ GdbServer::GdbServer(FEXCore::Context::Context* ctx, FEX::HLE::SignalDelegator* // Let GDB know that we have a signal this->Break(ThreadObject, Signal); - WaitForThreadWakeup(); + this->SyscallHandler->TM.SleepThread(this->CTX, ThreadObject); ThreadObject->GdbInfo.reset(); return true; @@ -604,8 +600,14 @@ GdbServer::HandledPacketType GdbServer::handleProgramOffsets() { GdbServer::HandledPacketType GdbServer::ThreadAction(char action, uint32_t tid) { switch (action) { case 'c': { + { + std::lock_guard lk(*SyscallHandler->TM.GetThreadsCreationMutex()); + auto Threads = SyscallHandler->TM.GetThreads(); + for (auto& Thread : *Threads) { + Thread->ThreadSleeping.NotifyOne(); + } + } SyscallHandler->TM.Run(); - ThreadBreakEvent.NotifyAll(); SyscallHandler->TM.WaitForThreadsToRun(); return {"", HandledPacketType::TYPE_ONLYACK}; } @@ -1450,6 +1452,13 @@ void GdbServer::StartThread() { FEXCore::Threads::SetSignalMask(OldMask); } +void GdbServer::OnThreadCreated(FEX::HLE::ThreadStateObject* ThreadObject) { + if (SyscallHandler->TM.GetThreadCount() == 1) { + // Sleep the first thread created. This is because FEX only supports attaching at startup currently. + SyscallHandler->TM.SleepThread(CTX, ThreadObject); + } +} + void GdbServer::OpenListenSocket() { const auto GdbUnixPath = fextl::fmt::format("{}/FEX_gdbserver/", FEXServerClient::GetTempFolder()); if (FHU::Filesystem::CreateDirectory(GdbUnixPath) == FHU::Filesystem::CreateDirectoryResult::ERROR) { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h index 0b4cc726c3..425eabb7ee 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h @@ -35,6 +35,8 @@ class GdbServer { LibraryMapChanged = true; } + void OnThreadCreated(FEX::HLE::ThreadStateObject* ThreadObject); + private: void Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal); @@ -52,9 +54,6 @@ class GdbServer { void SendACK(std::ostream& stream, bool NACK); - Event ThreadBreakEvent {}; - void WaitForThreadWakeup(); - struct HandledPacketType { fextl::string Response {}; enum ResponseType { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp index b556690abc..5f58a9d649 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp @@ -476,7 +476,7 @@ bool SignalDelegator::HandleSignalPause(FEXCore::Core::InternalThreadState* Thre // We need to be a little bit careful here // If we were already paused (due to GDB) and we are immediately stopping (due to gdb kill) // Then we need to ensure we don't double decrement our idle thread counter - if (ThreadObject->ThreadSleeping) { + if (ThreadObject->ThreadSleeping.HasWaiter()) { // If the thread was sleeping then its idle counter was decremented // Reincrement it here to not break logic FEX::HLE::_SyscallHandler->TM.IncrementIdleRefCount(); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h index 04b66c0239..f78e0e29b7 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h @@ -53,7 +53,7 @@ desc: Glue logic, STRACE magic namespace FEX { class CodeLoader; class GdbServer; -} +} // namespace FEX namespace FEXCore { namespace Context { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp index 75cb1d87e5..a47d6f840d 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +#include "LinuxSyscalls/GdbServer.h" #include "LinuxSyscalls/Syscalls.h" #include "LinuxSyscalls/SignalDelegator.h" @@ -29,6 +30,18 @@ FEX::HLE::ThreadStateObject* ThreadManager::CreateThread(uint64_t InitialRIP, ui return ThreadStateObject; } +void ThreadManager::TrackThread(FEX::HLE::ThreadStateObject* Thread) { + { + std::lock_guard lk(ThreadCreationMutex); + Threads.emplace_back(Thread); + } + + auto GdbServer = SyscallHandler->GetGdbServer(); + if (GdbServer) { + GdbServer->OnThreadCreated(Thread); + } +} + void ThreadManager::DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall) { { std::lock_guard lk(ThreadCreationMutex); @@ -71,7 +84,10 @@ void ThreadManager::NotifyPause() { // Tell all the threads that they should pause std::lock_guard lk(ThreadCreationMutex); for (auto& Thread : Threads) { - SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause); + if (!Thread->ThreadSleeping.HasWaiter()) { + // Only signal if it isn't already sleeping. + SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause); + } } } @@ -162,25 +178,20 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) { } } -void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) { - auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame); - +void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject) { --IdleWaitRefCount; IdleWaitCV.notify_all(); - ThreadObject->ThreadSleeping = true; - - // Go to sleep - ThreadObject->ThreadPaused.Wait(); + // Go to sleep. + ThreadObject->ThreadSleeping.Wait(); ++IdleWaitRefCount; - ThreadObject->ThreadSleeping = false; IdleWaitCV.notify_all(); } void ThreadManager::UnpauseThread(FEX::HLE::ThreadStateObject* Thread) { - Thread->ThreadPaused.NotifyOne(); + Thread->ThreadSleeping.NotifyOne(); } void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) { diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h index 516b1d065a..d18d26118e 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h @@ -22,6 +22,79 @@ namespace FEX::HLE { class SyscallHandler; class SignalDelegator; +// A latch that can be inspected to see if there is a waiter currently active. +// This allows us to remove the race condition between a thread trying to go asleep and something else telling it to go to sleep or wake up. +// +// Only one thread can ever wait on a latch, while another thread signals it. +class InspectableLatch final { +public: + bool Wait(struct timespec* Timeout = nullptr) { + while (true) { + uint32_t Expected = HAS_NO_WAITER; + const uint32_t Desired = HAS_WAITER; + + if (Mutex.compare_exchange_strong(Expected, Desired)) { + // We have latched, now futex. + constexpr int Op = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; + // WAIT will keep sleeping on the futex word while it is `val` + int Result = ::syscall(SYS_futex, &Mutex, Op, + Desired, // val + Timeout, // Timeout/val2 + nullptr, // Addr2 + 0); // val3 + + if (Timeout && Result == -1 && errno == ETIMEDOUT) { + return false; + } + } else if (Expected == HAS_SIGNALED) { + // Reset the latch once signaled + Mutex.store(HAS_NO_WAITER); + return true; + } + } + } + + template + bool WaitFor(const std::chrono::duration& time) { + struct timespec Timeout {}; + auto SecondsDuration = std::chrono::duration_cast(time); + Timeout.tv_sec = SecondsDuration.count(); + Timeout.tv_nsec = std::chrono::duration_cast(time - SecondsDuration).count(); + return Wait(&Timeout); + } + + void NotifyOne() { + DoNotify(1); + } + + bool HasWaiter() const { + return Mutex.load() == HAS_WAITER; + } + +private: + std::atomic Mutex {}; + constexpr static uint32_t HAS_NO_WAITER = 0; + constexpr static uint32_t HAS_WAITER = 1; + constexpr static uint32_t HAS_SIGNALED = 2; + + void DoNotify(int Waiters) { + uint32_t Expected = HAS_WAITER; + const uint32_t Desired = HAS_SIGNALED; + + // If the mutex is in a waiting state and we have CAS exchanged it to HAS_SIGNALED, then execute a futex syscall. + // otherwise just leave since nothing was waiting. + if (Mutex.compare_exchange_strong(Expected, Desired)) { + constexpr int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; + + ::syscall(SYS_futex, &Mutex, Op, + Waiters, // val - Number of waiters to wake + 0, // val2 + &Mutex, // Addr2 - Mutex to do the operation on + 0); // val3 + } + } +}; + enum class SignalEvent : uint32_t { Nothing, // If the guest uses our signal we need to know it was errant on our end Pause, @@ -81,8 +154,7 @@ struct ThreadStateObject : public FEXCore::Allocator::FEXAllocOperators { std::atomic SignalReason {SignalEvent::Nothing}; // Thread pause handling - std::atomic_bool ThreadSleeping {false}; - FEXCore::InterruptableConditionVariable ThreadPaused; + InspectableLatch ThreadSleeping; // GDB signal information struct GdbInfoStruct { @@ -117,10 +189,7 @@ class ThreadManager final { FEX::HLE::ThreadStateObject* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXCore::Core::CPUState* NewThreadState = nullptr, uint64_t ParentTID = 0, FEX::HLE::ThreadStateObject* InheritThread = nullptr); - void TrackThread(FEX::HLE::ThreadStateObject* Thread) { - std::lock_guard lk(ThreadCreationMutex); - Threads.emplace_back(Thread); - } + void TrackThread(FEX::HLE::ThreadStateObject* Thread); void DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall = false); void StopThread(FEX::HLE::ThreadStateObject* Thread); @@ -135,7 +204,11 @@ class ThreadManager final { void WaitForIdleWithTimeout(); void WaitForThreadsToRun(); - void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame); + void SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject); + void SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) { + auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame); + SleepThread(CTX, ThreadObject); + } void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child); @@ -170,10 +243,19 @@ class ThreadManager final { } } + FEXCore::ForkableUniqueMutex* GetThreadsCreationMutex() { + return &ThreadCreationMutex; + } + const fextl::vector* GetThreads() const { return &Threads; } + size_t GetThreadCount() { + std::lock_guard lk(ThreadCreationMutex); + return Threads.size(); + } + private: FEXCore::Context::Context* CTX; FEX::HLE::SyscallHandler* SyscallHandler;