diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 78b44330aedb..1273ffc1add4 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -28,17 +28,18 @@ namespace sycl { __SYCL_INLINE_VER_NAMESPACE(_V1) { namespace detail { +using LockGuard = std::lock_guard; +SpinLock GlobalHandler::MSyclGlobalHandlerProtector{}; + // Utility class to track references on object. -// Used for Scheduler now and created as thread_local object. -// Origin idea is to track usage of Scheduler from main and other used threads - -// they increment MCounter; and to use but not add extra reference by our -// thread_pool threads. For this control MIncrementCounter class member is used. -template class ObjectUsageCounter { +// Used for GlobalHandler now and created as thread_local object on the first +// Scheduler usage. Origin idea is to track usage of Scheduler from main and +// other used threads - they increment MCounter; and to use but not add extra +// reference by our thread_pool threads. For this control MIncrementCounter +// class member is used. +class ObjectUsageCounter { public: - // Note: -Wctad-maybe-unsupported may generate warning if no ResourceHandler - // type explicitly declared. - ObjectUsageCounter(std::unique_ptr &Obj, bool ModifyCounter) - : MModifyCounter(ModifyCounter), MObj(Obj) { + ObjectUsageCounter(bool ModifyCounter) : MModifyCounter(ModifyCounter) { if (MModifyCounter) MCounter++; } @@ -47,26 +48,35 @@ template class ObjectUsageCounter { return; MCounter--; - if (!MCounter && MObj) - MObj->releaseResources(); + if (!MCounter) { + LockGuard Guard(GlobalHandler::MSyclGlobalHandlerProtector); + GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr(); + if (RTGlobalObjHandler) { + RTGlobalObjHandler->drainThreadPool(); + if (RTGlobalObjHandler->MScheduler.Inst) + RTGlobalObjHandler->MScheduler.Inst->releaseResources(); + } + } } private: static std::atomic_uint MCounter; bool MModifyCounter; - std::unique_ptr &MObj; }; -template -std::atomic_uint ObjectUsageCounter::MCounter{0}; - -using LockGuard = std::lock_guard; +std::atomic_uint ObjectUsageCounter::MCounter{0}; GlobalHandler::GlobalHandler() = default; GlobalHandler::~GlobalHandler() = default; +GlobalHandler *&GlobalHandler::getInstancePtr() { + static GlobalHandler *RTGlobalObjHandler = new GlobalHandler(); + return RTGlobalObjHandler; +} + GlobalHandler &GlobalHandler::instance() { - static GlobalHandler *SyclGlobalObjectsHandler = new GlobalHandler(); - return *SyclGlobalObjectsHandler; + GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr(); + assert(RTGlobalObjHandler && "Handler must not be deallocated earlier"); + return *RTGlobalObjHandler; } template @@ -94,8 +104,7 @@ Scheduler &GlobalHandler::getScheduler() { } void GlobalHandler::registerSchedulerUsage(bool ModifyCounter) { - thread_local ObjectUsageCounter SchedulerCounter(MScheduler.Inst, - ModifyCounter); + thread_local ObjectUsageCounter SchedulerCounter(ModifyCounter); } ProgramManager &GlobalHandler::getProgramManager() { @@ -151,14 +160,14 @@ ThreadPool &GlobalHandler::getHostTaskThreadPool() { void GlobalHandler::releaseDefaultContexts() { // Release shared-pointers to SYCL objects. #ifndef _WIN32 - GlobalHandler::instance().MPlatformToDefaultContextCache.Inst.reset(nullptr); + MPlatformToDefaultContextCache.Inst.reset(nullptr); #else // Windows does not maintain dependencies between dynamically loaded libraries // and can unload SYCL runtime dependencies before sycl.dll's DllMain has // finished. To avoid calls to nowhere, intentionally leak platform to device // cache. This will prevent destructors from being called, thus no PI cleanup // routines will be called in the end. - GlobalHandler::instance().MPlatformToDefaultContextCache.Inst.release(); + MPlatformToDefaultContextCache.Inst.release(); #endif } @@ -178,8 +187,8 @@ void GlobalHandler::unloadPlugins() { // Call to GlobalHandler::instance().getPlugins() initializes plugins. If // user application has loaded SYCL runtime, and never called any APIs, // there's no need to load and unload plugins. - if (GlobalHandler::instance().MPlugins.Inst) { - for (plugin &Plugin : GlobalHandler::instance().getPlugins()) { + if (MPlugins.Inst) { + for (plugin &Plugin : getPlugins()) { // PluginParameter is reserved for future use that can control // some parameters in the plugin tear-down process. // Currently, it is not used. @@ -189,7 +198,7 @@ void GlobalHandler::unloadPlugins() { } } // Clear after unload to avoid uses after unload. - GlobalHandler::instance().getPlugins().clear(); + getPlugins().clear(); } void GlobalHandler::drainThreadPool() { @@ -198,34 +207,40 @@ void GlobalHandler::drainThreadPool() { } void shutdown() { + const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; + GlobalHandler *&Handler = GlobalHandler::getInstancePtr(); + if (!Handler) + return; + // Ensure neither host task is working so that no default context is accessed // upon its release + Handler->drainThreadPool(); + if (Handler->MScheduler.Inst) + Handler->MScheduler.Inst->releaseResources(); - if (GlobalHandler::instance().MScheduler.Inst) - GlobalHandler::instance().MScheduler.Inst->releaseResources(); - - if (GlobalHandler::instance().MHostTaskThreadPool.Inst) - GlobalHandler::instance().MHostTaskThreadPool.Inst->finishAndWait(); + if (Handler->MHostTaskThreadPool.Inst) + Handler->MHostTaskThreadPool.Inst->finishAndWait(); // If default contexts are requested after the first default contexts have // been released there may be a new default context. These must be released // prior to closing the plugins. // Note: Releasing a default context here may cause failures in plugins with // global state as the global state may have been released. - GlobalHandler::instance().releaseDefaultContexts(); + Handler->releaseDefaultContexts(); // First, release resources, that may access plugins. - GlobalHandler::instance().MPlatformCache.Inst.reset(nullptr); - GlobalHandler::instance().MScheduler.Inst.reset(nullptr); - GlobalHandler::instance().MProgramManager.Inst.reset(nullptr); + Handler->MPlatformCache.Inst.reset(nullptr); + Handler->MScheduler.Inst.reset(nullptr); + Handler->MProgramManager.Inst.reset(nullptr); // Clear the plugins and reset the instance if it was there. - GlobalHandler::instance().unloadPlugins(); - if (GlobalHandler::instance().MPlugins.Inst) - GlobalHandler::instance().MPlugins.Inst.reset(nullptr); + Handler->unloadPlugins(); + if (Handler->MPlugins.Inst) + Handler->MPlugins.Inst.reset(nullptr); // Release the rest of global resources. - delete &GlobalHandler::instance(); + delete Handler; + Handler = nullptr; } #ifdef _WIN32 diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index 3335705d5158..a9b29a15b939 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -83,6 +83,9 @@ class GlobalHandler { private: friend void shutdown(); + friend class ObjectUsageCounter; + static GlobalHandler *&getInstancePtr(); + static SpinLock MSyclGlobalHandlerProtector; // Constructor and destructor are declared out-of-line to allow incomplete // types as template arguments to unique_ptr. diff --git a/sycl/source/detail/scheduler/scheduler.cpp b/sycl/source/detail/scheduler/scheduler.cpp index e2e7f5df48ce..2a36fde74192 100644 --- a/sycl/source/detail/scheduler/scheduler.cpp +++ b/sycl/source/detail/scheduler/scheduler.cpp @@ -392,11 +392,6 @@ Scheduler::~Scheduler() { DefaultHostQueue.reset(); } void Scheduler::releaseResources() { #ifndef _WIN32 - if (DefaultHostQueue) { - DefaultHostQueue->wait(); - } - GlobalHandler::instance().drainThreadPool(); - // There might be some commands scheduled for post enqueue cleanup that // haven't been freed because of the graph mutex being locked at the time, // clean them up now.