From 7f1078110b4f73eda4ab6587e402ecf6fa3f30bb Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Tue, 3 Jan 2023 10:39:52 -0800 Subject: [PATCH 01/10] Fix segfault on program exit when user thread is not finished yet (caused by #6837) Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 64 ++++++++++++---------- sycl/source/detail/global_handler.hpp | 3 +- sycl/source/detail/scheduler/scheduler.cpp | 1 - 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 78b44330aedb..255c2bfb7e62 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -28,6 +28,8 @@ namespace sycl { __SYCL_INLINE_VER_NAMESPACE(_V1) { namespace detail { +GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler = new GlobalHandler(); + // Utility class to track references on object. // Used for Scheduler now and created as thread_local object. // Origin idea is to track usage of Scheduler from main and other used threads - @@ -37,7 +39,7 @@ template class ObjectUsageCounter { public: // Note: -Wctad-maybe-unsupported may generate warning if no ResourceHandler // type explicitly declared. - ObjectUsageCounter(std::unique_ptr &Obj, bool ModifyCounter) + ObjectUsageCounter(ResourceHandler *&Obj, bool ModifyCounter) : MModifyCounter(ModifyCounter), MObj(Obj) { if (MModifyCounter) MCounter++; @@ -54,7 +56,7 @@ template class ObjectUsageCounter { private: static std::atomic_uint MCounter; bool MModifyCounter; - std::unique_ptr &MObj; + ResourceHandler *&MObj; }; template std::atomic_uint ObjectUsageCounter::MCounter{0}; @@ -65,8 +67,9 @@ GlobalHandler::GlobalHandler() = default; GlobalHandler::~GlobalHandler() = default; GlobalHandler &GlobalHandler::instance() { - static GlobalHandler *SyclGlobalObjectsHandler = new GlobalHandler(); - return *SyclGlobalObjectsHandler; + assert(MSyclGlobalObjectsHandler && + "Handler must not be deallocated earlier"); + return *MSyclGlobalObjectsHandler; } template @@ -94,8 +97,8 @@ Scheduler &GlobalHandler::getScheduler() { } void GlobalHandler::registerSchedulerUsage(bool ModifyCounter) { - thread_local ObjectUsageCounter SchedulerCounter(MScheduler.Inst, - ModifyCounter); + thread_local ObjectUsageCounter SchedulerCounter( + MSyclGlobalObjectsHandler, ModifyCounter); } ProgramManager &GlobalHandler::getProgramManager() { @@ -151,14 +154,14 @@ ThreadPool &GlobalHandler::getHostTaskThreadPool() { void GlobalHandler::releaseDefaultContexts() { // Release shared-pointers to SYCL objects. #ifndef _WIN32 - GlobalHandler::instance().MPlatformToDefaultContextCache.Inst.reset(nullptr); + MPlatformToDefaultContextCache.Inst.reset(nullptr); #else // Windows does not maintain dependencies between dynamically loaded libraries // and can unload SYCL runtime dependencies before sycl.dll's DllMain has // finished. To avoid calls to nowhere, intentionally leak platform to device // cache. This will prevent destructors from being called, thus no PI cleanup // routines will be called in the end. - GlobalHandler::instance().MPlatformToDefaultContextCache.Inst.release(); + MPlatformToDefaultContextCache.Inst.release(); #endif } @@ -178,8 +181,8 @@ void GlobalHandler::unloadPlugins() { // Call to GlobalHandler::instance().getPlugins() initializes plugins. If // user application has loaded SYCL runtime, and never called any APIs, // there's no need to load and unload plugins. - if (GlobalHandler::instance().MPlugins.Inst) { - for (plugin &Plugin : GlobalHandler::instance().getPlugins()) { + if (MPlugins.Inst) { + for (plugin &Plugin : getPlugins()) { // PluginParameter is reserved for future use that can control // some parameters in the plugin tear-down process. // Currently, it is not used. @@ -189,43 +192,48 @@ void GlobalHandler::unloadPlugins() { } } // Clear after unload to avoid uses after unload. - GlobalHandler::instance().getPlugins().clear(); -} - -void GlobalHandler::drainThreadPool() { - if (MHostTaskThreadPool.Inst) - MHostTaskThreadPool.Inst->drain(); + getPlugins().clear(); } void shutdown() { + GlobalHandler *handler = nullptr; + std::swap(handler, GlobalHandler::MSyclGlobalObjectsHandler); + assert(handler && "Handler could not be deallocated earlier"); // Ensure neither host task is working so that no default context is accessed // upon its release - if (GlobalHandler::instance().MScheduler.Inst) - GlobalHandler::instance().MScheduler.Inst->releaseResources(); + if (handler->MScheduler.Inst) + handler->MScheduler.Inst->releaseResources(); - if (GlobalHandler::instance().MHostTaskThreadPool.Inst) - GlobalHandler::instance().MHostTaskThreadPool.Inst->finishAndWait(); + if (handler->MHostTaskThreadPool.Inst) + handler->MHostTaskThreadPool.Inst->finishAndWait(); // If default contexts are requested after the first default contexts have // been released there may be a new default context. These must be released // prior to closing the plugins. // Note: Releasing a default context here may cause failures in plugins with // global state as the global state may have been released. - GlobalHandler::instance().releaseDefaultContexts(); + handler->releaseDefaultContexts(); // First, release resources, that may access plugins. - GlobalHandler::instance().MPlatformCache.Inst.reset(nullptr); - GlobalHandler::instance().MScheduler.Inst.reset(nullptr); - GlobalHandler::instance().MProgramManager.Inst.reset(nullptr); + handler->MPlatformCache.Inst.reset(nullptr); + handler->MScheduler.Inst.reset(nullptr); + handler->MProgramManager.Inst.reset(nullptr); // Clear the plugins and reset the instance if it was there. - GlobalHandler::instance().unloadPlugins(); - if (GlobalHandler::instance().MPlugins.Inst) - GlobalHandler::instance().MPlugins.Inst.reset(nullptr); + handler->unloadPlugins(); + if (handler->MPlugins.Inst) + handler->MPlugins.Inst.reset(nullptr); // Release the rest of global resources. - delete &GlobalHandler::instance(); + delete handler; +} + +void GlobalHandler::releaseResources() { + if (MHostTaskThreadPool.Inst) + MHostTaskThreadPool.Inst->drain(); + if (MScheduler.Inst) + MScheduler.Inst->releaseResources(); } #ifdef _WIN32 diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index 3335705d5158..4d089fd48d2d 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -76,12 +76,13 @@ class GlobalHandler { void unloadPlugins(); void releaseDefaultContexts(); - void drainThreadPool(); // For testing purposes only void attachScheduler(Scheduler *Scheduler); + void releaseResources(); private: + static GlobalHandler *MSyclGlobalObjectsHandler; friend void shutdown(); // Constructor and destructor are declared out-of-line to allow incomplete diff --git a/sycl/source/detail/scheduler/scheduler.cpp b/sycl/source/detail/scheduler/scheduler.cpp index e2e7f5df48ce..74cd78f38812 100644 --- a/sycl/source/detail/scheduler/scheduler.cpp +++ b/sycl/source/detail/scheduler/scheduler.cpp @@ -395,7 +395,6 @@ void Scheduler::releaseResources() { if (DefaultHostQueue) { DefaultHostQueue->wait(); } - GlobalHandler::instance().drainThreadPool(); // There might be some commands scheduled for post enqueue cleanup that // haven't been freed because of the graph mutex being locked at the time, From 7aa073bf6838973dbc1e75c2c99875739551818d Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Wed, 4 Jan 2023 05:49:26 -0800 Subject: [PATCH 02/10] Add protection for handler destruction in case of many user threads Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 29 +++++++++++++++++++-------- sycl/source/detail/global_handler.hpp | 1 + 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 255c2bfb7e62..8dd8256f1551 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -28,7 +28,10 @@ namespace sycl { __SYCL_INLINE_VER_NAMESPACE(_V1) { namespace detail { +using LockGuard = std::lock_guard; + GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler = new GlobalHandler(); +SpinLock GlobalHandler::MSyclGlobalHandlerProtector{}; // Utility class to track references on object. // Used for Scheduler now and created as thread_local object. @@ -39,8 +42,9 @@ template class ObjectUsageCounter { public: // Note: -Wctad-maybe-unsupported may generate warning if no ResourceHandler // type explicitly declared. - ObjectUsageCounter(ResourceHandler *&Obj, bool ModifyCounter) - : MModifyCounter(ModifyCounter), MObj(Obj) { + ObjectUsageCounter(ResourceHandler *&Obj, SpinLock &ObjProtector, + bool ModifyCounter) + : MModifyCounter(ModifyCounter), MObj(Obj), MObjProtector(ObjProtector) { if (MModifyCounter) MCounter++; } @@ -49,24 +53,28 @@ template class ObjectUsageCounter { return; MCounter--; - if (!MCounter && MObj) - MObj->releaseResources(); + if (!MCounter) { + LockGuard Guard(MObjProtector); + if (MObj) + MObj->releaseResources(); + } } private: static std::atomic_uint MCounter; bool MModifyCounter; ResourceHandler *&MObj; + SpinLock &MObjProtector; }; template std::atomic_uint ObjectUsageCounter::MCounter{0}; -using LockGuard = std::lock_guard; - GlobalHandler::GlobalHandler() = default; GlobalHandler::~GlobalHandler() = default; GlobalHandler &GlobalHandler::instance() { + // No protection since sycl usage in parallel with main exit is not valid, + // otherwise MSyclGlobalObjectsHandler exists at any call to instance(). assert(MSyclGlobalObjectsHandler && "Handler must not be deallocated earlier"); return *MSyclGlobalObjectsHandler; @@ -98,7 +106,7 @@ Scheduler &GlobalHandler::getScheduler() { void GlobalHandler::registerSchedulerUsage(bool ModifyCounter) { thread_local ObjectUsageCounter SchedulerCounter( - MSyclGlobalObjectsHandler, ModifyCounter); + MSyclGlobalObjectsHandler, MSyclGlobalHandlerProtector, ModifyCounter); } ProgramManager &GlobalHandler::getProgramManager() { @@ -196,8 +204,13 @@ void GlobalHandler::unloadPlugins() { } void shutdown() { + { printf("%s\n", "shutdown"); }; + GlobalHandler *handler = nullptr; - std::swap(handler, GlobalHandler::MSyclGlobalObjectsHandler); + { + const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; + std::swap(handler, GlobalHandler::MSyclGlobalObjectsHandler); + } assert(handler && "Handler could not be deallocated earlier"); // Ensure neither host task is working so that no default context is accessed // upon its release diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index 4d089fd48d2d..e392f2d55c2e 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -83,6 +83,7 @@ class GlobalHandler { private: static GlobalHandler *MSyclGlobalObjectsHandler; + static SpinLock MSyclGlobalHandlerProtector; friend void shutdown(); // Constructor and destructor are declared out-of-line to allow incomplete From baad330afbb3a4cbaa169184a14d6028ef517679 Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Wed, 4 Jan 2023 06:16:11 -0800 Subject: [PATCH 03/10] restore drainThreadPool method Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 18 +++++++++++------- sycl/source/detail/global_handler.hpp | 4 +++- sycl/source/detail/scheduler/scheduler.cpp | 4 ---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 8dd8256f1551..1d340f10e9d5 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -34,10 +34,11 @@ GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler = new GlobalHandler(); SpinLock GlobalHandler::MSyclGlobalHandlerProtector{}; // Utility class to track references on object. -// Used for Scheduler now and created as thread_local object. -// Origin idea is to track usage of Scheduler from main and other used threads - -// they increment MCounter; and to use but not add extra reference by our -// thread_pool threads. For this control MIncrementCounter class member is used. +// Used for GlobalHandler now and created as thread_local object on the first +// Scheduler usage. Origin idea is to track usage of Scheduler from main and +// other used threads - they increment MCounter; and to use but not add extra +// reference by our thread_pool threads. For this control MIncrementCounter +// class member is used. template class ObjectUsageCounter { public: // Note: -Wctad-maybe-unsupported may generate warning if no ResourceHandler @@ -215,8 +216,7 @@ void shutdown() { // Ensure neither host task is working so that no default context is accessed // upon its release - if (handler->MScheduler.Inst) - handler->MScheduler.Inst->releaseResources(); + handler->releaseResources(); if (handler->MHostTaskThreadPool.Inst) handler->MHostTaskThreadPool.Inst->finishAndWait(); @@ -242,9 +242,13 @@ void shutdown() { delete handler; } -void GlobalHandler::releaseResources() { +void GlobalHandler::drainThreadPool() { if (MHostTaskThreadPool.Inst) MHostTaskThreadPool.Inst->drain(); +} + +void GlobalHandler::releaseResources() { + drainThreadPool(); if (MScheduler.Inst) MScheduler.Inst->releaseResources(); } diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index e392f2d55c2e..ee8fd66fd31e 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -77,9 +77,11 @@ class GlobalHandler { void unloadPlugins(); void releaseDefaultContexts(); + void releaseResources(); + void drainThreadPool(); + // For testing purposes only void attachScheduler(Scheduler *Scheduler); - void releaseResources(); private: static GlobalHandler *MSyclGlobalObjectsHandler; diff --git a/sycl/source/detail/scheduler/scheduler.cpp b/sycl/source/detail/scheduler/scheduler.cpp index 74cd78f38812..2a36fde74192 100644 --- a/sycl/source/detail/scheduler/scheduler.cpp +++ b/sycl/source/detail/scheduler/scheduler.cpp @@ -392,10 +392,6 @@ Scheduler::~Scheduler() { DefaultHostQueue.reset(); } void Scheduler::releaseResources() { #ifndef _WIN32 - if (DefaultHostQueue) { - DefaultHostQueue->wait(); - } - // There might be some commands scheduled for post enqueue cleanup that // haven't been freed because of the graph mutex being locked at the time, // clean them up now. From 2a42d623574c74836c62a1ea7452324cf3606612 Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Wed, 4 Jan 2023 07:30:54 -0800 Subject: [PATCH 04/10] remove print Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 1d340f10e9d5..1c5d0dd90c1a 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -205,8 +205,6 @@ void GlobalHandler::unloadPlugins() { } void shutdown() { - { printf("%s\n", "shutdown"); }; - GlobalHandler *handler = nullptr; { const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; From 9e296153178ccb5d9fa6ba4c2793cc34be97f979 Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Wed, 4 Jan 2023 12:26:28 -0800 Subject: [PATCH 05/10] fix comment with code style Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 1c5d0dd90c1a..aa9fd35cd6e8 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -205,39 +205,39 @@ void GlobalHandler::unloadPlugins() { } void shutdown() { - GlobalHandler *handler = nullptr; + GlobalHandler *Handler = nullptr; { const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; - std::swap(handler, GlobalHandler::MSyclGlobalObjectsHandler); + std::swap(Handler, GlobalHandler::MSyclGlobalObjectsHandler); } - assert(handler && "Handler could not be deallocated earlier"); + assert(Handler && "Handler could not be deallocated earlier"); // Ensure neither host task is working so that no default context is accessed // upon its release - handler->releaseResources(); + Handler->releaseResources(); - if (handler->MHostTaskThreadPool.Inst) - handler->MHostTaskThreadPool.Inst->finishAndWait(); + if (Handler->MHostTaskThreadPool.Inst) + Handler->MHostTaskThreadPool.Inst->finishAndWait(); // If default contexts are requested after the first default contexts have // been released there may be a new default context. These must be released // prior to closing the plugins. // Note: Releasing a default context here may cause failures in plugins with // global state as the global state may have been released. - handler->releaseDefaultContexts(); + Handler->releaseDefaultContexts(); // First, release resources, that may access plugins. - handler->MPlatformCache.Inst.reset(nullptr); - handler->MScheduler.Inst.reset(nullptr); - handler->MProgramManager.Inst.reset(nullptr); + Handler->MPlatformCache.Inst.reset(nullptr); + Handler->MScheduler.Inst.reset(nullptr); + Handler->MProgramManager.Inst.reset(nullptr); // Clear the plugins and reset the instance if it was there. - handler->unloadPlugins(); - if (handler->MPlugins.Inst) - handler->MPlugins.Inst.reset(nullptr); + Handler->unloadPlugins(); + if (Handler->MPlugins.Inst) + Handler->MPlugins.Inst.reset(nullptr); // Release the rest of global resources. - delete handler; + delete Handler; } void GlobalHandler::drainThreadPool() { From c3d0bc048d9d10936477d431e7bbc1a228fe9870 Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Thu, 5 Jan 2023 04:09:47 -0800 Subject: [PATCH 06/10] Fix comments about static var initinalization Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index aa9fd35cd6e8..0dbc496d1791 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -30,7 +30,7 @@ namespace detail { using LockGuard = std::lock_guard; -GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler = new GlobalHandler(); +GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler; SpinLock GlobalHandler::MSyclGlobalHandlerProtector{}; // Utility class to track references on object. @@ -74,6 +74,11 @@ GlobalHandler::GlobalHandler() = default; GlobalHandler::~GlobalHandler() = default; GlobalHandler &GlobalHandler::instance() { + static std::once_flag InitCreation; + std::call_once(InitCreation, []() { + const LockGuard Lock{MSyclGlobalHandlerProtector}; + MSyclGlobalObjectsHandler = new GlobalHandler(); + }); // No protection since sycl usage in parallel with main exit is not valid, // otherwise MSyclGlobalObjectsHandler exists at any call to instance(). assert(MSyclGlobalObjectsHandler && @@ -210,10 +215,11 @@ void shutdown() { const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; std::swap(Handler, GlobalHandler::MSyclGlobalObjectsHandler); } - assert(Handler && "Handler could not be deallocated earlier"); + if (!Handler) + return; + // Ensure neither host task is working so that no default context is accessed // upon its release - Handler->releaseResources(); if (Handler->MHostTaskThreadPool.Inst) From 2ca141871286a77d54b30af7ecd1f2c10867ceb2 Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Thu, 5 Jan 2023 04:14:27 -0800 Subject: [PATCH 07/10] remove outdated comment Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 0dbc496d1791..1f09a486338d 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -79,8 +79,6 @@ GlobalHandler &GlobalHandler::instance() { const LockGuard Lock{MSyclGlobalHandlerProtector}; MSyclGlobalObjectsHandler = new GlobalHandler(); }); - // No protection since sycl usage in parallel with main exit is not valid, - // otherwise MSyclGlobalObjectsHandler exists at any call to instance(). assert(MSyclGlobalObjectsHandler && "Handler must not be deallocated earlier"); return *MSyclGlobalObjectsHandler; @@ -209,6 +207,11 @@ void GlobalHandler::unloadPlugins() { getPlugins().clear(); } +void GlobalHandler::drainThreadPool() { + if (MHostTaskThreadPool.Inst) + MHostTaskThreadPool.Inst->drain(); +} + void shutdown() { GlobalHandler *Handler = nullptr; { @@ -246,11 +249,6 @@ void shutdown() { delete Handler; } -void GlobalHandler::drainThreadPool() { - if (MHostTaskThreadPool.Inst) - MHostTaskThreadPool.Inst->drain(); -} - void GlobalHandler::releaseResources() { drainThreadPool(); if (MScheduler.Inst) From 6e77bef043b40c80b727209917b153dcfb9a60fc Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Thu, 5 Jan 2023 05:01:05 -0800 Subject: [PATCH 08/10] Fix comments, simmplify code Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 47 +++++++++++---------------- sycl/source/detail/global_handler.hpp | 7 ++-- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 1f09a486338d..ff18e7207ccf 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -29,8 +29,6 @@ __SYCL_INLINE_VER_NAMESPACE(_V1) { namespace detail { using LockGuard = std::lock_guard; - -GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler; SpinLock GlobalHandler::MSyclGlobalHandlerProtector{}; // Utility class to track references on object. @@ -39,13 +37,9 @@ SpinLock GlobalHandler::MSyclGlobalHandlerProtector{}; // other used threads - they increment MCounter; and to use but not add extra // reference by our thread_pool threads. For this control MIncrementCounter // class member is used. -template class ObjectUsageCounter { +class ObjectUsageCounter { public: - // Note: -Wctad-maybe-unsupported may generate warning if no ResourceHandler - // type explicitly declared. - ObjectUsageCounter(ResourceHandler *&Obj, SpinLock &ObjProtector, - bool ModifyCounter) - : MModifyCounter(ModifyCounter), MObj(Obj), MObjProtector(ObjProtector) { + ObjectUsageCounter(bool ModifyCounter) : MModifyCounter(ModifyCounter) { if (MModifyCounter) MCounter++; } @@ -55,33 +49,31 @@ template class ObjectUsageCounter { MCounter--; if (!MCounter) { - LockGuard Guard(MObjProtector); - if (MObj) - MObj->releaseResources(); + LockGuard Guard(GlobalHandler::MSyclGlobalHandlerProtector); + GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr(); + if (RTGlobalObjHandler) + RTGlobalObjHandler->releaseSchedulerResources(); } } private: static std::atomic_uint MCounter; bool MModifyCounter; - ResourceHandler *&MObj; - SpinLock &MObjProtector; }; -template -std::atomic_uint ObjectUsageCounter::MCounter{0}; +std::atomic_uint ObjectUsageCounter::MCounter{0}; GlobalHandler::GlobalHandler() = default; GlobalHandler::~GlobalHandler() = default; +GlobalHandler *&GlobalHandler::getInstancePtr() { + static GlobalHandler *RTGlobalObjHandler = new GlobalHandler(); + return RTGlobalObjHandler; +} + GlobalHandler &GlobalHandler::instance() { - static std::once_flag InitCreation; - std::call_once(InitCreation, []() { - const LockGuard Lock{MSyclGlobalHandlerProtector}; - MSyclGlobalObjectsHandler = new GlobalHandler(); - }); - assert(MSyclGlobalObjectsHandler && - "Handler must not be deallocated earlier"); - return *MSyclGlobalObjectsHandler; + GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr(); + assert(RTGlobalObjHandler && "Handler must not be deallocated earlier"); + return *RTGlobalObjHandler; } template @@ -109,8 +101,7 @@ Scheduler &GlobalHandler::getScheduler() { } void GlobalHandler::registerSchedulerUsage(bool ModifyCounter) { - thread_local ObjectUsageCounter SchedulerCounter( - MSyclGlobalObjectsHandler, MSyclGlobalHandlerProtector, ModifyCounter); + thread_local ObjectUsageCounter SchedulerCounter(ModifyCounter); } ProgramManager &GlobalHandler::getProgramManager() { @@ -216,14 +207,14 @@ void shutdown() { GlobalHandler *Handler = nullptr; { const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; - std::swap(Handler, GlobalHandler::MSyclGlobalObjectsHandler); + std::swap(Handler, GlobalHandler::getInstancePtr()); } if (!Handler) return; // Ensure neither host task is working so that no default context is accessed // upon its release - Handler->releaseResources(); + Handler->releaseSchedulerResources(); if (Handler->MHostTaskThreadPool.Inst) Handler->MHostTaskThreadPool.Inst->finishAndWait(); @@ -249,7 +240,7 @@ void shutdown() { delete Handler; } -void GlobalHandler::releaseResources() { +void GlobalHandler::releaseSchedulerResources() { drainThreadPool(); if (MScheduler.Inst) MScheduler.Inst->releaseResources(); diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index ee8fd66fd31e..905a49b36e3c 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -77,16 +77,17 @@ class GlobalHandler { void unloadPlugins(); void releaseDefaultContexts(); - void releaseResources(); + void releaseSchedulerResources(); void drainThreadPool(); // For testing purposes only void attachScheduler(Scheduler *Scheduler); private: - static GlobalHandler *MSyclGlobalObjectsHandler; - static SpinLock MSyclGlobalHandlerProtector; friend void shutdown(); + friend class ObjectUsageCounter; + static GlobalHandler *&getInstancePtr(); + static SpinLock MSyclGlobalHandlerProtector; // Constructor and destructor are declared out-of-line to allow incomplete // types as template arguments to unique_ptr. From 4c0225ca5d94237fac05d3cd262cf8fe1c99ecee Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Thu, 5 Jan 2023 05:05:43 -0800 Subject: [PATCH 09/10] remove releaseresources function in global handler Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 17 ++++++++--------- sycl/source/detail/global_handler.hpp | 2 -- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index ff18e7207ccf..e54d2d9f9caa 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -51,8 +51,11 @@ class ObjectUsageCounter { if (!MCounter) { LockGuard Guard(GlobalHandler::MSyclGlobalHandlerProtector); GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr(); - if (RTGlobalObjHandler) - RTGlobalObjHandler->releaseSchedulerResources(); + if (RTGlobalObjHandler) { + RTGlobalObjHandler->drainThreadPool(); + if (RTGlobalObjHandler->MScheduler.Inst) + RTGlobalObjHandler->MScheduler.Inst->releaseResources(); + } } } @@ -214,7 +217,9 @@ void shutdown() { // Ensure neither host task is working so that no default context is accessed // upon its release - Handler->releaseSchedulerResources(); + Handler->drainThreadPool(); + if (Handler->MScheduler.Inst) + Handler->MScheduler.Inst->releaseResources(); if (Handler->MHostTaskThreadPool.Inst) Handler->MHostTaskThreadPool.Inst->finishAndWait(); @@ -240,12 +245,6 @@ void shutdown() { delete Handler; } -void GlobalHandler::releaseSchedulerResources() { - drainThreadPool(); - if (MScheduler.Inst) - MScheduler.Inst->releaseResources(); -} - #ifdef _WIN32 extern "C" __SYCL_EXPORT BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index 905a49b36e3c..a9b29a15b939 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -76,8 +76,6 @@ class GlobalHandler { void unloadPlugins(); void releaseDefaultContexts(); - - void releaseSchedulerResources(); void drainThreadPool(); // For testing purposes only From 63513dabc2542644083571a9d22ba8669bc70e64 Mon Sep 17 00:00:00 2001 From: "Tikhomirova, Kseniya" Date: Thu, 5 Jan 2023 09:10:37 -0800 Subject: [PATCH 10/10] eliminate GlobalHandler == nullptr during shutdown Signed-off-by: Tikhomirova, Kseniya --- sycl/source/detail/global_handler.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index e54d2d9f9caa..1273ffc1add4 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -207,11 +207,8 @@ void GlobalHandler::drainThreadPool() { } void shutdown() { - GlobalHandler *Handler = nullptr; - { - const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; - std::swap(Handler, GlobalHandler::getInstancePtr()); - } + const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector}; + GlobalHandler *&Handler = GlobalHandler::getInstancePtr(); if (!Handler) return; @@ -243,6 +240,7 @@ void shutdown() { // Release the rest of global resources. delete Handler; + Handler = nullptr; } #ifdef _WIN32