#5719 Partial detection of shutdown from task manager

indra/llcommon/llapp.h

@@ -285,6 +285,7 @@ class LL_COMMON_API LLApp
 
 #ifdef LL_WINDOWS
     virtual bool reportCrashToBugsplat(void* pExcepInfo /*EXCEPTION_POINTERS*/) { return false; }
+    virtual bool reportCustomToBugsplat(const std::string& description) { return false; }
 #endif
 
 public:

indra/llcommon/llwatchdog.cpp

@@ -173,17 +173,34 @@ void LLWatchdog::add(LLWatchdogEntry* e)
 {
     lockThread();
     mSuspects.insert(e);
+
+    if (!mFrozeList.empty())
+    {
+        mFrozeList.erase(e);
+        if (mFrozeList.empty())
+        {
+            // Clear error marker file if there is no frozen threads,
+            // viewer is responsive again.
+            mClearMarkerFnc();
+        }
+    }
     unlockThread();
 }
 
 void LLWatchdog::remove(LLWatchdogEntry* e)
 {
     lockThread();
     mSuspects.erase(e);
+    mFrozeList.erase(e);
     unlockThread();
 }
 
-void LLWatchdog::init(func_t set_error_state_callback)
+void LLWatchdog::init(
+    create_marker_func_t error_state_callback,
+    clear_marker_func_t clear_marker_callback,
+    report_func_t report_callback,
+    notify_func_t notify_callback,
+    bool crash_on_freeze)
 {
     if (!mSuspectsAccessMutex && !mTimer)
     {
@@ -196,7 +213,11 @@ void LLWatchdog::init(func_t set_error_state_callback)
         // start needs to use the mSuspectsAccessMutex
         mTimer->start();
     }
-    mCreateMarkerFnc = set_error_state_callback;
+    mCreateMarkerFnc = error_state_callback;
+    mClearMarkerFnc  = clear_marker_callback;
+    mCrashReportFnc = report_callback;
+    mNotifyFnc = notify_callback;
+    mCrashOnFreeze = crash_on_freeze;
 }
 
 void LLWatchdog::cleanup()
@@ -251,21 +272,47 @@ void LLWatchdog::run()
                 mTimer->stop();
             }
 
-            // Sets error marker file
-            mCreateMarkerFnc();
-            // Todo1: Warn user?
-            // Todo2: We probably want to report even if 5 seconds passed, just not error 'yet'.
             std::string last_state = (*result)->getLastState();
-            if (last_state.empty())
+            std::string description = "Watchdog timer for thread " + (*result)->getThreadName() + " expired";
+            if (!last_state.empty())
             {
-                LL_ERRS() << "Watchdog timer for thread " << (*result)->getThreadName()
-                    << " expired; assuming viewer is hung and crashing" << LL_ENDL;
+                description += " with state: " + last_state;
+            }
+            description += "; assuming viewer is hung and crashing";
+
+            if (!mCrashOnFreeze)
+            {
+                // Sets watchdog marker file
+                mCreateMarkerFnc(false);
+                // If it's mainloop and it somehow recovers, it will re-add itself
+                LLWatchdogEntry* froze_entry = *result;
+                mSuspects.erase(result);
+                mFrozeList.insert(froze_entry);
+                LL_WARNS() << description << LL_ENDL;
             }
             else
             {
-                LL_ERRS() << "Watchdog timer for thread " << (*result)->getThreadName()
-                    << " expired with state: " << last_state
-                    << "; assuming viewer is hung and crashing" << LL_ENDL;
+
+                if (!mCrashReportFnc(description))
+                {
+                    // Sets error marker file
+                    mCreateMarkerFnc(true);
+                    // If false is returned, then we failed to report the issue to bugsplat,
+                    // instead, Notify user, then crash viewer.
+                    // Todo: ask user if viewer should quit or wait?
+                    mNotifyFnc();
+                    LL_ERRS() << description << LL_ENDL;
+                }
+                else
+                {
+                    // Sets watchdog marker file
+                    mCreateMarkerFnc(false);
+                    // Already reported, don't report again.
+                    // If it's mainloop and it somehow recovers, it will re-add itself
+                    LLWatchdogEntry* froze_entry = *result;
+                    mSuspects.erase(result);
+                    mFrozeList.insert(froze_entry);
+                }
             }
         }
     }

indra/llcommon/llwatchdog.h

@@ -93,8 +93,16 @@ class LLWatchdog : public LLSimpleton<LLWatchdog>
     void add(LLWatchdogEntry* e);
     void remove(LLWatchdogEntry* e);
 
-    typedef std::function<void()> func_t;
-    void init(func_t set_error_state_callback);
+    typedef std::function<void(bool)> create_marker_func_t;
+    typedef std::function<void()> clear_marker_func_t;
+    typedef std::function<bool(std::string&)> report_func_t;
+    typedef std::function<void()> notify_func_t;
+    void init(
+        create_marker_func_t error_state_callback,
+        clear_marker_func_t clear_marker_callback,
+        report_func_t report_callback,
+        notify_func_t notify_callback,
+        bool crash_on_freeze);
     void run();
     void cleanup();
 
@@ -105,14 +113,19 @@ class LLWatchdog : public LLSimpleton<LLWatchdog>
 
     typedef std::set<LLWatchdogEntry*> SuspectsRegistry;
     SuspectsRegistry mSuspects;
+    SuspectsRegistry mFrozeList;
     LLMutex* mSuspectsAccessMutex;
     LLWatchdogTimerThread* mTimer;
     U64 mLastClockCount;
+    bool mCrashOnFreeze;
 
     // At the moment watchdog expects app to set markers in mCreateMarkerFnc,
     // but technically can be used to set any error states or do some cleanup
     // or show warnings.
-    func_t mCreateMarkerFnc;
+    create_marker_func_t mCreateMarkerFnc;
+    clear_marker_func_t mClearMarkerFnc;
+    report_func_t mCrashReportFnc;
+    notify_func_t mNotifyFnc;
 };
 
 #endif // LL_LLTHREADWATCHDOG_H

indra/llwindow/llwindowcallbacks.cpp

@@ -68,6 +68,10 @@ void LLWindowCallbacks::handleMouseLeave(LLWindow *window)
     return;
 }
 
+void LLWindowCallbacks::handlePreCloseRequest()
+{
+}
+
 bool LLWindowCallbacks::handleCloseRequest(LLWindow *window, bool from_user)
 {
     //allow the window to close

indra/llwindow/llwindowcallbacks.h

@@ -41,6 +41,8 @@ class LLWindowCallbacks
     virtual bool handleMouseDown(LLWindow *window,  LLCoordGL pos, MASK mask);
     virtual bool handleMouseUp(LLWindow *window,  LLCoordGL pos, MASK mask);
     virtual void handleMouseLeave(LLWindow *window);
+    // Called before close request is processed (ex: to create marker file in case OS is about to kill app).
+    virtual void handlePreCloseRequest();
     // return true to allow window to close, which will then cause handleQuit to be called
     virtual bool handleCloseRequest(LLWindow *window, bool from_user);
     virtual bool handleSessionExit(LLWindow* window);

indra/llwindow/llwindowwin32.cpp

@@ -508,6 +508,7 @@ LLWindowWin32::LLWindowWin32(LLWindowCallbacks* callbacks,
     :
     LLWindow(callbacks, fullscreen, flags),
     mAbsoluteCursorPosition(false),
+    mReceivedSCClose(false),
     mMaxGLVersion(max_gl_version),
     mMaxCores(max_cores)
 {
@@ -2524,8 +2525,15 @@ LRESULT CALLBACK LLWindowWin32::mainWindowProc(HWND h_wnd, UINT u_msg, WPARAM w_
         case WM_SYSCOMMAND:
         {
             LL_PROFILE_ZONE_NAMED_CATEGORY_WIN32("mwp - WM_SYSCOMMAND");
-            switch (w_param)
+            switch (w_param & 0xFFF0)
             {
+            case SC_CLOSE:
+                // User clicked close from system menu/taskbar or 'end process' from task manager
+                // Do nothing, will cause WM_CLOSE.
+                // If we don't get this message before WM_CLOSE, we are likely getting
+                // a kill from some external program. Win11 task manager Does cause SC_CLOSE.
+                window_imp->mReceivedSCClose = true;
+                break;
             case SC_KEYMENU:
                 // Disallow the ALT key from triggering the default system menu.
                 return 0;
@@ -2540,9 +2548,30 @@ LRESULT CALLBACK LLWindowWin32::mainWindowProc(HWND h_wnd, UINT u_msg, WPARAM w_
         case WM_CLOSE:
         {
             LL_PROFILE_ZONE_NAMED_CATEGORY_WIN32("mwp - WM_CLOSE");
-            // todo: WM_CLOSE can be caused by user and by task manager,
-            // distinguish these cases.
-            // For now assume it is always user.
+
+            window_imp->mCallbacks->handlePreCloseRequest(); // mark app as potentially closing
+            if (!window_imp->mReceivedSCClose)
+            {
+                // Some external program is trying to close the app.
+                // Assume that it's going to destroy process if it fails
+                // and try to fast-quit without confirmation or cleanup.
+                window_imp->post([=]()
+                {
+                    // Check if app needs cleanup or can be closed immediately.
+                    if (window_imp->mCallbacks->handleSessionExit(window_imp))
+                    {
+                        // Get the app to initiate cleanup.
+                        window_imp->mCallbacks->handleQuit(window_imp);
+                    }
+                });
+                return 0;
+            }
+            window_imp->mReceivedSCClose = false;
+
+            // There is no way to tell the difference between a user issued
+            // WM_CLOSE or task manager's WM_CLOSE.
+            // Assume it is a user and ask for confirmation, but create a marker file.
+            // If App keeps doing something after a second, or gets 'destroy' message clear the marker.
             window_imp->post([=]()
                 {
                     // Will the app allow the window to close?
@@ -2564,6 +2593,16 @@ LRESULT CALLBACK LLWindowWin32::mainWindowProc(HWND h_wnd, UINT u_msg, WPARAM w_
             }
             return 0;
         }
+        case WM_NCDESTROY:
+            LL_INFOS("Window") << "Received WM_NCDESTROY" << LL_ENDL;
+            break;
+        case WM_WTSSESSION_CHANGE:
+        {
+            // Detects Remote Desktop disconnects, fast user switching, session logoff
+            // w_param: WTS_CONSOLE_CONNECT, WTS_CONSOLE_DISCONNECT, WTS_SESSION_LOGOFF, etc.
+            LL_INFOS("Window") << "Received WM_WTSSESSION_CHANGE with wParam: " << (U32)w_param << LL_ENDL;
+            break;
+        }
         case WM_QUERYENDSESSION:
         {
             // Generally means that OS is going to shut down or user is going to log off.
@@ -2585,6 +2624,7 @@ LRESULT CALLBACK LLWindowWin32::mainWindowProc(HWND h_wnd, UINT u_msg, WPARAM w_
                 || (end_session_flags & ENDSESSION_CRITICAL) // will shutdown regardless of app state
                 || (end_session_flags & ENDSESSION_LOGOFF)) // logoff, can delay shutdown
             {
+                window_imp->mCallbacks->handlePreCloseRequest(); // mark app as closing
                 window_imp->post([=]()
                 {
                     LL_INFOS("Window") << "Shutting down due to session terminating" << LL_ENDL;
@@ -3318,7 +3358,18 @@ LRESULT CALLBACK LLWindowWin32::mainWindowProc(HWND h_wnd, UINT u_msg, WPARAM w_
 
         case WM_DISPLAYCHANGE:
         {
-            WINDOW_IMP_POST(window_imp->mCallbacks->handleDisplayChanged());
+            LL_PROFILE_ZONE_NAMED_CATEGORY_WIN32("mwp - WM_DISPLAYCHANGE");
+            window_imp->post([=]() {
+                window_imp->mCallbacks->handleDisplayChanged();
+                // Note: WM_DISPLAYCHANGE was passing to WM_SETFOCUS
+                // which might have been unintended and was messing with zones.
+                // handleFocus was copied over and return 0 added, but
+                // handleFocus might be not needed here.
+                // handleFocus resets mouse, closes popups and keys, which
+                // we probablt should do on 'display change'.
+                window_imp->mCallbacks->handleFocus(window_imp);
+            });
+            return 0;
         }
 
         case WM_SETFOCUS:

indra/llwindow/llwindowwin32.h

@@ -238,6 +238,7 @@ class LLWindowWin32 : public LLWindow
     LPWSTR      mIconResource;
     LPWSTR      mIconSmallResource;
     bool        mInputProcessingPaused;
+    bool        mReceivedSCClose; // received SC_CLOSE and expecting WM_CLOSE
 
     // The following variables are for Language Text Input control.
     // They are all static, since one context is shared by all LLWindowWin32