Fixed x64 application termination after exceptions in a file dialog callback.

Since Windows 7 exceptions thrown in a file dialog callback (possibly by third-party utilities) aren't swallowed anymore. Make use of SetProcessUserModeExceptionPolicy (available in Windows 7 SP1) to temporarily restore the old behaviour and prevent the crashing (or summoning of the Program Compatibility Assistant) of an x64 application.

Closes #13674.

git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@69908 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

diff --git a/src/msw/filedlg.cpp b/src/msw/filedlg.cpp
index e45376548d4357b4846958cd97cbc814d045b1ad..4fa0e56e758e0400bcab72717b4d2bc2e0d3cfd6 100644 (file)
--- a/src/msw/filedlg.cpp
+++ b/src/msw/filedlg.cpp
@@ -43,7 +43,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "wx/dynlib.h"
 #include "wx/filename.h"
+#include "wx/scopeguard.h"
 #include "wx/tokenzr.h"
 
 // ----------------------------------------------------------------------------
@@ -74,6 +76,78 @@ static wxRect gs_rectDialog(0, 0, 428, 266);
 
 IMPLEMENT_CLASS(wxFileDialog, wxFileDialogBase)
 
+// ----------------------------------------------------------------------------
+
+namespace
+{
+
+#if wxUSE_DYNLIB_CLASS
+
+typedef BOOL (WINAPI *GetProcessUserModeExceptionPolicy_t)(LPDWORD);
+typedef BOOL (WINAPI *SetProcessUserModeExceptionPolicy_t)(DWORD);
+
+GetProcessUserModeExceptionPolicy_t gs_pfnGetProcessUserModeExceptionPolicy
+    = (GetProcessUserModeExceptionPolicy_t) -1;
+
+SetProcessUserModeExceptionPolicy_t gs_pfnSetProcessUserModeExceptionPolicy
+    = (SetProcessUserModeExceptionPolicy_t) -1;
+
+DWORD gs_oldExceptionPolicyFlags = 0;
+
+bool gs_changedPolicy = false;
+
+#endif // #if wxUSE_DYNLIB_CLASS
+
+/*
+Since Windows 7 by default (callback) exceptions aren't swallowed anymore
+with native x64 applications. Exceptions can occur in a file dialog when
+using the hook procedure in combination with third-party utilities.
+Since Windows 7 SP1 the swallowing of exceptions can be enabled again
+by using SetProcessUserModeExceptionPolicy.
+*/
+void ChangeExceptionPolicy()
+{
+#if wxUSE_DYNLIB_CLASS
+    gs_changedPolicy = false;
+
+    wxLoadedDLL dllKernel32(wxT("kernel32.dll"));
+
+    if ( gs_pfnGetProcessUserModeExceptionPolicy
+        == (GetProcessUserModeExceptionPolicy_t) -1)
+    {
+        wxDL_INIT_FUNC(gs_pfn, GetProcessUserModeExceptionPolicy, dllKernel32);
+        wxDL_INIT_FUNC(gs_pfn, SetProcessUserModeExceptionPolicy, dllKernel32);
+    }
+
+    if ( !gs_pfnGetProcessUserModeExceptionPolicy
+        || !gs_pfnSetProcessUserModeExceptionPolicy
+        || !gs_pfnGetProcessUserModeExceptionPolicy(&gs_oldExceptionPolicyFlags) )
+    {
+        return;
+    }
+
+    if ( gs_pfnSetProcessUserModeExceptionPolicy(gs_oldExceptionPolicyFlags
+        | 0x1 /* PROCESS_CALLBACK_FILTER_ENABLED */ ) )
+    {
+        gs_changedPolicy = true;
+    }
+
+#endif // wxUSE_DYNLIB_CLASS
+}
+
+void RestoreExceptionPolicy()
+{
+#if wxUSE_DYNLIB_CLASS
+    if (gs_changedPolicy)
+    {
+        gs_changedPolicy = false;
+        (void) gs_pfnSetProcessUserModeExceptionPolicy(gs_oldExceptionPolicyFlags);
+    }
+#endif // wxUSE_DYNLIB_CLASS
+}
+
+} // unnamed namespace
+
 // ----------------------------------------------------------------------------
 // hook function for moving the dialog
 // ----------------------------------------------------------------------------
@@ -400,12 +474,15 @@ int wxFileDialog::ShowModal()
     */
     if (m_bMovedWindow || HasExtraControlCreator()) // we need these flags.
     {
+        ChangeExceptionPolicy();
         msw_flags |= OFN_EXPLORER|OFN_ENABLEHOOK;
 #ifndef __WXWINCE__
         msw_flags |= OFN_ENABLESIZING;
 #endif
     }
 
+    wxON_BLOCK_EXIT0(RestoreExceptionPolicy);
+
     if ( HasFdFlag(wxFD_MULTIPLE) )
     {
         // OFN_EXPLORER must always be specified with OFN_ALLOWMULTISELECT