tests/test.cpp

   1 ///////////////////////////////////////////////////////////////////////////////
   2 // Name:        test.cpp
   3 // Purpose:     Test program for wxWidgets
   4 // Author:      Mike Wetherell
   5 // RCS-ID:      $Id$
   6 // Copyright:   (c) 2004 Mike Wetherell
   7 // Licence:     wxWindows licence
   8 ///////////////////////////////////////////////////////////////////////////////
   9
  10 // ----------------------------------------------------------------------------
  11 // headers
  12 // ----------------------------------------------------------------------------
  13
  14 // For compilers that support precompilation, includes "wx/wx.h"
  15 // and "wx/cppunit.h"
  16 #include "testprec.h"
  17
  18 #ifdef __BORLANDC__
  19     #pragma hdrstop
  20 #endif
  21
  22 // for all others, include the necessary headers
  23 #ifndef WX_PRECOMP
  24     #include "wx/wx.h"
  25 #endif
  26
  27 #include "wx/beforestd.h"
  28 #ifdef __VISUALC__
  29     #pragma warning(disable:4100)
  30 #endif
  31
  32 #include <cppunit/TestListener.h>
  33 #include <cppunit/Protector.h>
  34 #include <cppunit/Test.h>
  35 #include <cppunit/TestResult.h>
  36 #include <cppunit/TestFailure.h>
  37 #include <cppunit/TestResultCollector.h>
  38
  39 #ifdef __VISUALC__
  40     #pragma warning(default:4100)
  41 #endif
  42 #include "wx/afterstd.h"
  43
  44 #include "wx/cmdline.h"
  45 #include <exception>
  46 #include <iostream>
  47
  48 #ifdef __WINDOWS__
  49     #include "wx/msw/msvcrt.h"
  50 #endif
  51
  52 #ifdef __WXOSX__
  53     #include "wx/osx/private.h"
  54 #endif
  55
  56 #if wxUSE_GUI
  57     #include "testableframe.h"
  58 #endif
  59
  60 #include "wx/socket.h"
  61 #include "wx/evtloop.h"
  62
  63 using namespace std;
  64
  65 using CppUnit::Test;
  66 using CppUnit::TestSuite;
  67 using CppUnit::TestFactoryRegistry;
  68
  69
  70 // ----------------------------------------------------------------------------
  71 // helper classes
  72 // ----------------------------------------------------------------------------
  73
  74 // exception class for MSVC debug CRT assertion failures
  75 #ifdef wxUSE_VC_CRTDBG
  76
  77 struct CrtAssertFailure
  78 {
  79     CrtAssertFailure(const char *message) : m_msg(message) { }
  80
  81     const wxString m_msg;
  82
  83     wxDECLARE_NO_ASSIGN_CLASS(CrtAssertFailure);
  84 };
  85
  86 #endif // wxUSE_VC_CRTDBG
  87
  88 #if wxDEBUG_LEVEL
  89
  90 // Information about the last not yet handled assertion.
  91 static wxString s_lastAssertMessage;
  92
  93 static wxString FormatAssertMessage(const wxString& file,
  94                                     int line,
  95                                     const wxString& func,
  96                                     const wxString& cond,
  97                                     const wxString& msg)
  98 {
  99     wxString str;
 100     str << "wxWidgets assert: " << cond << " failed "
 101            "at " << file << ":" << line << " in " << func
 102         << " with message '" << msg << "'";
 103     return str;
 104 }
 105
 106 static void TestAssertHandler(const wxString& file,
 107                               int line,
 108                               const wxString& func,
 109                               const wxString& cond,
 110                               const wxString& msg)
 111 {
 112     // Determine whether we can safely throw an exception to just make the test
 113     // fail or whether we need to abort (in this case "msg" will contain the
 114     // explanation why did we decide to do it).
 115     wxString abortReason;
 116
 117     const wxString
 118         assertMessage = FormatAssertMessage(file, line, func, cond, msg);
 119
 120     if ( !wxIsMainThread() )
 121     {
 122         // Exceptions thrown from worker threads are not caught currently and
 123         // so we'd just die without any useful information -- abort instead.
 124         abortReason << assertMessage << "in a worker thread.";
 125     }
 126     else if ( uncaught_exception() )
 127     {
 128         // Throwing while already handling an exception would result in
 129         // terminate() being called and we wouldn't get any useful information
 130         // about why the test failed then.
 131         if ( s_lastAssertMessage.empty() )
 132         {
 133             abortReason << assertMessage << "while handling an exception";
 134         }
 135         else // In this case the exception is due to a previous assert.
 136         {
 137             abortReason << s_lastAssertMessage << "\n  and another "
 138                         << assertMessage << " while handling it.";
 139         }
 140     }
 141     else // Can "safely" throw from here.
 142     {
 143         // Remember this in case another assert happens while handling this
 144         // exception: we want to show the original assert as it's usually more
 145         // useful to determine the real root of the problem.
 146         s_lastAssertMessage = assertMessage;
 147
 148         throw TestAssertFailure(file, line, func, cond, msg);
 149     }
 150
 151     wxFputs(abortReason, stderr);
 152     fflush(stderr);
 153     _exit(-1);
 154 }
 155
 156 #endif // wxDEBUG_LEVEL
 157
 158 // this function should only be called from a catch clause
 159 static string GetExceptionMessage()
 160 {
 161     wxString msg;
 162
 163     try
 164     {
 165         throw;
 166     }
 167 #if wxDEBUG_LEVEL
 168     catch ( TestAssertFailure& )
 169     {
 170         msg = s_lastAssertMessage;
 171         s_lastAssertMessage.clear();
 172     }
 173 #endif // wxDEBUG_LEVEL
 174 #ifdef wxUSE_VC_CRTDBG
 175     catch ( CrtAssertFailure& e )
 176     {
 177         msg << "CRT assert failure: " << e.m_msg;
 178     }
 179 #endif // wxUSE_VC_CRTDBG
 180     catch ( std::exception& e )
 181     {
 182         msg << "std::exception: " << e.what();
 183     }
 184     catch ( ... )
 185     {
 186         msg = "Unknown exception caught.";
 187     }
 188
 189     return string(msg.mb_str());
 190 }
 191
 192 // Protector adding handling of wx-specific (this includes MSVC debug CRT in
 193 // this context) exceptions
 194 class wxUnitTestProtector : public CppUnit::Protector
 195 {
 196 public:
 197     virtual bool protect(const CppUnit::Functor &functor,
 198                          const CppUnit::ProtectorContext& context)
 199     {
 200         try
 201         {
 202             return functor();
 203         }
 204         catch ( std::exception& )
 205         {
 206             // cppunit deals with the standard exceptions itself, let it do as
 207             // it output more details (especially for std::exception-derived
 208             // CppUnit::Exception) than we do
 209             throw;
 210         }
 211         catch ( ... )
 212         {
 213             reportError(context, CppUnit::Message("Uncaught exception",
 214                                                   GetExceptionMessage()));
 215         }
 216
 217         return false;
 218     }
 219 };
 220
 221 // Displays the test name before starting to execute it: this helps with
 222 // diagnosing where exactly does a test crash or hang when/if it does.
 223 class DetailListener : public CppUnit::TestListener
 224 {
 225 public:
 226     DetailListener(bool doTiming = false):
 227         CppUnit::TestListener(),
 228         m_timing(doTiming)
 229     {
 230     }
 231
 232     virtual void startTest(CppUnit::Test *test)
 233     {
 234         printf("  %-60s  ", test->getName().c_str());
 235         m_result = RESULT_OK;
 236         m_watch.Start();
 237     }
 238
 239     virtual void addFailure(const CppUnit::TestFailure& failure)
 240     {
 241         m_result = failure.isError() ? RESULT_ERROR : RESULT_FAIL;
 242     }
 243
 244     virtual void endTest(CppUnit::Test * WXUNUSED(test))
 245     {
 246         m_watch.Pause();
 247         printf("%s", GetResultStr(m_result));
 248         if (m_timing)
 249             printf("  %6ld ms", m_watch.Time());
 250         printf("\n");
 251     }
 252
 253 protected :
 254     enum ResultType
 255     {
 256         RESULT_OK = 0,
 257         RESULT_FAIL,
 258         RESULT_ERROR,
 259         RESULT_MAX
 260     };
 261
 262     const char* GetResultStr(ResultType type) const
 263     {
 264         static const char *resultTypeNames[] =
 265         {
 266             "  OK",
 267             "FAIL",
 268             " ERR"
 269         };
 270
 271         wxCOMPILE_TIME_ASSERT( WXSIZEOF(resultTypeNames) == RESULT_MAX,
 272                                ResultTypeNamesMismatch );
 273
 274         return resultTypeNames[type];
 275     }
 276
 277     bool m_timing;
 278     wxStopWatch m_watch;
 279     ResultType m_result;
 280 };
 281
 282 #if wxUSE_GUI
 283     typedef wxApp TestAppBase;
 284 #else
 285     typedef wxAppConsole TestAppBase;
 286 #endif
 287
 288 // The application class
 289 //
 290 class TestApp : public TestAppBase
 291 {
 292 public:
 293     TestApp();
 294
 295     // standard overrides
 296     virtual void OnInitCmdLine(wxCmdLineParser& parser);
 297     virtual bool OnCmdLineParsed(wxCmdLineParser& parser);
 298     virtual bool OnInit();
 299     virtual int  OnExit();
 300
 301     // used by events propagation test
 302     virtual int FilterEvent(wxEvent& event);
 303     virtual bool ProcessEvent(wxEvent& event);
 304
 305     void SetFilterEventFunc(FilterEventFunc f) { m_filterEventFunc = f; }
 306     void SetProcessEventFunc(ProcessEventFunc f) { m_processEventFunc = f; }
 307
 308     // In console applications we run the tests directly from the overridden
 309     // OnRun(), but in the GUI ones we run them when we get the first call to
 310     // our EVT_IDLE handler to ensure that we do everything from inside the
 311     // main event loop. This is especially important under wxOSX/Cocoa where
 312     // the main event loop is different from the others but it's also safer to
 313     // do it like this in the other ports as we test the GUI code in the same
 314     // context as it's used usually, in normal programs, and it might behave
 315     // differently without the event loop.
 316 #if wxUSE_GUI
 317     void OnIdle(wxIdleEvent& event)
 318     {
 319         if ( m_runTests )
 320         {
 321             m_runTests = false;
 322
 323 #ifdef __WXOSX__
 324             // we need to wait until the window is activated and fully ready
 325             // otherwise no events can be posted
 326             wxEventLoopBase* const loop = wxEventLoop::GetActive();
 327             if ( loop )
 328             {
 329                 loop->DispatchTimeout(1000);
 330                 loop->Yield();
 331             }
 332 #endif // __WXOSX__
 333
 334             m_exitcode = RunTests();
 335             ExitMainLoop();
 336         }
 337
 338         event.Skip();
 339     }
 340 #else // !wxUSE_GUI
 341     virtual int OnRun()
 342     {
 343         m_exitcode = RunTests();
 344         return m_exitcode;
 345     }
 346 #endif // wxUSE_GUI/!wxUSE_GUI
 347
 348 private:
 349     void List(Test *test, const string& parent = "") const;
 350
 351     // call List() if m_list or runner.addTest() otherwise
 352     void AddTest(CppUnit::TestRunner& runner, Test *test)
 353     {
 354         if (m_list)
 355             List(test);
 356         else
 357             runner.addTest(test);
 358     }
 359
 360     int RunTests();
 361
 362     // flag telling us whether we should run tests from our EVT_IDLE handler
 363     bool m_runTests;
 364
 365     // command lines options/parameters
 366     bool m_list;
 367     bool m_longlist;
 368     bool m_detail;
 369     bool m_timing;
 370     wxArrayString m_registries;
 371     wxLocale *m_locale;
 372
 373     // event handling hooks
 374     FilterEventFunc m_filterEventFunc;
 375     ProcessEventFunc m_processEventFunc;
 376
 377     // the program exit code
 378     int m_exitcode;
 379 };
 380
 381 IMPLEMENT_APP_NO_MAIN(TestApp)
 382
 383
 384 // ----------------------------------------------------------------------------
 385 // global functions
 386 // ----------------------------------------------------------------------------
 387
 388 #ifdef wxUSE_VC_CRTDBG
 389
 390 static int TestCrtReportHook(int reportType, char *message, int *)
 391 {
 392     if ( reportType != _CRT_ASSERT )
 393         return FALSE;
 394
 395     throw CrtAssertFailure(message);
 396 }
 397
 398 #endif // wxUSE_VC_CRTDBG
 399
 400 int main(int argc, char **argv)
 401 {
 402     // tests can be ran non-interactively so make sure we don't show any assert
 403     // dialog boxes -- neither our own nor from MSVC debug CRT -- which would
 404     // prevent them from completing
 405
 406 #if wxDEBUG_LEVEL
 407     wxSetAssertHandler(TestAssertHandler);
 408 #endif // wxDEBUG_LEVEL
 409
 410 #ifdef wxUSE_VC_CRTDBG
 411     _CrtSetReportHook(TestCrtReportHook);
 412 #endif // wxUSE_VC_CRTDBG
 413
 414     try
 415     {
 416         return wxEntry(argc, argv);
 417     }
 418     catch ( ... )
 419     {
 420         cerr << "\n" << GetExceptionMessage() << endl;
 421     }
 422
 423     return -1;
 424 }
 425
 426 extern void SetFilterEventFunc(FilterEventFunc func)
 427 {
 428     wxGetApp().SetFilterEventFunc(func);
 429 }
 430
 431 extern void SetProcessEventFunc(ProcessEventFunc func)
 432 {
 433     wxGetApp().SetProcessEventFunc(func);
 434 }
 435
 436 extern bool IsNetworkAvailable()
 437 {
 438     // NOTE: we could use wxDialUpManager here if it was in wxNet; since it's in
 439     //       wxCore we use a simple rough test:
 440
 441     wxSocketBase::Initialize();
 442
 443     wxIPV4address addr;
 444     if (!addr.Hostname("www.google.com") || !addr.Service("www"))
 445     {
 446         wxSocketBase::Shutdown();
 447         return false;
 448     }
 449
 450     wxSocketClient sock;
 451     sock.SetTimeout(10);    // 10 secs
 452     bool online = sock.Connect(addr);
 453
 454     wxSocketBase::Shutdown();
 455
 456     return online;
 457 }
 458
 459 extern bool IsAutomaticTest()
 460 {
 461     static int s_isAutomatic = -1;
 462     if ( s_isAutomatic == -1 )
 463     {
 464         // Allow setting an environment variable to emulate buildslave user for
 465         // testing.
 466         wxString username;
 467         if ( !wxGetEnv("WX_TEST_USER", &username) )
 468             username = wxGetUserId();
 469
 470         username.MakeLower();
 471         s_isAutomatic = username.Matches("buildslave*") ||
 472                             username.Matches("sandbox*");
 473     }
 474
 475     return s_isAutomatic == 1;
 476 }
 477
 478 // helper of RunTests(): gets the test with the given name, returning NULL (and
 479 // not an empty test suite) if there is no such test
 480 static Test *GetTestByName(const wxString& name)
 481 {
 482     Test *
 483       test = TestFactoryRegistry::getRegistry(string(name.mb_str())).makeTest();
 484     if ( test )
 485     {
 486         TestSuite * const suite = dynamic_cast<TestSuite *>(test);
 487         if ( !suite || !suite->countTestCases() )
 488         {
 489             // it's a bogus test, don't use it
 490             delete test;
 491             test = NULL;
 492         }
 493     }
 494
 495     return test;
 496 }
 497
 498
 499 // ----------------------------------------------------------------------------
 500 // TestApp
 501 // ----------------------------------------------------------------------------
 502
 503 TestApp::TestApp()
 504   : m_list(false),
 505     m_longlist(false)
 506 {
 507     m_runTests = true;
 508
 509     m_filterEventFunc = NULL;
 510     m_processEventFunc = NULL;
 511
 512     m_locale = NULL;
 513
 514     m_exitcode = EXIT_SUCCESS;
 515 }
 516
 517 // Init
 518 //
 519 bool TestApp::OnInit()
 520 {
 521     if ( !TestAppBase::OnInit() )
 522         return false;
 523
 524 #if wxUSE_GUI
 525     cout << "Test program for wxWidgets GUI features\n"
 526 #else
 527     cout << "Test program for wxWidgets non-GUI features\n"
 528 #endif
 529          << "build: " << WX_BUILD_OPTIONS_SIGNATURE << "\n"
 530          << "running under " << wxGetOsDescription()
 531          << " as " << wxGetUserId() << std::endl;
 532
 533     if ( m_detail )
 534     {
 535         // Output some important information about the test environment.
 536         cout << "Running under " << wxGetOsDescription() << ", "
 537                 "locale is " << setlocale(LC_ALL, NULL) << std::endl;
 538     }
 539
 540 #if wxUSE_GUI
 541     // create a hidden parent window to be used as parent for the GUI controls
 542     wxTestableFrame* frame = new wxTestableFrame();
 543     frame->Show();
 544
 545     Connect(wxEVT_IDLE, wxIdleEventHandler(TestApp::OnIdle));
 546 #endif // wxUSE_GUI
 547
 548     return true;
 549 }
 550
 551 // The table of command line options
 552 //
 553 void TestApp::OnInitCmdLine(wxCmdLineParser& parser)
 554 {
 555     TestAppBase::OnInitCmdLine(parser);
 556
 557     static const wxCmdLineEntryDesc cmdLineDesc[] = {
 558         { wxCMD_LINE_SWITCH, "l", "list",
 559             "list the test suites, do not run them",
 560             wxCMD_LINE_VAL_NONE, 0 },
 561         { wxCMD_LINE_SWITCH, "L", "longlist",
 562             "list the test cases, do not run them",
 563             wxCMD_LINE_VAL_NONE, 0 },
 564         { wxCMD_LINE_SWITCH, "d", "detail",
 565             "print the test case names, run them",
 566             wxCMD_LINE_VAL_NONE, 0 },
 567         { wxCMD_LINE_SWITCH, "t", "timing",
 568             "print names and measure running time of individual test, run them",
 569             wxCMD_LINE_VAL_NONE, 0 },
 570         { wxCMD_LINE_OPTION, "", "locale",
 571             "locale to use when running the program",
 572             wxCMD_LINE_VAL_STRING, 0 },
 573         { wxCMD_LINE_PARAM, NULL, NULL, "REGISTRY", wxCMD_LINE_VAL_STRING,
 574             wxCMD_LINE_PARAM_OPTIONAL | wxCMD_LINE_PARAM_MULTIPLE },
 575         wxCMD_LINE_DESC_END
 576     };
 577
 578     parser.SetDesc(cmdLineDesc);
 579 }
 580
 581 // Handle command line options
 582 //
 583 bool TestApp::OnCmdLineParsed(wxCmdLineParser& parser)
 584 {
 585     if (parser.GetParamCount())
 586     {
 587         for (size_t i = 0; i < parser.GetParamCount(); i++)
 588             m_registries.push_back(parser.GetParam(i));
 589     }
 590
 591     m_longlist = parser.Found("longlist");
 592     m_list = m_longlist || parser.Found("list");
 593     m_timing = parser.Found("timing");
 594     m_detail = !m_timing && parser.Found("detail");
 595
 596     wxString loc;
 597     if ( parser.Found("locale", &loc) )
 598     {
 599         const wxLanguageInfo * const info = wxLocale::FindLanguageInfo(loc);
 600         if ( !info )
 601         {
 602             cerr << "Locale \"" << string(loc.mb_str()) << "\" is unknown.\n";
 603             return false;
 604         }
 605
 606         m_locale = new wxLocale(info->Language);
 607         if ( !m_locale->IsOk() )
 608         {
 609             cerr << "Using locale \"" << string(loc.mb_str()) << "\" failed.\n";
 610             return false;
 611         }
 612     }
 613
 614     return TestAppBase::OnCmdLineParsed(parser);
 615 }
 616
 617 // Event handling
 618 int TestApp::FilterEvent(wxEvent& event)
 619 {
 620     if ( m_filterEventFunc )
 621         return (*m_filterEventFunc)(event);
 622
 623     return TestAppBase::FilterEvent(event);
 624 }
 625
 626 bool TestApp::ProcessEvent(wxEvent& event)
 627 {
 628     if ( m_processEventFunc )
 629         return (*m_processEventFunc)(event);
 630
 631     return TestAppBase::ProcessEvent(event);
 632 }
 633
 634 // Run
 635 //
 636 int TestApp::RunTests()
 637 {
 638 #if wxUSE_LOG
 639     // Switch off logging unless --verbose
 640     bool verbose = wxLog::GetVerbose();
 641     wxLog::EnableLogging(verbose);
 642 #else
 643     bool verbose = false;
 644 #endif
 645
 646     CppUnit::TextTestRunner runner;
 647
 648     if ( m_registries.empty() )
 649     {
 650         // run or list all tests which use the CPPUNIT_TEST_SUITE_REGISTRATION() macro
 651         // (i.e. those registered in the "All tests" registry); if there are other
 652         // tests not registered with the CPPUNIT_TEST_SUITE_REGISTRATION() macro
 653         // then they won't be listed/run!
 654         AddTest(runner, TestFactoryRegistry::getRegistry().makeTest());
 655
 656         if (m_list)
 657         {
 658             cout << "\nNote that the list above is not complete as it doesn't include the \n";
 659             cout << "tests disabled by default.\n";
 660         }
 661     }
 662     else // run only the selected tests
 663     {
 664         for (size_t i = 0; i < m_registries.size(); i++)
 665         {
 666             const wxString reg = m_registries[i];
 667             Test *test = GetTestByName(reg);
 668
 669             if ( !test && !reg.EndsWith("TestCase") )
 670             {
 671                 test = GetTestByName(reg + "TestCase");
 672             }
 673
 674             if ( !test )
 675             {
 676                 cerr << "No such test suite: " << string(reg.mb_str()) << endl;
 677                 return 2;
 678             }
 679
 680             AddTest(runner, test);
 681         }
 682     }
 683
 684     if ( m_list )
 685         return EXIT_SUCCESS;
 686
 687     runner.setOutputter(new CppUnit::CompilerOutputter(&runner.result(), cout));
 688
 689     // there is a bug
 690     // (http://sf.net/tracker/index.php?func=detail&aid=1649369&group_id=11795&atid=111795)
 691     // in some versions of cppunit: they write progress dots to cout (and not
 692     // cerr) and don't flush it so all the dots appear at once at the end which
 693     // is not very useful so unbuffer cout to work around this
 694     cout.setf(ios::unitbuf);
 695
 696     // add detail listener if needed
 697     DetailListener detailListener(m_timing);
 698     if ( m_detail || m_timing )
 699         runner.eventManager().addListener(&detailListener);
 700
 701     // finally ensure that we report our own exceptions nicely instead of
 702     // giving "uncaught exception of unknown type" messages
 703     runner.eventManager().pushProtector(new wxUnitTestProtector);
 704
 705     bool printProgress = !(verbose || m_detail || m_timing);
 706     runner.run("", false, true, printProgress);
 707
 708     return runner.result().testFailures() == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
 709 }
 710
 711 int TestApp::OnExit()
 712 {
 713     delete m_locale;
 714
 715 #if wxUSE_GUI
 716     delete GetTopWindow();
 717 #endif // wxUSE_GUI
 718
 719     return m_exitcode;
 720 }
 721
 722 // List the tests
 723 //
 724 void TestApp::List(Test *test, const string& parent /*=""*/) const
 725 {
 726     TestSuite *suite = dynamic_cast<TestSuite*>(test);
 727     string name;
 728
 729     if (suite) {
 730         // take the last component of the name and append to the parent
 731         name = test->getName();
 732         string::size_type i = name.find_last_of(".:");
 733         if (i != string::npos)
 734             name = name.substr(i + 1);
 735         name = parent + "." + name;
 736
 737         // drop the 1st component from the display and indent
 738         if (parent != "") {
 739             string::size_type j = i = name.find('.', 1);
 740             while ((j = name.find('.', j + 1)) != string::npos)
 741                 cout << "  ";
 742             cout << "  " << name.substr(i + 1) << "\n";
 743         }
 744
 745         typedef vector<Test*> Tests;
 746         typedef Tests::const_iterator Iter;
 747
 748         const Tests& tests = suite->getTests();
 749
 750         for (Iter it = tests.begin(); it != tests.end(); ++it)
 751             List(*it, name);
 752     }
 753     else if (m_longlist) {
 754         string::size_type i = 0;
 755         while ((i = parent.find('.', i + 1)) != string::npos)
 756             cout << "  ";
 757         cout << "  " << test->getName() << "\n";
 758     }
 759 }