X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/729e4ab9bc6618bc3d8a898e575df7f4019e29ca..d25163bfc042dbef00577180ee21dd3460fc3715:/icuSources/test/intltest/tsmthred.cpp?ds=sidebyside diff --git a/icuSources/test/intltest/tsmthred.cpp b/icuSources/test/intltest/tsmthred.cpp index 653cb163..c9df4373 100644 --- a/icuSources/test/intltest/tsmthred.cpp +++ b/icuSources/test/intltest/tsmthred.cpp @@ -1,6 +1,6 @@ /******************************************************************** - * COPYRIGHT: - * Copyright (c) 1999-2010, International Business Machines Corporation and + * COPYRIGHT: + * Copyright (c) 1999-2014, International Business Machines Corporation and * others. All Rights Reserved. ********************************************************************/ @@ -24,22 +24,33 @@ #include "unicode/uloc.h" #include "unicode/locid.h" #include "putilimp.h" -#if !defined(U_WINDOWS) && !defined(XP_MAC) && !defined(U_RHAPSODY) -#define POSIX 1 +#include "intltest.h" +#include "tsmthred.h" +#include "unicode/ushape.h" +#include "unicode/translit.h" +#include "sharedobject.h" +#include "unifiedcache.h" +#include "uassert.h" + +#if U_PLATFORM_USES_ONLY_WIN32_API + /* Prefer native Windows APIs even if POSIX is implemented (i.e., on Cygwin). */ +# undef POSIX +#elif U_PLATFORM_IMPLEMENTS_POSIX +# define POSIX +#else +# undef POSIX #endif /* Needed by z/OS to get usleep */ -#if defined(OS390) +#if U_PLATFORM == U_PF_OS390 #define __DOT1 1 #define __UU -#define _XOPEN_SOURCE_EXTENDED 1 #ifndef _XPG4_2 #define _XPG4_2 #endif #include -/*#include "platform_xopen_source_extended.h"*/ #endif -#if defined(POSIX) || defined(U_SOLARIS) || defined(U_AIX) || defined(U_HPUX) +#if defined(POSIX) #define HAVE_IMP @@ -58,11 +69,11 @@ #define __EXTENSIONS__ #endif -#if defined(OS390) +#if U_PLATFORM == U_PF_OS390 #include #endif -#if !defined(OS390) +#if U_PLATFORM != U_PF_OS390 #include #endif @@ -73,7 +84,7 @@ /* Define __USE_XOPEN_EXTENDED for Linux and glibc. */ #ifndef __USE_XOPEN_EXTENDED -#define __USE_XOPEN_EXTENDED +#define __USE_XOPEN_EXTENDED #endif /* Define _INCLUDE_XOPEN_SOURCE_EXTENDED for HP/UX (11?). */ @@ -89,10 +100,6 @@ #undef sleep #endif - - -#include "tsmthred.h" - #define TSMTHREAD_FAIL(msg) errln("%s at file %s, line %d", msg, __FILE__, __LINE__) #define TSMTHREAD_ASSERT(expr) {if (!(expr)) {TSMTHREAD_FAIL("Fail");}} @@ -107,7 +114,7 @@ MultithreadTest::~MultithreadTest() #if (ICU_USE_THREADS==0) -void MultithreadTest::runIndexedTest( int32_t index, UBool exec, +void MultithreadTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ ) { if (exec) logln("TestSuite MultithreadTest: "); @@ -132,7 +139,7 @@ void MultithreadTest::runIndexedTest( int32_t index, UBool exec, #include "unicode/choicfmt.h" #include "unicode/msgfmt.h" #include "unicode/locid.h" -#include "unicode/ucol.h" +#include "unicode/coll.h" #include "unicode/calendar.h" #include "ucaconf.h" @@ -140,7 +147,7 @@ void SimpleThread::errorFunc() { // *(char *)0 = 3; // Force entry into a debugger via a crash; } -void MultithreadTest::runIndexedTest( int32_t index, UBool exec, +void MultithreadTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ ) { if (exec) logln("TestSuite MultithreadTest: "); @@ -176,12 +183,38 @@ void MultithreadTest::runIndexedTest( int32_t index, UBool exec, break; case 4: - name = "TestString"; + name = "TestString"; if (exec) { TestString(); } break; + case 5: + name = "TestArabicShapingThreads"; + if (exec) { + TestArabicShapingThreads(); + } + break; + + case 6: + name = "TestAnyTranslit"; + if (exec) { + TestAnyTranslit(); + } + break; + + case 7: + name = "TestConditionVariables"; + if (exec) { + TestConditionVariables(); + } + break; + case 8: + name = "TestUnifiedCache"; + if (exec) { + TestUnifiedCache(); + } + break; default: name = ""; break; //needed to end loop @@ -199,17 +232,87 @@ void MultithreadTest::runIndexedTest( int32_t index, UBool exec, // //----------------------------------------------------------------------------------- #define THREADTEST_NRTHREADS 8 +#define ARABICSHAPE_THREADTEST 30 class TestThreadsThread : public SimpleThread { public: TestThreadsThread(char* whatToChange) { fWhatToChange = whatToChange; } - virtual void run() { SimpleThread::sleep(1000); + virtual void run() { SimpleThread::sleep(1000); Mutex m; - *fWhatToChange = '*'; + *fWhatToChange = '*'; } private: char *fWhatToChange; +}; +//----------------------------------------------------------------------------------- +// +// TestArabicShapeThreads -- see if calls to u_shapeArabic in many threads works successfully +// +// Set up N threads pointing at N chars. When they are started, they will make calls to doTailTest which tests +// u_shapeArabic, if the calls are successful it will the set * chars. +// At the end we make sure all threads managed to run u_shapeArabic successfully. +// This is a unit test for ticket 9473 +// +//----------------------------------------------------------------------------------- +class TestArabicShapeThreads : public SimpleThread +{ +public: + TestArabicShapeThreads(char* whatToChange) { fWhatToChange = whatToChange;} + virtual void run() { + if(doTailTest()==TRUE) + *fWhatToChange = '*'; + } +private: + char *fWhatToChange; + + UBool doTailTest(void) { + static const UChar src[] = { 0x0020, 0x0633, 0 }; + static const UChar dst_old[] = { 0xFEB1, 0x200B,0 }; + static const UChar dst_new[] = { 0xFEB1, 0xFE73,0 }; + UChar dst[3] = { 0x0000, 0x0000,0 }; + int32_t length; + UErrorCode status; + IntlTest inteltst = IntlTest(); + + status = U_ZERO_ERROR; + length = u_shapeArabic(src, -1, dst, UPRV_LENGTHOF(dst), + U_SHAPE_LETTERS_SHAPE|U_SHAPE_SEEN_TWOCELL_NEAR, &status); + if(U_FAILURE(status)) { + inteltst.errln("Fail: status %s\n", u_errorName(status)); + return FALSE; + } else if(length!=2) { + inteltst.errln("Fail: len %d expected 3\n", length); + return FALSE; + } else if(u_strncmp(dst,dst_old,UPRV_LENGTHOF(dst))) { + inteltst.errln("Fail: got U+%04X U+%04X expected U+%04X U+%04X\n", + dst[0],dst[1],dst_old[0],dst_old[1]); + return FALSE; + } + + + //"Trying new tail + status = U_ZERO_ERROR; + length = u_shapeArabic(src, -1, dst, UPRV_LENGTHOF(dst), + U_SHAPE_LETTERS_SHAPE|U_SHAPE_SEEN_TWOCELL_NEAR|U_SHAPE_TAIL_NEW_UNICODE, &status); + if(U_FAILURE(status)) { + inteltst.errln("Fail: status %s\n", u_errorName(status)); + return FALSE; + } else if(length!=2) { + inteltst.errln("Fail: len %d expected 3\n", length); + return FALSE; + } else if(u_strncmp(dst,dst_new,UPRV_LENGTHOF(dst))) { + inteltst.errln("Fail: got U+%04X U+%04X expected U+%04X U+%04X\n", + dst[0],dst[1],dst_new[0],dst_new[1]); + return FALSE; + } + + + return TRUE; + +} + + }; void MultithreadTest::TestThreads() @@ -259,7 +362,7 @@ void MultithreadTest::TestThreads() } } umtx_unlock(NULL); - + if(count == THREADTEST_NRTHREADS) { logln("->" + UnicodeString(threadTestChars) + "<- Got all threads! cya"); @@ -282,6 +385,78 @@ void MultithreadTest::TestThreads() } +void MultithreadTest::TestArabicShapingThreads() +{ + char threadTestChars[ARABICSHAPE_THREADTEST + 1]; + SimpleThread *threads[ARABICSHAPE_THREADTEST]; + int32_t numThreadsStarted = 0; + + int32_t i; + + for(i=0;i do TestArabicShapingThreads <- Firing off threads.. "); + for(i=0;istart() != 0) { + errln("Error starting thread %d", i); + } + else { + numThreadsStarted++; + } + //SimpleThread::sleep(100); + logln(" Subthread started."); + } + + logln("Waiting for threads to be set.."); + if (numThreadsStarted == 0) { + errln("No threads could be started for testing!"); + return; + } + + int32_t patience = 100; // seconds to wait + + while(patience--) + { + int32_t count = 0; + umtx_lock(NULL); + for(i=0;iTestArabicShapingThreads <- Got all threads! cya"); + for(i=0;i TestArabicShapingThreads <- Waiting.."); + SimpleThread::sleep(500); + } + + errln("-> TestArabicShapingThreads <- PATIENCE EXCEEDED!! Still missing some."); + for(i=0;i 24) { @@ -394,11 +569,6 @@ void MultithreadTest::TestMutex() } // All threads made it by both mutexes. - // Destroy the test mutexes. - umtx_destroy(&gTestMutexA); - umtx_destroy(&gTestMutexB); - gTestMutexA=NULL; - gTestMutexB=NULL; for (i=0; i 0); } - UBool getError(UnicodeString& fillinError) { fillinError = fErrorString; return (fErrors > 0); } + UBool getError() { return (fErrors > 0); } + UBool getError(UnicodeString& fillinError) { fillinError = fErrorString; return (fErrors > 0); } virtual ~ThreadWithStatus(){} protected: ThreadWithStatus() : fErrors(0) {} - void error(const UnicodeString &error) { - fErrors++; fErrorString = error; - SimpleThread::errorFunc(); + void error(const UnicodeString &error) { + fErrors++; fErrorString = error; + SimpleThread::errorFunc(); } void error() { error("An error occured."); } private: @@ -434,7 +604,7 @@ private: //------------------------------------------------------------------------------------------- // -// TestMultithreadedIntl. Test ICU Formatting n a multi-threaded environment +// TestMultithreadedIntl. Test ICU Formatting n a multi-threaded environment // //------------------------------------------------------------------------------------------- @@ -476,9 +646,8 @@ UnicodeString showDifference(const UnicodeString& expected, const UnicodeString& // //------------------------------------------------------------------------------------------- -const int kFormatThreadIterations = 20; // # of iterations per thread -const int kFormatThreadThreads = 10; // # of threads to spawn -const int kFormatThreadPatience = 60; // time in seconds to wait for all threads +const int kFormatThreadIterations = 100; // # of iterations per thread +const int kFormatThreadThreads = 10; // # of threads to spawn #if !UCONFIG_NO_FORMATTING @@ -494,7 +663,7 @@ struct FormatThreadTestData // "Someone from {2} is receiving a #{0} error - {1}. Their telephone call is costing {3 number,currency}." -void formatErrorMessage(UErrorCode &realStatus, const UnicodeString& pattern, const Locale& theLocale, +static void formatErrorMessage(UErrorCode &realStatus, const UnicodeString& pattern, const Locale& theLocale, UErrorCode inStatus0, /* statusString 1 */ const Locale &inCountry2, double currency3, // these numbers are the message arguments. UnicodeString &result) { @@ -516,18 +685,115 @@ void formatErrorMessage(UErrorCode &realStatus, const UnicodeString& pattern, co MessageFormat *fmt = new MessageFormat("MessageFormat's API is broken!!!!!!!!!!!",realStatus); fmt->setLocale(theLocale); fmt->applyPattern(pattern, realStatus); - + if (U_FAILURE(realStatus)) { delete fmt; return; } - FieldPosition ignore = 0; + FieldPosition ignore = 0; fmt->format(myArgs,4,result,ignore,realStatus); delete fmt; } +/** + * Shared formatters & data used by instances of ThreadSafeFormat. + * Exactly one instance of this class is created, and it is then shared concurrently + * by the multiple instances of ThreadSafeFormat. + */ +class ThreadSafeFormatSharedData { + public: + ThreadSafeFormatSharedData(UErrorCode &status); + ~ThreadSafeFormatSharedData(); + LocalPointer fFormat; + Formattable fYDDThing; + Formattable fBBDThing; + UnicodeString fYDDStr; + UnicodeString fBBDStr; +}; + +const ThreadSafeFormatSharedData *gSharedData = NULL; + +ThreadSafeFormatSharedData::ThreadSafeFormatSharedData(UErrorCode &status) { + fFormat.adoptInstead(NumberFormat::createCurrencyInstance(Locale::getUS(), status)); + static const UChar kYDD[] = { 0x59, 0x44, 0x44, 0x00 }; + static const UChar kBBD[] = { 0x42, 0x42, 0x44, 0x00 }; + fYDDThing.adoptObject(new CurrencyAmount(123.456, kYDD, status)); + fBBDThing.adoptObject(new CurrencyAmount(987.654, kBBD, status)); + if (U_FAILURE(status)) { + return; + } + fFormat->format(fYDDThing, fYDDStr, NULL, status); + fFormat->format(fBBDThing, fBBDStr, NULL, status); + gSharedData = this; +} + +ThreadSafeFormatSharedData::~ThreadSafeFormatSharedData() { + gSharedData = NULL; +} + +/** + * Class for thread-safe testing of format. + * Instances of this class appear as members of class FormatThreadTest. + * Multiple instances of FormatThreadTest coexist. + * ThreadSafeFormat::doStuff() is called concurrently to test the thread safety of + * various shared format operations. + */ +class ThreadSafeFormat { +public: + /* give a unique offset to each thread */ + ThreadSafeFormat(UErrorCode &status); + UBool doStuff(int32_t offset, UnicodeString &appendErr, UErrorCode &status) const; +private: + LocalPointer fFormat; // formatter - en_US constructed currency +}; + + +ThreadSafeFormat::ThreadSafeFormat(UErrorCode &status) { + fFormat.adoptInstead(NumberFormat::createCurrencyInstance(Locale::getUS(), status)); +} + +static const UChar kUSD[] = { 0x55, 0x53, 0x44, 0x00 }; + +UBool ThreadSafeFormat::doStuff(int32_t offset, UnicodeString &appendErr, UErrorCode &status) const { + UBool okay = TRUE; + + if(u_strcmp(fFormat->getCurrency(), kUSD)) { + appendErr.append("fFormat currency != ") + .append(kUSD) + .append(", =") + .append(fFormat->getCurrency()) + .append("! "); + okay = FALSE; + } + + if(u_strcmp(gSharedData->fFormat->getCurrency(), kUSD)) { + appendErr.append("gFormat currency != ") + .append(kUSD) + .append(", =") + .append(gSharedData->fFormat->getCurrency()) + .append("! "); + okay = FALSE; + } + UnicodeString str; + const UnicodeString *o=NULL; + Formattable f; + const NumberFormat *nf = NULL; // only operate on it as const. + switch(offset%4) { + case 0: f = gSharedData->fYDDThing; o = &gSharedData->fYDDStr; nf = gSharedData->fFormat.getAlias(); break; + case 1: f = gSharedData->fBBDThing; o = &gSharedData->fBBDStr; nf = gSharedData->fFormat.getAlias(); break; + case 2: f = gSharedData->fYDDThing; o = &gSharedData->fYDDStr; nf = fFormat.getAlias(); break; + case 3: f = gSharedData->fBBDThing; o = &gSharedData->fBBDStr; nf = fFormat.getAlias(); break; + } + nf->format(f, str, NULL, status); + + if(*o != str) { + appendErr.append(showDifference(*o, str)); + okay = FALSE; + } + return okay; +} UBool U_CALLCONV isAcceptable(void *, const char *, const char *, const UDataInfo *) { return TRUE; @@ -543,13 +809,18 @@ public: int fNum; int fTraceInfo; + LocalPointer fTSF; + FormatThreadTest() // constructor is NOT multithread safe. : ThreadWithStatus(), fNum(0), fTraceInfo(0), + fTSF(NULL), fOffset(0) // the locale to use { + UErrorCode status = U_ZERO_ERROR; // TODO: rearrange code to allow checking of status. + fTSF.adoptInstead(new ThreadSafeFormat(status)); static int32_t fgOffset = 0; fgOffset += 3; fOffset = fgOffset; @@ -563,7 +834,7 @@ public: UErrorCode status = U_ZERO_ERROR; #if 0 - // debugging code, + // debugging code, for (int i=0; i<4000; i++) { status = U_ZERO_ERROR; UDataMemory *data1 = udata_openChoice(0, "res", "en_US", isAcceptable, 0, &status); @@ -579,7 +850,7 @@ public: #endif #if 0 - // debugging code, + // debugging code, int m; for (m=0; m<4000; m++) { status = U_ZERO_ERROR; @@ -609,7 +880,7 @@ public: #endif // Keep this data here to avoid static initialization. - FormatThreadTestData kNumberFormatTestData[] = + FormatThreadTestData kNumberFormatTestData[] = { FormatThreadTestData((double)5.0, UnicodeString("5", "")), FormatThreadTestData( 6.0, UnicodeString("6", "")), @@ -619,74 +890,72 @@ public: FormatThreadTestData( 12345, UnicodeString("12,345", "")), FormatThreadTestData( 81890.23, UnicodeString("81,890.23", "")), }; - int32_t kNumberFormatTestDataLength = (int32_t)(sizeof(kNumberFormatTestData) / - sizeof(kNumberFormatTestData[0])); - + int32_t kNumberFormatTestDataLength = UPRV_LENGTHOF(kNumberFormatTestData); + // Keep this data here to avoid static initialization. - FormatThreadTestData kPercentFormatTestData[] = + FormatThreadTestData kPercentFormatTestData[] = { FormatThreadTestData((double)5.0, CharsToUnicodeString("500\\u00a0%")), FormatThreadTestData( 1.0, CharsToUnicodeString("100\\u00a0%")), FormatThreadTestData( 0.26, CharsToUnicodeString("26\\u00a0%")), - FormatThreadTestData( + FormatThreadTestData( 16384.99, CharsToUnicodeString("1\\u00a0638\\u00a0499\\u00a0%")), // U+00a0 = NBSP - FormatThreadTestData( + FormatThreadTestData( 81890.23, CharsToUnicodeString("8\\u00a0189\\u00a0023\\u00a0%")), }; - int32_t kPercentFormatTestDataLength = - (int32_t)(sizeof(kPercentFormatTestData) / sizeof(kPercentFormatTestData[0])); + int32_t kPercentFormatTestDataLength = UPRV_LENGTHOF(kPercentFormatTestData); int32_t iteration; - + status = U_ZERO_ERROR; LocalPointer formatter(NumberFormat::createInstance(Locale::getEnglish(),status)); if(U_FAILURE(status)) { error("Error on NumberFormat::createInstance()."); goto cleanupAndReturn; } - + percentFormatter.adoptInstead(NumberFormat::createPercentInstance(Locale::getFrench(),status)); if(U_FAILURE(status)) { error("Error on NumberFormat::createPercentInstance()."); goto cleanupAndReturn; } - + for(iteration = 0;!getError() && iterationformat(kNumberFormatTestData[whichLine].number, output); - + if(0 != output.compare(kNumberFormatTestData[whichLine].string)) { - error("format().. expected " + kNumberFormatTestData[whichLine].string + error("format().. expected " + kNumberFormatTestData[whichLine].string + " got " + output); goto cleanupAndReturn; } - + // Now check percent. output.remove(); whichLine = (iteration + fOffset)%kPercentFormatTestDataLength; - + percentFormatter->format(kPercentFormatTestData[whichLine].number, output); if(0 != output.compare(kPercentFormatTestData[whichLine].string)) { - error("percent format().. \n" + + error("percent format().. \n" + showDifference(kPercentFormatTestData[whichLine].string,output)); goto cleanupAndReturn; } - - // Test message error + + // Test message error const int kNumberOfMessageTests = 3; UErrorCode statusToCheck; UnicodeString patternToCheck; Locale messageLocale; Locale countryToCheck; double currencyToCheck; - + UnicodeString expected; - + // load the cases. switch((iteration+fOffset) % kNumberOfMessageTests) { @@ -704,12 +973,14 @@ public: break; case 1: statusToCheck= U_INDEX_OUTOFBOUNDS_ERROR; - patternToCheck= "1:A customer in {2} is receiving a #{0} error - {1}. Their telephone call is costing {3,number,currency}."; // number,currency + patternToCheck= "1:A customer in {2} is receiving a #{0} error - {1}. " + "Their telephone call is costing {3,number,currency}."; // number,currency messageLocale= Locale("de","DE@currency=DEM"); countryToCheck= Locale("","BF"); currencyToCheck= 2.32; expected= CharsToUnicodeString( - "1:A customer in Burkina Faso is receiving a #8 error - U_INDEX_OUTOFBOUNDS_ERROR. Their telephone call is costing 2,32\\u00A0DM."); + "1:A customer in Burkina Faso is receiving a #8 error - U_INDEX_OUTOFBOUNDS_ERROR. " + "Their telephone call is costing 2,32\\u00A0DM."); break; case 2: statusToCheck= U_MEMORY_ALLOCATION_ERROR; @@ -725,7 +996,7 @@ public: " \\u00f6S\\u00A040.193,12 on memory."); break; } - + UnicodeString result; UErrorCode status = U_ZERO_ERROR; formatErrorMessage(status,patternToCheck,messageLocale,statusToCheck, @@ -737,19 +1008,27 @@ public: ", error = " + tmp); goto cleanupAndReturn; } - + if(result != expected) { error("PatternFormat: \n" + showDifference(expected,result)); goto cleanupAndReturn; } + // test the Thread Safe Format + UnicodeString appendErr; + if(!fTSF->doStuff(fNum, appendErr, status)) { + error(appendErr); + goto cleanupAndReturn; + } } /* end of for loop */ - + + + cleanupAndReturn: // while (fNum == 4) {SimpleThread::sleep(10000);} // Force a failure by preventing thread from finishing fTraceInfo = 2; } - + private: int32_t fOffset; // where we are testing from. }; @@ -763,6 +1042,11 @@ void MultithreadTest::TestThreadedIntl() UBool haveDisplayedInfo[kFormatThreadThreads]; static const int32_t PATIENCE_SECONDS = 45; + UErrorCode threadSafeErr = U_ZERO_ERROR; + + ThreadSafeFormatSharedData sharedData(threadSafeErr); + assertSuccess("initializing ThreadSafeFormat", threadSafeErr, TRUE); + // // Create and start the test threads // @@ -785,13 +1069,18 @@ void MultithreadTest::TestThreadedIntl() UBool stillRunning; UDate startTime, endTime; startTime = Calendar::getNow(); + double lastComplaint = 0; do { /* Spin until the test threads complete. */ stillRunning = FALSE; endTime = Calendar::getNow(); - if (((int32_t)(endTime - startTime)/U_MILLIS_PER_SECOND) > PATIENCE_SECONDS) { + double elapsedSeconds = ((int32_t)(endTime - startTime)/U_MILLIS_PER_SECOND); + if (elapsedSeconds > PATIENCE_SECONDS) { errln("Patience exceeded. Test is taking too long."); return; + } else if((elapsedSeconds-lastComplaint) > 2.0) { + infoln("%.1f seconds elapsed (still waiting..)", elapsedSeconds); + lastComplaint = elapsedSeconds; } /* The following sleep must be here because the *BSD operating systems @@ -816,6 +1105,7 @@ void MultithreadTest::TestThreadedIntl() // // All threads have finished. // + assertSuccess("finalizing ThreadSafeFormat", threadSafeErr, TRUE); } #endif /* #if !UCONFIG_NO_FORMATTING */ @@ -838,78 +1128,93 @@ struct Line { int32_t buflen; } ; +static UBool +skipLineBecauseOfBug(const UChar *s, int32_t length) { + // TODO: Fix ICU ticket #8052 + if(length >= 3 && + (s[0] == 0xfb2 || s[0] == 0xfb3) && + s[1] == 0x334 && + (s[2] == 0xf73 || s[2] == 0xf75 || s[2] == 0xf81)) { + return TRUE; + } + return FALSE; +} + +static UCollationResult +normalizeResult(int32_t result) { + return result<0 ? UCOL_LESS : result==0 ? UCOL_EQUAL : UCOL_GREATER; +} + class CollatorThreadTest : public ThreadWithStatus { -private: - const UCollator *coll; +private: + const Collator *coll; const Line *lines; int32_t noLines; + UBool isAtLeastUCA62; public: CollatorThreadTest() : ThreadWithStatus(), coll(NULL), lines(NULL), - noLines(0) + noLines(0), + isAtLeastUCA62(TRUE) { }; - void setCollator(UCollator *c, Line *l, int32_t nl) + void setCollator(Collator *c, Line *l, int32_t nl, UBool atLeastUCA62) { coll = c; lines = l; noLines = nl; + isAtLeastUCA62 = atLeastUCA62; } virtual void run() { - //sleep(10000); - int32_t line = 0; - uint8_t sk1[1024], sk2[1024]; uint8_t *oldSk = NULL, *newSk = sk1; - int32_t resLen = 0, oldLen = 0; + int32_t oldLen = 0; + int32_t prev = 0; int32_t i = 0; for(i = 0; i < noLines; i++) { - resLen = ucol_getSortKey(coll, lines[i].buff, lines[i].buflen, newSk, 1024); + if(lines[i].buflen == 0) { continue; } + + if(skipLineBecauseOfBug(lines[i].buff, lines[i].buflen)) { continue; } - int32_t res = 0, cmpres = 0, cmpres2 = 0; + int32_t resLen = coll->getSortKey(lines[i].buff, lines[i].buflen, newSk, 1024); if(oldSk != NULL) { - res = strcmp((char *)oldSk, (char *)newSk); - cmpres = ucol_strcoll(coll, lines[i-1].buff, lines[i-1].buflen, lines[i].buff, lines[i].buflen); - cmpres2 = ucol_strcoll(coll, lines[i].buff, lines[i].buflen, lines[i-1].buff, lines[i-1].buflen); - //cmpres = res; - //cmpres2 = -cmpres; + int32_t skres = strcmp((char *)oldSk, (char *)newSk); + int32_t cmpres = coll->compare(lines[prev].buff, lines[prev].buflen, lines[i].buff, lines[i].buflen); + int32_t cmpres2 = coll->compare(lines[i].buff, lines[i].buflen, lines[prev].buff, lines[prev].buflen); if(cmpres != -cmpres2) { - error("Compare result not symmetrical on line "+ line); + error(UnicodeString("Compare result not symmetrical on line ") + (i + 1)); break; } - if(((res&0x80000000) != (cmpres&0x80000000)) || (res == 0 && cmpres != 0) || (res != 0 && cmpres == 0)) { - error(UnicodeString("Difference between ucol_strcoll and sortkey compare on line ")+ UnicodeString(line)); + if(cmpres != normalizeResult(skres)) { + error(UnicodeString("Difference between coll->compare and sortkey compare on line ") + (i + 1)); break; } + int32_t res = cmpres; + if(res == 0 && !isAtLeastUCA62) { + // Up to UCA 6.1, the collation test files use a custom tie-breaker, + // comparing the raw input strings. + res = u_strcmpCodePointOrder(lines[prev].buff, lines[i].buff); + // Starting with UCA 6.2, the collation test files use the standard UCA tie-breaker, + // comparing the NFD versions of the input strings, + // which we do via setting strength=identical. + } if(res > 0) { - error(UnicodeString("Line %i is not greater or equal than previous line ")+ UnicodeString(i)); + error(UnicodeString("Line is not greater or equal than previous line, for line ") + (i + 1)); break; - } else if(res == 0) { /* equal */ - res = u_strcmpCodePointOrder(lines[i-1].buff, lines[i].buff); - if (res == 0) { - error(UnicodeString("Probable error in test file on line %i (comparing identical strings)")+ UnicodeString(i)); - break; - } - /* - * UCA 6.0 test files can have lines that compare == if they are - * different strings but canonically equivalent. - else if (res > 0) { - error(UnicodeString("Sortkeys are identical, but code point compare gives >0 on line ")+ UnicodeString(i)); - break; - } - */ } } oldSk = newSk; oldLen = resLen; + (void)oldLen; // Suppress set but not used warning. + prev = i; newSk = (newSk == sk1)?sk2:sk1; } @@ -963,7 +1268,7 @@ void MultithreadTest::TestCollators() if (testFile == 0) { *(buffer+bufLen) = 0; dataerrln("could not open any of the conformance test files, tried opening base %s", buffer); - return; + return; } else { infoln( "INFO: Working with the stub file.\n" @@ -974,26 +1279,25 @@ void MultithreadTest::TestCollators() } } - Line *lines = new Line[200000]; - memset(lines, 0, sizeof(Line)*200000); + LocalArray lines(new Line[200000]); + memset(lines.getAlias(), 0, sizeof(Line)*200000); int32_t lineNum = 0; UChar bufferU[1024]; - int32_t buflen = 0; uint32_t first = 0; - uint32_t offset = 0; while (fgets(buffer, 1024, testFile) != NULL) { - offset = 0; - if(*buffer == 0 || strlen(buffer) < 3 || buffer[0] == '#') { - continue; + if(*buffer == 0 || buffer[0] == '#') { + // Store empty and comment lines so that errors are reported + // for the real test file lines. + lines[lineNum].buflen = 0; + lines[lineNum].buff[0] = 0; + } else { + int32_t buflen = u_parseString(buffer, bufferU, 1024, &first, &status); + lines[lineNum].buflen = buflen; + u_memcpy(lines[lineNum].buff, bufferU, buflen); + lines[lineNum].buff[buflen] = 0; } - offset = u_parseString(buffer, bufferU, 1024, &first, &status); - buflen = offset; - bufferU[offset++] = 0; - lines[lineNum].buflen = buflen; - //lines[lineNum].buff = new UChar[buflen+1]; - u_memcpy(lines[lineNum].buff, bufferU, buflen); lineNum++; } fclose(testFile); @@ -1002,16 +1306,21 @@ void MultithreadTest::TestCollators() return; } - UCollator *coll = ucol_open("root", &status); + UVersionInfo uniVersion; + static const UVersionInfo v62 = { 6, 2, 0, 0 }; + u_getUnicodeVersion(uniVersion); + UBool isAtLeastUCA62 = uprv_memcmp(uniVersion, v62, 4) >= 0; + + LocalPointer coll(Collator::createInstance(Locale::getRoot(), status)); if(U_FAILURE(status)) { errcheckln(status, "Couldn't open UCA collator"); return; } - ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); - ucol_setAttribute(coll, UCOL_CASE_FIRST, UCOL_OFF, &status); - ucol_setAttribute(coll, UCOL_CASE_LEVEL, UCOL_OFF, &status); - ucol_setAttribute(coll, UCOL_STRENGTH, UCOL_TERTIARY, &status); - ucol_setAttribute(coll, UCOL_ALTERNATE_HANDLING, UCOL_NON_IGNORABLE, &status); + coll->setAttribute(UCOL_NORMALIZATION_MODE, UCOL_ON, status); + coll->setAttribute(UCOL_CASE_FIRST, UCOL_OFF, status); + coll->setAttribute(UCOL_CASE_LEVEL, UCOL_OFF, status); + coll->setAttribute(UCOL_STRENGTH, isAtLeastUCA62 ? UCOL_IDENTICAL : UCOL_TERTIARY, status); + coll->setAttribute(UCOL_ALTERNATE_HANDLING, UCOL_NON_IGNORABLE, status); int32_t noSpawned = 0; int32_t spawnResult = 0; @@ -1021,7 +1330,7 @@ void MultithreadTest::TestCollators() int32_t j = 0; for(j = 0; j < kCollatorThreadThreads; j++) { //logln("Setting collator %i", j); - tests[j].setCollator(coll, lines, lineNum); + tests[j].setCollator(coll.getAlias(), lines.getAlias(), lineNum, isAtLeastUCA62); } for(j = 0; j < kCollatorThreadThreads; j++) { log("%i ", j); @@ -1074,12 +1383,6 @@ void MultithreadTest::TestCollators() errln("There were errors."); SimpleThread::errorFunc(); } - ucol_close(coll); - //for(i = 0; i < lineNum; i++) { - //delete[] lines[i].buff; - //} - delete[] lines; - return; } @@ -1087,7 +1390,6 @@ void MultithreadTest::TestCollators() } errln("patience exceeded. "); SimpleThread::errorFunc(); - ucol_close(coll); } #endif /* #if !UCONFIG_NO_COLLATION */ @@ -1097,7 +1399,7 @@ void MultithreadTest::TestCollators() //------------------------------------------------------------------------------------------- // -// StringThreadTest2 +// StringThreadTest2 // //------------------------------------------------------------------------------------------- @@ -1144,7 +1446,7 @@ public: // while (fNum == 4) {SimpleThread::sleep(10000);} // Force a failure by preventing thread from finishing fTraceInfo = 2; } - + }; // ** The actual test function. @@ -1164,13 +1466,13 @@ void MultithreadTest::TestString() for(j = 0; j < kStringThreadThreads; j++) { tests[j] = new StringThreadTest2(testString, j); } - + logln(UnicodeString("Spawning: ") + kStringThreadThreads + " threads * " + kStringThreadIterations + " iterations each."); for(j = 0; j < kStringThreadThreads; j++) { int32_t threadStatus = tests[j]->start(); if (threadStatus != 0) { errln("System Error %d starting thread number %d.", threadStatus, j); - SimpleThread::errorFunc(); + SimpleThread::errorFunc(); goto cleanupAndReturn; } } @@ -1187,9 +1489,9 @@ void MultithreadTest::TestString() if (tests[i]->isRunning() == FALSE) { completed++; - + logln(UnicodeString("Test #") + i + " is complete.. "); - + UnicodeString theErr; if(tests[i]->getError(theErr)) { @@ -1199,7 +1501,7 @@ void MultithreadTest::TestString() // print out the error, too, if any. } } - + if(completed == kStringThreadThreads) { logln("Done!"); @@ -1219,7 +1521,7 @@ void MultithreadTest::TestString() } if (terrs > 0) { - SimpleThread::errorFunc(); + SimpleThread::errorFunc(); } cleanupAndReturn: @@ -1237,4 +1539,266 @@ cleanupAndReturn: } } + +// Test for ticket #10673, race in cache code in AnyTransliterator. +// It's difficult to make the original unsafe code actually fail, but +// this test will fairly reliably take the code path for races in +// populating the cache. + +#if !UCONFIG_NO_TRANSLITERATION +class TxThread: public SimpleThread { + private: + Transliterator *fSharedTranslit; + public: + UBool fSuccess; + TxThread(Transliterator *tx) : fSharedTranslit(tx), fSuccess(FALSE) {}; + ~TxThread(); + void run(); +}; + +TxThread::~TxThread() {} +void TxThread::run() { + UnicodeString greekString("\\u03B4\\u03B9\\u03B1\\u03C6\\u03BF\\u03C1\\u03B5\\u03C4\\u03B9\\u03BA\\u03BF\\u03CD\\u03C2"); + greekString = greekString.unescape(); + fSharedTranslit->transliterate(greekString); + fSuccess = greekString[0] == 0x64; // 'd'. The whole transliterated string is "diaphoretikous" (accented u). +} +#endif + + +void MultithreadTest::TestAnyTranslit() { +#if !UCONFIG_NO_TRANSLITERATION + UErrorCode status = U_ZERO_ERROR; + LocalPointer tx(Transliterator::createInstance("Any-Latin", UTRANS_FORWARD, status)); + if (U_FAILURE(status)) { + dataerrln("File %s, Line %d: Error, status = %s", __FILE__, __LINE__, u_errorName(status)); + return; + } + TxThread * threads[4]; + int32_t i; + for (i=0; i<4; i++) { + threads[i] = new TxThread(tx.getAlias()); + } + for (i=0; i<4; i++) { + threads[i]->start(); + } + int32_t patience = 100; + UBool success; + UBool someThreadRunning; + do { + someThreadRunning = FALSE; + success = TRUE; + for (i=0; i<4; i++) { + if (threads[i]->isRunning()) { + someThreadRunning = TRUE; + SimpleThread::sleep(10); + break; + } else { + if (threads[i]->fSuccess == FALSE) { + success = FALSE; + } + } + } + } while (someThreadRunning && --patience > 0); + + if (patience <= 0) { + errln("File %s, Line %d: Error, one or more threads did not complete.", __FILE__, __LINE__); + } + if (success == FALSE) { + errln("File %s, Line %d: Error, transliteration result incorrect.", __FILE__, __LINE__); + } + + for (i=0; i<4; i++) { + delete threads[i]; + } +#endif // !UCONFIG_NO_TRANSLITERATION +} + + +// Condition Variables Test +// Create a swarm of threads. +// Using a mutex and a condition variables each thread +// Increments a global count of started threads. +// Broadcasts that it has started. +// Waits on the condition that all threads have started. +// Increments a global count of finished threads. +// Waits on the condition that all threads have finished. +// Exits. + +class CondThread: public SimpleThread { + public: + CondThread() :fFinished(false) {}; + ~CondThread() {}; + void run(); + bool fFinished; +}; + +static UMutex gCTMutex = U_MUTEX_INITIALIZER; +static UConditionVar gCTConditionVar = U_CONDITION_INITIALIZER; +int gConditionTestOne = 1; // Value one. Non-const, extern linkage to inhibit + // compiler assuming a known value. +int gStartedThreads; +int gFinishedThreads; +static const int NUMTHREADS = 10; + +static MultithreadTest *gThisTest = NULL; // Make test frame work functions available to + // non-member functions. + +// Worker thread function. +void CondThread::run() { + umtx_lock(&gCTMutex); + gStartedThreads += gConditionTestOne; + umtx_condBroadcast(&gCTConditionVar); + + while (gStartedThreads < NUMTHREADS) { + if (gFinishedThreads != 0) { + gThisTest->errln("File %s, Line %d: Error, gStartedThreads = %d, gFinishedThreads = %d", + __FILE__, __LINE__, gStartedThreads, gFinishedThreads); + } + umtx_condWait(&gCTConditionVar, &gCTMutex); + } + + gFinishedThreads += gConditionTestOne; + fFinished = true; + umtx_condBroadcast(&gCTConditionVar); + + while (gFinishedThreads < NUMTHREADS) { + umtx_condWait(&gCTConditionVar, &gCTMutex); + } + umtx_unlock(&gCTMutex); +} + +void MultithreadTest::TestConditionVariables() { + gThisTest = this; + gStartedThreads = 0; + gFinishedThreads = 0; + int i; + + umtx_lock(&gCTMutex); + CondThread *threads[NUMTHREADS]; + for (i=0; istart(); + } + + while (gStartedThreads < NUMTHREADS) { + umtx_condWait(&gCTConditionVar, &gCTMutex); + } + + while (gFinishedThreads < NUMTHREADS) { + umtx_condWait(&gCTConditionVar, &gCTMutex); + } + + umtx_unlock(&gCTMutex); + + for (i=0; ifFinished) { + errln("File %s, Line %d: Error, threads[%d]->fFinished == false", __FILE__, __LINE__, i); + } + delete threads[i]; + } +} + +static const char *gCacheLocales[] = {"en_US", "en_GB", "fr_FR", "fr"}; +static int32_t gObjectsCreated = 0; +static const int32_t CACHE_LOAD = 3; + +class UCTMultiThreadItem : public SharedObject { + public: + char *value; + UCTMultiThreadItem(const char *x) : value(NULL) { + value = uprv_strdup(x); + } + virtual ~UCTMultiThreadItem() { + uprv_free(value); + } +}; + +U_NAMESPACE_BEGIN + +template<> U_EXPORT +const UCTMultiThreadItem *LocaleCacheKey::createObject( + const void * /*unused*/, UErrorCode & /* status */) const { + // Since multiple threads are hitting the cache for the first time, + // no objects should be created yet. + umtx_lock(&gCTMutex); + if (gObjectsCreated != 0) { + gThisTest->errln("Expected no objects to be created yet."); + } + umtx_unlock(&gCTMutex); + + // Big, expensive object that takes 1 second to create. + SimpleThread::sleep(1000); + + // Log that we created an object. + umtx_lock(&gCTMutex); + ++gObjectsCreated; + umtx_unlock(&gCTMutex); + UCTMultiThreadItem *result = new UCTMultiThreadItem(fLoc.getName()); + result->addRef(); + return result; +} + +U_NAMESPACE_END + +class UnifiedCacheThread: public SimpleThread { + public: + UnifiedCacheThread(const char *loc) : fLoc(loc) {}; + ~UnifiedCacheThread() {}; + void run(); + const char *fLoc; +}; + +void UnifiedCacheThread::run() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = UnifiedCache::getInstance(status); + U_ASSERT(status == U_ZERO_ERROR); + const UCTMultiThreadItem *item = NULL; + cache->get(LocaleCacheKey(fLoc), item, status); + U_ASSERT(item != NULL); + if (uprv_strcmp(fLoc, item->value)) { + gThisTest->errln("Expected %s, got %s", fLoc, item->value); + } + item->removeRef(); + + // Mark this thread as finished + umtx_lock(&gCTMutex); + ++gFinishedThreads; + umtx_condBroadcast(&gCTConditionVar); + umtx_unlock(&gCTMutex); +} + +void MultithreadTest::TestUnifiedCache() { + UErrorCode status = U_ZERO_ERROR; + const UnifiedCache *cache = UnifiedCache::getInstance(status); + U_ASSERT(cache != NULL); + cache->flush(); + gThisTest = this; + gFinishedThreads = 0; + gObjectsCreated = 0; + + UnifiedCacheThread *threads[CACHE_LOAD][UPRV_LENGTHOF(gCacheLocales)]; + for (int32_t i=0; istart(); + } + } + // Wait on all the threads to complete verify that LENGTHOF(gCacheLocales) + // objects were created. + umtx_lock(&gCTMutex); + while (gFinishedThreads < CACHE_LOAD*UPRV_LENGTHOF(gCacheLocales)) { + umtx_condWait(&gCTConditionVar, &gCTMutex); + } + assertEquals("Objects created", UPRV_LENGTHOF(gCacheLocales), gObjectsCreated); + umtx_unlock(&gCTMutex); + + // clean up threads + for (int32_t i=0; i