icuSources/test/perf/utfperf/utfperf.cpp

   1 /*
   2  **********************************************************************
   3  *   Copyright (C) 2002-2007, International Business Machines
   4  *   Corporation and others.  All Rights Reserved.
   5  **********************************************************************
   6  *   file name:  utfperf.cpp
   7  *   encoding:   US-ASCII
   8  *   tab size:   8 (not used)
   9  *   indentation:4
  10  *
  11  *   created on: 2005Nov17
  12  *   created by: Raymond Yang
  13  *
  14  *   Ported from utfper.c created by Markus W. Scherer
  15  *   Performance test program for Unicode converters
  16  */
  17
  18 #include <stdio.h>
  19 #include <stdlib.h>
  20 #include "unicode/uperf.h"
  21 #include "uoptions.h"
  22
  23 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  24
  25 /* definitions and text buffers */
  26
  27 #define INPUT_CAPACITY (1024*1024)
  28 #define INTERMEDIATE_CAPACITY 4096
  29 #define INTERMEDIATE_SMALL_CAPACITY 20
  30 #define PIVOT_CAPACITY 1024
  31 #define OUTPUT_CAPACITY INPUT_CAPACITY
  32
  33 static char utf8[INPUT_CAPACITY];
  34 static UChar pivot[INTERMEDIATE_CAPACITY];
  35
  36 static UChar output[OUTPUT_CAPACITY];
  37 static char intermediate[OUTPUT_CAPACITY];
  38
  39 static int32_t utf8Length, encodedLength, outputLength, countInputCodePoints;
  40
  41 static int32_t fromUCallbackCount;
  42
  43 // Command-line options specific to utfperf.
  44 // Options do not have abbreviations: Force readable command lines.
  45 // (Using U+0001 for abbreviation characters.)
  46 enum {
  47     CHARSET,
  48     CHUNK_LENGTH,
  49     PIVOT_LENGTH,
  50     UTFPERF_OPTIONS_COUNT
  51 };
  52
  53 static UOption options[UTFPERF_OPTIONS_COUNT]={
  54     UOPTION_DEF("charset",  '\x01', UOPT_REQUIRES_ARG),
  55     UOPTION_DEF("chunk",    '\x01', UOPT_REQUIRES_ARG),
  56     UOPTION_DEF("pivot",    '\x01', UOPT_REQUIRES_ARG)
  57 };
  58
  59 static const char *const utfperf_usage =
  60     "\t--charset   Charset for which to test performance, e.g. windows-1251.\n"
  61     "\t            Default: UTF-8\n"
  62     "\t--chunk     Length (in bytes) of charset output chunks. [4096]\n"
  63     "\t--pivot     Length (in UChars) of the UTF-16 pivot buffer, if applicable.\n"
  64     "\t            [1024]\n";
  65
  66 // Test object.
  67 class  UtfPerformanceTest : public UPerfTest{
  68 public:
  69     UtfPerformanceTest(int32_t argc, const char *argv[], UErrorCode &status)
  70             : UPerfTest(argc, argv, options, LENGTHOF(options), utfperf_usage, status) {
  71         if (U_SUCCESS(status)) {
  72             charset = options[CHARSET].value;
  73
  74             chunkLength = atoi(options[CHUNK_LENGTH].value);
  75             if (chunkLength < 1 || OUTPUT_CAPACITY < chunkLength) {
  76                 fprintf(stderr, "error: chunk length must be 1..%ld\n", (long)OUTPUT_CAPACITY);
  77                 status = U_ILLEGAL_ARGUMENT_ERROR;
  78             }
  79
  80             pivotLength = atoi(options[PIVOT_LENGTH].value);
  81             if (pivotLength < 1 || PIVOT_CAPACITY < pivotLength) {
  82                 fprintf(stderr, "error: pivot length must be 1..%ld\n", (long)PIVOT_CAPACITY);
  83                 status = U_ILLEGAL_ARGUMENT_ERROR;
  84             }
  85
  86             int32_t inputLength;
  87             UPerfTest::getBuffer(inputLength, status);
  88             countInputCodePoints = u_countChar32(buffer, bufferLen);
  89             u_strToUTF8(utf8, (int32_t)sizeof(utf8), &utf8Length, buffer, bufferLen, &status);
  90         }
  91     }
  92
  93     virtual UPerfFunction* runIndexedTest(int32_t index, UBool exec, const char* &name, char* par = NULL);
  94
  95     const UChar *getBuffer() const { return buffer; }
  96     int32_t getBufferLen() const { return bufferLen; }
  97
  98     const char *charset;
  99     int32_t chunkLength, pivotLength;
 100 };
 101
 102 U_CDECL_BEGIN
 103 // Custom callback for counting callback calls.
 104 static void U_CALLCONV
 105 fromUCallback(const void *context,
 106               UConverterFromUnicodeArgs *fromUArgs,
 107               const UChar *codeUnits,
 108               int32_t length,
 109               UChar32 codePoint,
 110               UConverterCallbackReason reason,
 111               UErrorCode *pErrorCode) {
 112     if (reason <= UCNV_IRREGULAR) {
 113         ++fromUCallbackCount;
 114     }
 115     UCNV_FROM_U_CALLBACK_SUBSTITUTE(context, fromUArgs, codeUnits, length, codePoint, reason, pErrorCode);
 116 }
 117 U_CDECL_END
 118
 119 // Base class for Roundtrip, FromUnicode and FromUTF8 with common setup.
 120 class Command : public UPerfFunction {
 121 protected:
 122     Command(const UtfPerformanceTest &testcase)
 123             : testcase(testcase),
 124               input(testcase.getBuffer()), inputLength(testcase.getBufferLen()),
 125               errorCode(U_ZERO_ERROR) {
 126         cnv=ucnv_open(testcase.charset, &errorCode);
 127         if (U_FAILURE(errorCode)) {
 128             fprintf(stderr, "error opening converter for \"%s\" - %s\n", testcase.charset, u_errorName(errorCode));
 129         }
 130         ucnv_setFromUCallBack(cnv, fromUCallback, NULL, NULL, NULL, &errorCode);
 131     }
 132 public:
 133     virtual ~Command(){
 134         if(U_SUCCESS(errorCode)) {
 135             ucnv_close(cnv);
 136         }
 137     }
 138     // virtual void call(UErrorCode* pErrorCode) { ... }
 139     virtual long getOperationsPerIteration(){
 140         return countInputCodePoints;
 141     }
 142
 143     const UtfPerformanceTest &testcase;
 144     const UChar *input;
 145     int32_t inputLength;
 146     UErrorCode errorCode;
 147     UConverter *cnv;
 148 };
 149
 150 // Test roundtrip UTF-16->encoding->UTF-16.
 151 class Roundtrip : public Command {
 152 protected:
 153     Roundtrip(const UtfPerformanceTest &testcase) : Command(testcase) {}
 154 public:
 155     static UPerfFunction* get(const UtfPerformanceTest &testcase) {
 156         Roundtrip * t = new Roundtrip(testcase);
 157         if (U_SUCCESS(t->errorCode)){
 158             return t;
 159         } else {
 160             delete t;
 161             return NULL;
 162         }
 163     }
 164     virtual void call(UErrorCode* pErrorCode){
 165         const UChar *pIn, *pInLimit;
 166         UChar *pOut, *pOutLimit;
 167         char *pInter, *pInterLimit;
 168         const char *p;
 169         UBool flush;
 170
 171         ucnv_reset(cnv);
 172         fromUCallbackCount=0;
 173
 174         pIn=input;
 175         pInLimit=input+inputLength;
 176
 177         pOut=output;
 178         pOutLimit=output+OUTPUT_CAPACITY;
 179
 180         pInterLimit=intermediate+testcase.chunkLength;
 181
 182         encodedLength=outputLength=0;
 183         flush=FALSE;
 184
 185         do {
 186             /* convert a block of [pIn..pInLimit[ to the encoding in intermediate[] */
 187             pInter=intermediate;
 188             ucnv_fromUnicode(cnv, &pInter, pInterLimit, &pIn, pInLimit, NULL, TRUE, pErrorCode);
 189             encodedLength+=(int32_t)(pInter-intermediate);
 190
 191             if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 192                 /* make sure that we convert once more to really flush */
 193                 *pErrorCode=U_ZERO_ERROR;
 194             } else if(U_FAILURE(*pErrorCode)) {
 195                 return;
 196             } else if(pIn==pInLimit) {
 197                 flush=TRUE;
 198             }
 199
 200             /* convert the block [intermediate..pInter[ back to UTF-16 */
 201             p=intermediate;
 202             ucnv_toUnicode(cnv, &pOut, pOutLimit,&p, pInter,NULL, flush,pErrorCode);
 203             if(U_FAILURE(*pErrorCode)) {
 204                 return;
 205             }
 206             /* intermediate must have been consumed (p==pInter) because of the converter semantics */
 207         } while(!flush);
 208
 209         outputLength=pOut-output;
 210         if(inputLength!=outputLength) {
 211             fprintf(stderr, "error: roundtrip failed, inputLength %d!=outputLength %d\n", inputLength, outputLength);
 212             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 213         }
 214     }
 215 };
 216
 217 // Test one-way conversion UTF-16->encoding.
 218 class FromUnicode : public Command {
 219 protected:
 220     FromUnicode(const UtfPerformanceTest &testcase) : Command(testcase) {}
 221 public:
 222     static UPerfFunction* get(const UtfPerformanceTest &testcase) {
 223         FromUnicode * t = new FromUnicode(testcase);
 224         if (U_SUCCESS(t->errorCode)){
 225             return t;
 226         } else {
 227             delete t;
 228             return NULL;
 229         }
 230     }
 231     virtual void call(UErrorCode* pErrorCode){
 232         const UChar *pIn, *pInLimit;
 233         char *pInter, *pInterLimit;
 234
 235         ucnv_resetFromUnicode(cnv);
 236         fromUCallbackCount=0;
 237
 238         pIn=input;
 239         pInLimit=input+inputLength;
 240
 241         pInterLimit=intermediate+testcase.chunkLength;
 242
 243         encodedLength=0;
 244
 245         for(;;) {
 246             pInter=intermediate;
 247             ucnv_fromUnicode(cnv, &pInter, pInterLimit, &pIn, pInLimit, NULL, TRUE, pErrorCode);
 248             encodedLength+=(int32_t)(pInter-intermediate);
 249
 250             if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 251                 /* make sure that we convert once more to really flush */
 252                 *pErrorCode=U_ZERO_ERROR;
 253             } else if(U_FAILURE(*pErrorCode)) {
 254                 return;
 255             } else {
 256                 break;  // all done
 257             }
 258         }
 259     }
 260 };
 261
 262 // Test one-way conversion UTF-8->encoding.
 263 class FromUTF8 : public Command {
 264 protected:
 265     FromUTF8(const UtfPerformanceTest &testcase)
 266             : Command(testcase),
 267               utf8Cnv(NULL),
 268               input8(utf8), input8Length(utf8Length) {
 269         utf8Cnv=ucnv_open("UTF-8", &errorCode);
 270     }
 271 public:
 272     static UPerfFunction* get(const UtfPerformanceTest &testcase) {
 273         FromUTF8 * t = new FromUTF8(testcase);
 274         if (U_SUCCESS(t->errorCode)){
 275             return t;
 276         } else {
 277             delete t;
 278             return NULL;
 279         }
 280     }
 281     ~FromUTF8() {
 282         ucnv_close(utf8Cnv);
 283     }
 284     virtual void call(UErrorCode* pErrorCode){
 285         const char *pIn, *pInLimit;
 286         char *pInter, *pInterLimit;
 287         UChar *pivotSource, *pivotTarget, *pivotLimit;
 288
 289         ucnv_resetToUnicode(utf8Cnv);
 290         ucnv_resetFromUnicode(cnv);
 291         fromUCallbackCount=0;
 292
 293         pIn=input8;
 294         pInLimit=input8+input8Length;
 295
 296         pInterLimit=intermediate+testcase.chunkLength;
 297
 298         pivotSource=pivotTarget=pivot;
 299         pivotLimit=pivot+testcase.pivotLength;
 300
 301         encodedLength=0;
 302
 303         for(;;) {
 304             pInter=intermediate;
 305             ucnv_convertEx(cnv, utf8Cnv,
 306                            &pInter, pInterLimit,
 307                            &pIn, pInLimit,
 308                            pivot, &pivotSource, &pivotTarget, pivotLimit,
 309                            FALSE, TRUE, pErrorCode);
 310             encodedLength+=(int32_t)(pInter-intermediate);
 311
 312             if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 313                 /* make sure that we convert once more to really flush */
 314                 *pErrorCode=U_ZERO_ERROR;
 315             } else if(U_FAILURE(*pErrorCode)) {
 316                 return;
 317             } else {
 318                 break;  // all done
 319             }
 320         }
 321     }
 322 protected:
 323     UConverter *utf8Cnv;
 324     const char *input8;
 325     int32_t input8Length;
 326 };
 327
 328 UPerfFunction* UtfPerformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* par) {
 329     switch (index) {
 330         case 0: name = "Roundtrip";     if (exec) return Roundtrip::get(*this); break;
 331         case 1: name = "FromUnicode";   if (exec) return FromUnicode::get(*this); break;
 332         case 2: name = "FromUTF8";      if (exec) return FromUTF8::get(*this); break;
 333         default: name = ""; break;
 334     }
 335     return NULL;
 336 }
 337
 338 int main(int argc, const char *argv[])
 339 {
 340     // Default values for command-line options.
 341     options[CHARSET].value = "UTF-8";
 342     options[CHUNK_LENGTH].value = "4096";
 343     options[PIVOT_LENGTH].value = "1024";
 344
 345     UErrorCode status = U_ZERO_ERROR;
 346     UtfPerformanceTest test(argc, argv, status);
 347
 348         if (U_FAILURE(status)){
 349         printf("The error is %s\n", u_errorName(status));
 350         test.usage();
 351         return status;
 352     }
 353
 354     if (test.run() == FALSE){
 355         fprintf(stderr, "FAILED: Tests could not be run please check the "
 356                                     "arguments.\n");
 357         return -1;
 358     }
 359
 360     if (fromUCallbackCount > 0) {
 361         printf("Number of fromUnicode callback calls in the last iteration: %ld\n", (long)fromUCallbackCount);
 362     }
 363
 364     return 0;
 365 }