--- /dev/null
+/*
+**********************************************************************
+* Copyright (C) 2002, International Business Machines
+* Corporation and others. All Rights Reserved.
+**********************************************************************
+* file name: utfperf.c
+* encoding: US-ASCII
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2002apr17
+* created by: Markus W. Scherer
+*
+* Performance test program for Unicode converters
+* (converters that support all Unicode code points).
+* Takes a UTF-8 file as input.
+*/
+
+#include <stdio.h>
+#include <string.h>
+
+#include <fcntl.h> /* for _O_BINARY */
+#include <io.h> /* for _setmode() */
+
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+# include <windows.h>
+#else
+# include <sys/time.h>
+ static unsigned long
+ timeGetTime() {
+ struct timeval t;
+
+ gettimeofday(&t, 0);
+ return t.tv_sec*1000+t.tv_usec/1000;
+ };
+#endif
+
+#include "unicode/utypes.h"
+#include "unicode/ucnv.h"
+#include "unicode/ustring.h"
+
+/* definitions and text buffers */
+
+#define INPUT_CAPACITY (1024*1024)
+#define INTERMEDIATE_CAPACITY 4096
+#define INTERMEDIATE_SMALL_CAPACITY 20
+#define OUTPUT_CAPACITY INPUT_CAPACITY
+
+#define TARGET_MEASURE_TIME_MS 2000
+
+#define PERCENT(a, b) (int)(((a)*200+1)/(2*(b)))
+
+#define ARRAY_LENGTH(a) (sizeof(a)/sizeof((a)[0]))
+
+static UChar input[INPUT_CAPACITY], output[OUTPUT_CAPACITY];
+static char intermediate[INTERMEDIATE_CAPACITY];
+
+static int32_t inputLength, encodedLength, outputLength, countInputCodePoints;
+
+static int32_t utf8Length=0;
+static double utf8Time=0.;
+
+static const char *const
+utfNames[]={
+ "UTF-8", /* UTF-8 should always be first to serve as percentage reference */
+ "SCSU", "BOCU-1" /*, "CESU-8" *//*, "UTF-16BE", "UTF-16LE"*//*, "GB18030"*/
+};
+
+/* functions */
+
+typedef void
+RoundtripFn(UConverter *cnv, int32_t intermediateCapacity, UErrorCode *pErrorCode);
+
+static void
+roundtrip(UConverter *cnv, int32_t intermediateCapacity, UErrorCode *pErrorCode) {
+ const UChar *pIn, *pInLimit;
+ UChar *pOut, *pOutLimit;
+ char *pInter, *pInterLimit, *p;
+ UBool flush;
+
+ ucnv_reset(cnv);
+
+ pIn=input;
+ pInLimit=input+inputLength;
+
+ pOut=output;
+ pOutLimit=output+OUTPUT_CAPACITY;
+
+ pInterLimit=intermediate+intermediateCapacity;
+
+ encodedLength=outputLength=0;
+ flush=FALSE;
+
+ while(pIn<pInLimit || !flush) {
+ /* convert a block of [pIn..pInLimit[ to the encoding in intermediate[] */
+ pInter=intermediate;
+ flush=(UBool)(pIn==pInLimit);
+ ucnv_fromUnicode(cnv,
+ &pInter, pInterLimit,
+ &pIn, pInLimit,
+ NULL, flush,
+ pErrorCode);
+ encodedLength+=(int32_t)(pInter-intermediate);
+
+ if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
+ /* in case flush was TRUE make sure that we convert once more to really flush */
+ flush=FALSE;
+ *pErrorCode=U_ZERO_ERROR;
+ } else if(U_FAILURE(*pErrorCode)) {
+ return;
+ }
+
+ /* convert the block [intermediate..pInter[ back to UTF-16 */
+ p=intermediate;
+ ucnv_toUnicode(cnv,
+ &pOut, pOutLimit,
+ &p, pInter,
+ NULL, flush,
+ pErrorCode);
+ if(U_FAILURE(*pErrorCode)) {
+ return;
+ }
+ /* intermediate must have been consumed (p==pInter) because of the converter semantics */
+ }
+
+ outputLength=pOut-output;
+ if(inputLength!=outputLength) {
+ fprintf(stderr, "error: roundtrip failed, inputLength %d!=outputLength %d\n", inputLength, outputLength);
+ *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
+ }
+}
+
+static void
+noop(UConverter *cnv, int32_t intermediateCapacity, UErrorCode *pErrorCode) {
+ /* do nothing */
+}
+
+static unsigned long
+measureRoundtrips(RoundtripFn *fn, UConverter *cnv, const char *encName, int32_t intermediateCapacity, int32_t n) {
+ unsigned long _time;
+ UErrorCode errorCode;
+
+ _time=timeGetTime();
+ errorCode=U_ZERO_ERROR;
+ do {
+ fn(cnv, intermediateCapacity, &errorCode);
+ } while(U_SUCCESS(errorCode) && --n>0);
+ _time=timeGetTime()-_time;
+
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error in roundtrip conversion (%s): %s\n", encName, u_errorName(errorCode));
+ return 0x7fffffff;
+ }
+
+ if(0!=u_memcmp(input, output, inputLength)) {
+ fprintf(stderr, "error: roundtrip failed, input[]!=output[]\n");
+ return 0x7fffffff;
+ }
+
+ return _time;
+}
+
+static void
+perEncAndCapacity(UConverter *cnv, const char *encName, int32_t intermediateCapacity) {
+ double rtTime;
+ unsigned long _time;
+ int32_t n;
+
+ /*printf("test performance for %s with intermediate capacity %d\n", encName, intermediateCapacity);*/
+
+ /* warm up caches and estimate loop time */
+ n=10;
+ for(;;) {
+ _time=measureRoundtrips(roundtrip, cnv, encName, intermediateCapacity, n);
+ if(_time<500 && _time<TARGET_MEASURE_TIME_MS/10) {
+ n*=10;
+ } else {
+ break;
+ }
+ }
+
+ if(_time<TARGET_MEASURE_TIME_MS) {
+ n=(n*TARGET_MEASURE_TIME_MS)/_time+1;
+ }
+
+ /* run actual measurement with a target test time of 10s */
+ _time=measureRoundtrips(roundtrip, cnv, encName, intermediateCapacity, n);
+
+ /* subtract same number of loops over no-operation function */
+ _time-=measureRoundtrips(noop, cnv, encName, intermediateCapacity, n);
+
+ rtTime=((double)_time*1000.)/(double)n;
+
+ /* report */
+ printf("* performance report for %8s:\n", encName);
+ printf(" intermediate buffer capacity %8d B\n", intermediateCapacity);
+ if(intermediateCapacity==INTERMEDIATE_CAPACITY && utf8Length!=0) {
+ printf(" number of encoding bytes %8d B (%3d%% of UTF-8)\n", encodedLength, PERCENT(encodedLength, utf8Length));
+ printf(" roundtrip conversion time %8g μs (%3d%% of UTF-8)\n", rtTime, PERCENT(rtTime, utf8Time));
+ } else {
+ printf(" number of encoding bytes %8d B\n", encodedLength);
+ printf(" roundtrip conversion time %8g μs\n", rtTime);
+ }
+ printf(" average bytes/code point %8g B/cp\n", (double)encodedLength/countInputCodePoints);
+ puts("");
+
+ /* set UTF-8 values */
+ if(intermediateCapacity==INTERMEDIATE_CAPACITY && 0==strcmp(encName, "UTF-8")) {
+ utf8Length=encodedLength;
+ utf8Time=rtTime;
+ }
+}
+
+static void
+perEnc(UConverter *cnv, const char *encName) {
+ /*printf("test performance for %s\n", encName);*/
+ perEncAndCapacity(cnv, encName, INTERMEDIATE_CAPACITY);
+ perEncAndCapacity(cnv, encName, INTERMEDIATE_SMALL_CAPACITY);
+}
+
+static void
+testPerformance() {
+ UConverter *cnv;
+ UErrorCode errorCode;
+ int32_t i;
+
+ printf("number of code points %8d cp\n", countInputCodePoints);
+ printf("platform endianness: %8s-endian\n", U_IS_BIG_ENDIAN ? "big" : "little");
+ puts("");
+ for(i=0; i<ARRAY_LENGTH(utfNames); ++i) {
+ errorCode=U_ZERO_ERROR;
+ cnv=ucnv_open(utfNames[i], &errorCode);
+ if(U_SUCCESS(errorCode)) {
+ perEnc(cnv, utfNames[i]);
+ ucnv_close(cnv);
+ } else {
+ fprintf(stderr, "error opening converter for \"%s\" - %s\n", utfNames[i], u_errorName(errorCode));
+ }
+ }
+}
+
+/* read a complete block from the input file */
+static int32_t
+readBlock(FILE *in) {
+ int length, blockLength;
+
+ blockLength=0;
+ while(blockLength<INTERMEDIATE_CAPACITY && !feof(in)) {
+ length=fread(intermediate, 1, INTERMEDIATE_CAPACITY-blockLength, in);
+ if(length<0 || ferror(in)) {
+ return -1;
+ }
+ blockLength+=length;
+ }
+
+ return (int32_t)blockLength;
+}
+
+static UBool
+readInput(FILE *in, const char *encName) {
+ UConverter *cnv;
+ UChar *pOut, *pOutLimit;
+ const char *p, *limit;
+ int32_t length;
+ UErrorCode errorCode;
+
+ pOut=input;
+ pOutLimit=input+INPUT_CAPACITY;
+
+ errorCode=U_ZERO_ERROR;
+
+ /* read the first block and open the converter */
+ length=readBlock(in);
+ if(length<0) {
+ return FALSE;
+ }
+
+ if(encName==NULL) {
+ int32_t signatureLength;
+ encName=ucnv_detectUnicodeSignature(intermediate, length,
+ &signatureLength,
+ &errorCode);
+ if(U_FAILURE(errorCode) || encName==NULL) {
+ /* default to UTF-8 */
+ printf("no Unicode signature - using UTF-8\n");
+ encName="UTF-8";
+ errorCode=U_ZERO_ERROR;
+ } else {
+ printf("detected signature for %s (removing %d bytes)\n", encName, signatureLength);
+ /* remove signature byte sequence */
+ memmove(intermediate, intermediate+signatureLength, length-=signatureLength);
+ }
+ }
+
+ cnv=ucnv_open(encName, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: unable to ucnv_open(\"%s\") - %s\n", encName, u_errorName(errorCode));
+ return FALSE;
+ }
+
+ while(length>0) {
+ /* convert the block */
+ p=intermediate;
+ limit=p+length;
+
+ ucnv_toUnicode(cnv,
+ &pOut, pOutLimit,
+ &p, limit,
+ NULL, FALSE,
+ &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error converting input to UTF-16: %s\n", u_errorName(errorCode));
+ ucnv_close(cnv);
+ return FALSE;
+ }
+
+ /* read the next block */
+ length=readBlock(in);
+ if(length<0) {
+ ucnv_close(cnv);
+ return FALSE;
+ }
+ }
+
+ /* flush the converter */
+ ucnv_toUnicode(cnv,
+ &pOut, pOutLimit,
+ &p, p,
+ NULL, TRUE,
+ &errorCode);
+ ucnv_close(cnv);
+
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error converting input to UTF-16: %s\n", u_errorName(errorCode));
+ return FALSE;
+ }
+
+ inputLength=(int32_t)(pOut-input);
+ countInputCodePoints=u_countChar32(input, inputLength);
+ if(inputLength<=0) {
+ fprintf(stderr, "warning: input is empty\n");
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static void
+showUsage(const char *myName) {
+ fprintf(stderr,
+ "Usage:\n"
+ "%s [-e encoding-name] filename | '-'\n"
+ " encoding-name must be the name of an encoding supported by ICU\n"
+ " the filename of the input file with text to be used\n"
+ " can be a dash (-) for standard input\n",
+ myName);
+}
+
+/*
+ * Read file using some encoding, convert to 1M UTF-16 input buffer.
+ * For each UTF to be tested:
+ * n times:
+ * convert from UTF-16 input buffer to UTF, 4kB buffer
+ * convert from 4kB buffer to 1M UTF-16 output buffer
+ * adjust n so that time elapsed is 10s (#define)
+ * ->divide 10s by time, increase n by that factor, run 2nd time
+ * n times:
+ * empty function
+ * subtract out loop/function overhead
+ * display #code points - #UTF bytes - time per roundtrip
+ *
+ * * do the same again with an intermediate buffer size of 20 instead of 4kB
+ *
+ * Test following UTFs:
+ * UTF-16BE, UTF-16LE, UTF-8, SCSU, BOCU-1, CESU-8
+ *
+ * Command-line arguments:
+ * - encoding (default UTF-8, detect BOM)
+ * - filename (allow "-")
+ */
+extern int
+main(int argc, const char *argv[]) {
+ FILE *in;
+ const char *myName, *encName, *filename, *basename;
+
+ myName=argv[0];
+ if(argc<2) {
+ showUsage(myName);
+ return 1;
+ }
+
+ /* get encoding name argument */
+ if(argv[1][0]=='-' && argv[1][1]=='e') {
+ encName=argv[1]+2;
+ --argc;
+ ++argv;
+ if(*encName==0) {
+ if(argc<2) {
+ showUsage(myName);
+ return 1;
+ }
+ encName=argv[1];
+ --argc;
+ ++argv;
+ }
+ } else {
+ encName=NULL;
+ }
+
+ /* get filename argument */
+ if(argc<2) {
+ showUsage(myName);
+ return 1;
+ }
+ filename=argv[1];
+ if(filename[0]=='-' && filename[1]==0) {
+ filename="(standard input)";
+ in=stdin;
+ /* set stdin to binary mode */
+ _setmode(_fileno(stdin), _O_BINARY);
+ } else {
+ in=fopen(filename, "rb");
+ if(in==NULL) {
+ fprintf(stderr, "error opening \"%s\"\n", filename);
+ showUsage(myName);
+ return 2;
+ }
+ }
+
+ /* read input */
+ basename=strrchr(filename, U_FILE_SEP_CHAR);
+ if(basename!=NULL) {
+ ++basename;
+ } else {
+ basename=filename;
+ }
+ printf("# testing converter performance with file \"%s\"\n", basename);
+ if(!readInput(in, encName)) {
+ fprintf(stderr, "error reading \"%s\" (encoding %s)\n", filename, encName);
+ showUsage(myName);
+ return 2;
+ }
+ if(in!=stdin) {
+ fclose(in);
+ }
+
+ /* test performance */
+ testPerformance();
+ return 0;
+}
projects / apple / icu.git / blobdiff