X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/b75a7d8f3b4adbae880cab104ce2c6a50eee4db2..729e4ab9bc6618bc3d8a898e575df7f4019e29ca:/icuSources/tools/genrb/genrb.c?ds=sidebyside diff --git a/icuSources/tools/genrb/genrb.c b/icuSources/tools/genrb/genrb.c index 827e3302..4cf47e0e 100644 --- a/icuSources/tools/genrb/genrb.c +++ b/icuSources/tools/genrb/genrb.c @@ -1,7 +1,7 @@ /* ******************************************************************************* * -* Copyright (C) 1998-2003, International Business Machines +* Copyright (C) 1998-2010, International Business Machines * Corporation and others. All Rights Reserved. * ******************************************************************************* @@ -17,17 +17,20 @@ */ #include "genrb.h" +#include "unicode/uclean.h" + +#include "ucmndata.h" /* TODO: for reading the pool bundle */ /* Protos */ -static void processFile(const char *filename, const char* cp, const char *inputDir, const char *outputDir, const char *packageName, UErrorCode *status); -static char *make_res_filename(const char *filename, const char *outputDir, +void processFile(const char *filename, const char* cp, const char *inputDir, const char *outputDir, const char *packageName, UErrorCode *status); +static char *make_res_filename(const char *filename, const char *outputDir, const char *packageName, UErrorCode *status); /* File suffixes */ #define RES_SUFFIX ".res" #define COL_SUFFIX ".col" -static char theCurrentFileName[4096]; +static char theCurrentFileName[2048]; const char *gCurrentFileName = theCurrentFileName; #ifdef XP_MAC_CONSOLE #include @@ -46,14 +49,18 @@ enum ICUDATADIR, WRITE_JAVA, COPYRIGHT, - PACKAGE_NAME, + /* PACKAGE_NAME, This option is deprecated and should not be used ever. */ BUNDLE_NAME, - WRITE_XML, - TOUCHFILE, + WRITE_XLIFF, STRICT, NO_BINARY_COLLATION, /*added by Jing*/ - LANGUAGE + LANGUAGE, + NO_COLLATION_RULES, + FORMAT_VERSION, + WRITE_POOL_BUNDLE, + USE_POOL_BUNDLE, + INCLUDE_UNIHAN_COLL }; UOption options[]={ @@ -68,25 +75,42 @@ UOption options[]={ UOPTION_ICUDATADIR, UOPTION_WRITE_JAVA, UOPTION_COPYRIGHT, - UOPTION_PACKAGE_NAME, + /* UOPTION_PACKAGE_NAME, This option is deprecated and should not be used ever. */ UOPTION_BUNDLE_NAME, - UOPTION_DEF( "write-xml", 'x', UOPT_NO_ARG), - UOPTION_DEF( "touchfile", 't', UOPT_NO_ARG), - UOPTION_DEF( "strict", 'k', UOPT_NO_ARG), /* 14 */ - UOPTION_DEF( "noBinaryCollation", 'C', UOPT_NO_ARG),/* 15 */ - /*added by Jing*/ - UOPTION_DEF( "language", 'l', UOPT_REQUIRES_ARG) + UOPTION_DEF("write-xliff", 'x', UOPT_OPTIONAL_ARG), + UOPTION_DEF("strict", 'k', UOPT_NO_ARG), /* 14 */ + UOPTION_DEF("noBinaryCollation", 'C', UOPT_NO_ARG),/* 15 */ + UOPTION_DEF("language", 'l', UOPT_REQUIRES_ARG), /* 16 */ + UOPTION_DEF("omitCollationRules", 'R', UOPT_NO_ARG),/* 17 */ + UOPTION_DEF("formatVersion", '\x01', UOPT_REQUIRES_ARG),/* 18 */ + UOPTION_DEF("writePoolBundle", '\x01', UOPT_NO_ARG),/* 19 */ + UOPTION_DEF("usePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 20 */ + UOPTION_DEF("includeUnihanColl", '\x01', UOPT_NO_ARG),/* 21 */ /* temporary, don't display in usage info */ }; static UBool write_java = FALSE; -static UBool write_xml = FALSE; -static UBool touchfile = FALSE; +static UBool write_xliff = FALSE; static const char* outputEnc =""; static const char* gPackageName=NULL; static const char* bundleName=NULL; +static struct SRBRoot *newPoolBundle = NULL; + UBool gIncludeUnihanColl = FALSE; + +/* TODO: separate header file for ResFile? */ +typedef struct ResFile { + uint8_t *fBytes; + const int32_t *fIndexes; + const char *fKeys; + int32_t fKeysLength; + int32_t fKeysCount; + int32_t fChecksum; +} ResFile; + +static ResFile poolBundle = { NULL }; + /*added by Jing*/ static const char* language = NULL; - +static const char* xliffOutputFileName = NULL; int main(int argc, char* argv[]) @@ -97,7 +121,7 @@ main(int argc, const char *inputDir = NULL; const char *encoding = ""; int i; - + U_MAIN_INIT_ARGS(argc, argv); argc = u_parseArgs(argc, argv, (int32_t)(sizeof(options)/sizeof(options[0])), options); @@ -108,6 +132,24 @@ main(int argc, } else if(argc<2) { argc = -1; } + if(options[WRITE_POOL_BUNDLE].doesOccur && options[USE_POOL_BUNDLE].doesOccur) { + fprintf(stderr, "%s: cannot combine --writePoolBundle and --usePoolBundle\n", argv[0]); + argc = -1; + } + if(options[FORMAT_VERSION].doesOccur) { + const char *s = options[FORMAT_VERSION].value; + if(uprv_strlen(s) != 1 || (s[0] != '1' && s[0] != '2')) { + fprintf(stderr, "%s: unsupported --formatVersion %s\n", argv[0], s); + argc = -1; + } else if(s[0] == '1' && + (options[WRITE_POOL_BUNDLE].doesOccur || options[USE_POOL_BUNDLE].doesOccur) + ) { + fprintf(stderr, "%s: cannot combine --formatVersion 1 with --writePoolBundle or --usePoolBundle\n", argv[0]); + argc = -1; + } else { + setFormatVersion(s[0] - '0'); + } + } if(options[VERSION].doesOccur) { fprintf(stderr, @@ -119,7 +161,7 @@ main(int argc, if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) { /* - * Broken into chucks because the C89 standard says the minimum + * Broken into chunks because the C89 standard says the minimum * required supported string length is 509 bytes. */ fprintf(stderr, @@ -143,18 +185,33 @@ main(int argc, u_getDataDirectory(), u_getDataDirectory(), u_getDataDirectory()); fprintf(stderr, "\t-j or --write-java write a Java ListResourceBundle for ICU4J, followed by optional encoding\n" - "\t defaults to ASCII and \\uXXXX format.\n" + "\t defaults to ASCII and \\uXXXX format.\n"); + /* This option is deprecated and should not be used ever. "\t-p or --package-name For ICU4J: package name for writing the ListResourceBundle for ICU4J,\n" - "\t defaults to com.ibm.icu.impl.data\n" - "\t For ICU4C: Package name on output. Specfiying\n" - "\t 'ICUDATA' defaults to the current ICU4C data name.\n"); + "\t defaults to com.ibm.icu.impl.data\n"); */ fprintf(stderr, "\t-b or --bundle-name bundle name for writing the ListResourceBundle for ICU4J,\n" "\t defaults to LocaleElements\n" - "\t-x or --write-xml write a XML file for the resource bundle.\n" + "\t-x or --write-xliff write an XLIFF file for the resource bundle. Followed by\n" + "\t an optional output file name.\n" "\t-k or --strict use pedantic parsing of syntax\n" /*added by Jing*/ - "\t-l or --language language code compliant with ISO 639.\n"); + "\t-l or --language for XLIFF: language code compliant with BCP 47.\n"); + fprintf(stderr, + "\t-C or --noBinaryCollation do not generate binary collation image;\n" + "\t makes .res file smaller but collator instantiation much slower;\n" + "\t maintains ability to get tailoring rules\n" + "\t-R or --omitCollationRules do not include collation (tailoring) rules;\n" + "\t makes .res file smaller and maintains collator instantiation speed\n" + "\t but tailoring rules will not be available (they are rarely used)\n"); + fprintf(stderr, + "\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n" + "\t for example, --formatVersion 1\n"); + fprintf(stderr, + "\t --writePoolBundle write a pool.res file with all of the keys of all input bundles\n" + "\t --usePoolBundle [path-to-pool.res] point to keys from the pool.res keys pool bundle if they are available there;\n" + "\t makes .res files smaller but dependent on the pool bundle\n" + "\t (--writePoolBundle and --usePoolBundle cannot be combined)\n"); return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; } @@ -180,6 +237,7 @@ main(int argc, if(options[DESTDIR].doesOccur) { outputDir = options[DESTDIR].value; } + /* This option is deprecated and should never be used. if(options[PACKAGE_NAME].doesOccur) { gPackageName = options[PACKAGE_NAME].value; if(!strcmp(gPackageName, "ICUDATA")) @@ -190,16 +248,7 @@ main(int argc, { gPackageName = NULL; } - } - - if(options[TOUCHFILE].doesOccur) { - if(gPackageName == NULL) { - fprintf(stderr, "%s: Don't use touchfile (-t) option with no package.\n", - argv[0]); - return -1; - } - touchfile = TRUE; - } + }*/ if(options[ENCODING].doesOccur) { encoding = options[ENCODING].value; @@ -208,6 +257,18 @@ main(int argc, if(options[ICUDATADIR].doesOccur) { u_setDataDirectory(options[ICUDATADIR].value); } + /* Initialize ICU */ + u_init(&status); + if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) { + /* Note: u_init() will try to open ICU property data. + * failures here are expected when building ICU from scratch. + * ignore them. + */ + fprintf(stderr, "%s: can not initialize ICU. status = %s\n", + argv[0], u_errorName(status)); + exit(1); + } + status = U_ZERO_ERROR; if(options[WRITE_JAVA].doesOccur) { write_java = TRUE; outputEnc = options[WRITE_JAVA].value; @@ -217,26 +278,141 @@ main(int argc, bundleName = options[BUNDLE_NAME].value; } - if(options[WRITE_XML].doesOccur) { - write_xml = TRUE; + if(options[WRITE_XLIFF].doesOccur) { + write_xliff = TRUE; + if(options[WRITE_XLIFF].value != NULL){ + xliffOutputFileName = options[WRITE_XLIFF].value; + } } - if(options[NO_BINARY_COLLATION].doesOccur) { - initParser(FALSE); - } else { - initParser(TRUE); - } - + initParser(options[NO_BINARY_COLLATION].doesOccur, options[NO_COLLATION_RULES].doesOccur); + /*added by Jing*/ if(options[LANGUAGE].doesOccur) { language = options[LANGUAGE].value; } + if(options[WRITE_POOL_BUNDLE].doesOccur) { + newPoolBundle = bundle_open(NULL, TRUE, &status); + if(U_FAILURE(status)) { + fprintf(stderr, "unable to create an empty bundle for the pool keys: %s\n", u_errorName(status)); + return status; + } else { + const char *poolResName = "pool.res"; + char *nameWithoutSuffix = uprv_malloc(uprv_strlen(poolResName) + 1); + if (nameWithoutSuffix == NULL) { + fprintf(stderr, "out of memory error\n"); + return U_MEMORY_ALLOCATION_ERROR; + } + uprv_strcpy(nameWithoutSuffix, poolResName); + *uprv_strrchr(nameWithoutSuffix, '.') = 0; + newPoolBundle->fLocale = nameWithoutSuffix; + } + } + + if(options[USE_POOL_BUNDLE].doesOccur) { + const char *poolResName = "pool.res"; + FileStream *poolFile; + int32_t poolFileSize; + int32_t indexLength; + /* + * TODO: Consolidate inputDir/filename handling from main() and processFile() + * into a common function, and use it here as well. + * Try to create toolutil functions for dealing with dir/filenames and + * loading ICU data files without udata_open(). + * Share code with icupkg? + * Also, make_res_filename() seems to be unused. Review and remove. + */ + if (options[USE_POOL_BUNDLE].value!=NULL) { + uprv_strcpy(theCurrentFileName, options[USE_POOL_BUNDLE].value); + uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING); + } else if (inputDir) { + uprv_strcpy(theCurrentFileName, inputDir); + uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING); + } else { + *theCurrentFileName = 0; + } + uprv_strcat(theCurrentFileName, poolResName); + poolFile = T_FileStream_open(theCurrentFileName, "rb"); + if (poolFile == NULL) { + fprintf(stderr, "unable to open pool bundle file %s\n", theCurrentFileName); + return 1; + } + poolFileSize = T_FileStream_size(poolFile); + if (poolFileSize < 32) { + fprintf(stderr, "the pool bundle file %s is too small\n", theCurrentFileName); + return 1; + } + poolBundle.fBytes = (uint8_t *)uprv_malloc((poolFileSize + 15) & ~15); + if (poolFileSize > 0 && poolBundle.fBytes == NULL) { + fprintf(stderr, "unable to allocate memory for the pool bundle file %s\n", theCurrentFileName); + return U_MEMORY_ALLOCATION_ERROR; + } else { + UDataSwapper *ds; + const DataHeader *header; + int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize); + int32_t keysBottom; + if (bytesRead != poolFileSize) { + fprintf(stderr, "unable to read the pool bundle file %s\n", theCurrentFileName); + return 1; + } + /* + * Swap the pool bundle so that a single checked-in file can be used. + * The swapper functions also test that the data looks like + * a well-formed .res file. + */ + ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead, + U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status); + if (U_FAILURE(status)) { + fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n", + theCurrentFileName, u_errorName(status)); + return status; + } + ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status); + udata_closeSwapper(ds); + if (U_FAILURE(status)) { + fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n", + theCurrentFileName, u_errorName(status)); + return status; + } + header = (const DataHeader *)poolBundle.fBytes; + if (header->info.formatVersion[0]!=2) { + fprintf(stderr, "invalid format of pool bundle file %s\n", theCurrentFileName); + return U_INVALID_FORMAT_ERROR; + } + poolBundle.fKeys = (const char *)header + header->dataHeader.headerSize; + poolBundle.fIndexes = (const int32_t *)poolBundle.fKeys + 1; + indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff; + if (indexLength <= URES_INDEX_POOL_CHECKSUM) { + fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", theCurrentFileName); + return U_INVALID_FORMAT_ERROR; + } + keysBottom = (1 + indexLength) * 4; + poolBundle.fKeys += keysBottom; + poolBundle.fKeysLength = (poolBundle.fIndexes[URES_INDEX_KEYS_TOP] * 4) - keysBottom; + poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM]; + } + for (i = 0; i < poolBundle.fKeysLength; ++i) { + if (poolBundle.fKeys[i] == 0) { + ++poolBundle.fKeysCount; + } + } + T_FileStream_close(poolFile); + setUsePoolBundle(TRUE); + } + + if(options[INCLUDE_UNIHAN_COLL].doesOccur) { + gIncludeUnihanColl = TRUE; + } + + if((argc-1)!=1) { + printf("genrb number of files: %d\n", argc - 1); + } /* generate the binary files */ for(i = 1; i < argc; ++i) { status = U_ZERO_ERROR; arg = getLongPathname(argv[i]); - + if (inputDir) { uprv_strcpy(theCurrentFileName, inputDir); uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING); @@ -251,11 +427,27 @@ main(int argc, processFile(arg, encoding, inputDir, outputDir, gPackageName, &status); } + uprv_free(poolBundle.fBytes); + + if(options[WRITE_POOL_BUNDLE].doesOccur) { + char outputFileName[256]; + bundle_write(newPoolBundle, outputDir, NULL, outputFileName, sizeof(outputFileName), &status); + bundle_close(newPoolBundle, &status); + if(U_FAILURE(status)) { + fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status)); + } + } + + /* Dont return warnings as a failure */ + if (U_SUCCESS(status)) { + return 0; + } + return status; } /* Process a file */ -static void +void processFile(const char *filename, const char *cp, const char *inputDir, const char *outputDir, const char *packageName, UErrorCode *status) { /*FileStream *in = NULL;*/ struct SRBRoot *data = NULL; @@ -265,10 +457,11 @@ processFile(const char *filename, const char *cp, const char *inputDir, const ch char *inputDirBuf = NULL; char outputFileName[256]; - + int32_t dirlen = 0; int32_t filelen = 0; - + + if (status==NULL || U_FAILURE(*status)) { return; } @@ -284,12 +477,12 @@ processFile(const char *filename, const char *cp, const char *inputDir, const ch openFileName[0] = '\0'; if (filenameBegin != NULL) { /* - * When a filename ../../../data/root.txt is specified, + * When a filename ../../../data/root.txt is specified, * we presume that the input directory is ../../../data * This is very important when the resource file includes * another file, like UCARules.txt or thaidict.brk. */ - int32_t filenameSize = filenameBegin - filename + 1; + int32_t filenameSize = (int32_t)(filenameBegin - filename + 1); inputDirBuf = uprv_strncpy((char *)uprv_malloc(filenameSize), filename, filenameSize); /* test for NULL */ @@ -316,7 +509,7 @@ processFile(const char *filename, const char *cp, const char *inputDir, const ch openFileName[0] = '\0'; /* - * append the input dir to openFileName if the first char in + * append the input dir to openFileName if the first char in * filename is not file seperation char and the last char input directory is not '.'. * This is to support : * genrb -s. /home/icu/data @@ -341,16 +534,15 @@ processFile(const char *filename, const char *cp, const char *inputDir, const ch } uprv_strcpy(openFileName, inputDir); - + } } uprv_strcat(openFileName, filename); ucbuf = ucbuf_open(openFileName, &cp,getShowWarning(),TRUE, status); - if(*status == U_FILE_ACCESS_ERROR) { - + fprintf(stderr, "couldn't open file %s\n", openFileName == NULL ? filename : openFileName); goto finish; } @@ -359,58 +551,52 @@ processFile(const char *filename, const char *cp, const char *inputDir, const ch goto finish; } /* auto detected popular encodings? */ - if (cp!=NULL) { + if (cp!=NULL && isVerbose()) { printf("autodetected encoding %s\n", cp); } /* Parse the data into an SRBRoot */ - data = parse(ucbuf, inputDir, status); + data = parse(ucbuf, inputDir, outputDir, status); if (data == NULL || U_FAILURE(*status)) { fprintf(stderr, "couldn't parse the file %s. Error:%s\n", filename,u_errorName(*status)); goto finish; } - - /* Determine the target rb filename */ - rbname = make_res_filename(filename, outputDir, packageName, status); - if(touchfile == TRUE) { - FileStream *q; - char msg[1024]; - char *tfname = NULL; - - tfname = make_res_filename(filename, outputDir, NULL, status); - - if(U_FAILURE(*status)) - { - fprintf(stderr, "Error writing touchfile for \"%s\"\n", filename); - *status = U_FILE_ACCESS_ERROR; - } else { - uprv_strcat(tfname, ".res"); - sprintf(msg, "This empty file tells nmake that %s in package %s has been updated.\n", - filename, packageName); - - q = T_FileStream_open(tfname, "w"); - if(q == NULL) - { - fprintf(stderr, "Error writing touchfile \"%s\"\n", tfname); - *status = U_FILE_ACCESS_ERROR; - } - else - { - T_FileStream_write(q, msg, uprv_strlen(msg)); - T_FileStream_close(q); + if(options[WRITE_POOL_BUNDLE].doesOccur) { + int32_t newKeysLength; + const char *newKeys, *newKeysLimit; + bundle_compactKeys(data, status); + newKeys = bundle_getKeyBytes(data, &newKeysLength); + bundle_addKeyBytes(newPoolBundle, newKeys, newKeysLength, status); + if(U_FAILURE(*status)) { + fprintf(stderr, "bundle_compactKeys(%s) or bundle_getKeyBytes() failed: %s\n", + filename, u_errorName(*status)); + goto finish; + } + /* count the number of just-added key strings */ + for(newKeysLimit = newKeys + newKeysLength; newKeys < newKeysLimit; ++newKeys) { + if(*newKeys == 0) { + ++newPoolBundle->fKeysCount; } - uprv_free(tfname); } - } + + if(options[USE_POOL_BUNDLE].doesOccur) { + data->fPoolBundleKeys = poolBundle.fKeys; + data->fPoolBundleKeysLength = poolBundle.fKeysLength; + data->fPoolBundleKeysCount = poolBundle.fKeysCount; + data->fPoolChecksum = poolBundle.fChecksum; + } + + /* Determine the target rb filename */ + rbname = make_res_filename(filename, outputDir, packageName, status); if(U_FAILURE(*status)) { fprintf(stderr, "couldn't make the res fileName for bundle %s. Error:%s\n", filename,u_errorName(*status)); goto finish; } if(write_java== TRUE){ bundle_write_java(data,outputDir,outputEnc, outputFileName, sizeof(outputFileName),packageName,bundleName,status); - }else if(write_xml ==TRUE){ - bundle_write_xml(data,outputDir,outputEnc, filename, outputFileName, sizeof(outputFileName),language, packageName,status); + }else if(write_xliff ==TRUE){ + bundle_write_xml(data,outputDir,outputEnc, filename, outputFileName, sizeof(outputFileName),language, xliffOutputFileName,status); }else{ /* Write the data to the file */ bundle_write(data, outputDir, packageName, outputFileName, sizeof(outputFileName), status); @@ -457,7 +643,7 @@ make_res_filename(const char *filename, if(packageName != NULL) { - pkgLen = 1 + uprv_strlen(packageName); + pkgLen = (int32_t)(1 + uprv_strlen(packageName)); } /* setup */