/*
*******************************************************************************
*
-* Copyright (C) 1998-2003, International Business Machines
+* Copyright (C) 1998-2010, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
*/
#include "genrb.h"
+#include "unicode/uclean.h"
+
+#include "ucmndata.h" /* TODO: for reading the pool bundle */
/* Protos */
-static void processFile(const char *filename, const char* cp, const char *inputDir, const char *outputDir, const char *packageName, UErrorCode *status);
-static char *make_res_filename(const char *filename, const char *outputDir,
+void processFile(const char *filename, const char* cp, const char *inputDir, const char *outputDir, const char *packageName, UErrorCode *status);
+static char *make_res_filename(const char *filename, const char *outputDir,
const char *packageName, UErrorCode *status);
/* File suffixes */
#define RES_SUFFIX ".res"
#define COL_SUFFIX ".col"
-static char theCurrentFileName[4096];
+static char theCurrentFileName[2048];
const char *gCurrentFileName = theCurrentFileName;
#ifdef XP_MAC_CONSOLE
#include <console.h>
ICUDATADIR,
WRITE_JAVA,
COPYRIGHT,
- PACKAGE_NAME,
+ /* PACKAGE_NAME, This option is deprecated and should not be used ever. */
BUNDLE_NAME,
- WRITE_XML,
- TOUCHFILE,
+ WRITE_XLIFF,
STRICT,
NO_BINARY_COLLATION,
/*added by Jing*/
- LANGUAGE
+ LANGUAGE,
+ NO_COLLATION_RULES,
+ FORMAT_VERSION,
+ WRITE_POOL_BUNDLE,
+ USE_POOL_BUNDLE,
+ INCLUDE_UNIHAN_COLL
};
UOption options[]={
UOPTION_ICUDATADIR,
UOPTION_WRITE_JAVA,
UOPTION_COPYRIGHT,
- UOPTION_PACKAGE_NAME,
+ /* UOPTION_PACKAGE_NAME, This option is deprecated and should not be used ever. */
UOPTION_BUNDLE_NAME,
- UOPTION_DEF( "write-xml", 'x', UOPT_NO_ARG),
- UOPTION_DEF( "touchfile", 't', UOPT_NO_ARG),
- UOPTION_DEF( "strict", 'k', UOPT_NO_ARG), /* 14 */
- UOPTION_DEF( "noBinaryCollation", 'C', UOPT_NO_ARG),/* 15 */
- /*added by Jing*/
- UOPTION_DEF( "language", 'l', UOPT_REQUIRES_ARG)
+ UOPTION_DEF("write-xliff", 'x', UOPT_OPTIONAL_ARG),
+ UOPTION_DEF("strict", 'k', UOPT_NO_ARG), /* 14 */
+ UOPTION_DEF("noBinaryCollation", 'C', UOPT_NO_ARG),/* 15 */
+ UOPTION_DEF("language", 'l', UOPT_REQUIRES_ARG), /* 16 */
+ UOPTION_DEF("omitCollationRules", 'R', UOPT_NO_ARG),/* 17 */
+ UOPTION_DEF("formatVersion", '\x01', UOPT_REQUIRES_ARG),/* 18 */
+ UOPTION_DEF("writePoolBundle", '\x01', UOPT_NO_ARG),/* 19 */
+ UOPTION_DEF("usePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 20 */
+ UOPTION_DEF("includeUnihanColl", '\x01', UOPT_NO_ARG),/* 21 */ /* temporary, don't display in usage info */
};
static UBool write_java = FALSE;
-static UBool write_xml = FALSE;
-static UBool touchfile = FALSE;
+static UBool write_xliff = FALSE;
static const char* outputEnc ="";
static const char* gPackageName=NULL;
static const char* bundleName=NULL;
+static struct SRBRoot *newPoolBundle = NULL;
+ UBool gIncludeUnihanColl = FALSE;
+
+/* TODO: separate header file for ResFile? */
+typedef struct ResFile {
+ uint8_t *fBytes;
+ const int32_t *fIndexes;
+ const char *fKeys;
+ int32_t fKeysLength;
+ int32_t fKeysCount;
+ int32_t fChecksum;
+} ResFile;
+
+static ResFile poolBundle = { NULL };
+
/*added by Jing*/
static const char* language = NULL;
-
+static const char* xliffOutputFileName = NULL;
int
main(int argc,
char* argv[])
const char *inputDir = NULL;
const char *encoding = "";
int i;
-
+
U_MAIN_INIT_ARGS(argc, argv);
argc = u_parseArgs(argc, argv, (int32_t)(sizeof(options)/sizeof(options[0])), options);
} else if(argc<2) {
argc = -1;
}
+ if(options[WRITE_POOL_BUNDLE].doesOccur && options[USE_POOL_BUNDLE].doesOccur) {
+ fprintf(stderr, "%s: cannot combine --writePoolBundle and --usePoolBundle\n", argv[0]);
+ argc = -1;
+ }
+ if(options[FORMAT_VERSION].doesOccur) {
+ const char *s = options[FORMAT_VERSION].value;
+ if(uprv_strlen(s) != 1 || (s[0] != '1' && s[0] != '2')) {
+ fprintf(stderr, "%s: unsupported --formatVersion %s\n", argv[0], s);
+ argc = -1;
+ } else if(s[0] == '1' &&
+ (options[WRITE_POOL_BUNDLE].doesOccur || options[USE_POOL_BUNDLE].doesOccur)
+ ) {
+ fprintf(stderr, "%s: cannot combine --formatVersion 1 with --writePoolBundle or --usePoolBundle\n", argv[0]);
+ argc = -1;
+ } else {
+ setFormatVersion(s[0] - '0');
+ }
+ }
if(options[VERSION].doesOccur) {
fprintf(stderr,
if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
/*
- * Broken into chucks because the C89 standard says the minimum
+ * Broken into chunks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
u_getDataDirectory(), u_getDataDirectory(), u_getDataDirectory());
fprintf(stderr,
"\t-j or --write-java write a Java ListResourceBundle for ICU4J, followed by optional encoding\n"
- "\t defaults to ASCII and \\uXXXX format.\n"
+ "\t defaults to ASCII and \\uXXXX format.\n");
+ /* This option is deprecated and should not be used ever.
"\t-p or --package-name For ICU4J: package name for writing the ListResourceBundle for ICU4J,\n"
- "\t defaults to com.ibm.icu.impl.data\n"
- "\t For ICU4C: Package name on output. Specfiying\n"
- "\t 'ICUDATA' defaults to the current ICU4C data name.\n");
+ "\t defaults to com.ibm.icu.impl.data\n"); */
fprintf(stderr,
"\t-b or --bundle-name bundle name for writing the ListResourceBundle for ICU4J,\n"
"\t defaults to LocaleElements\n"
- "\t-x or --write-xml write a XML file for the resource bundle.\n"
+ "\t-x or --write-xliff write an XLIFF file for the resource bundle. Followed by\n"
+ "\t an optional output file name.\n"
"\t-k or --strict use pedantic parsing of syntax\n"
/*added by Jing*/
- "\t-l or --language language code compliant with ISO 639.\n");
+ "\t-l or --language for XLIFF: language code compliant with BCP 47.\n");
+ fprintf(stderr,
+ "\t-C or --noBinaryCollation do not generate binary collation image;\n"
+ "\t makes .res file smaller but collator instantiation much slower;\n"
+ "\t maintains ability to get tailoring rules\n"
+ "\t-R or --omitCollationRules do not include collation (tailoring) rules;\n"
+ "\t makes .res file smaller and maintains collator instantiation speed\n"
+ "\t but tailoring rules will not be available (they are rarely used)\n");
+ fprintf(stderr,
+ "\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n"
+ "\t for example, --formatVersion 1\n");
+ fprintf(stderr,
+ "\t --writePoolBundle write a pool.res file with all of the keys of all input bundles\n"
+ "\t --usePoolBundle [path-to-pool.res] point to keys from the pool.res keys pool bundle if they are available there;\n"
+ "\t makes .res files smaller but dependent on the pool bundle\n"
+ "\t (--writePoolBundle and --usePoolBundle cannot be combined)\n");
return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
if(options[DESTDIR].doesOccur) {
outputDir = options[DESTDIR].value;
}
+ /* This option is deprecated and should never be used.
if(options[PACKAGE_NAME].doesOccur) {
gPackageName = options[PACKAGE_NAME].value;
if(!strcmp(gPackageName, "ICUDATA"))
{
gPackageName = NULL;
}
- }
-
- if(options[TOUCHFILE].doesOccur) {
- if(gPackageName == NULL) {
- fprintf(stderr, "%s: Don't use touchfile (-t) option with no package.\n",
- argv[0]);
- return -1;
- }
- touchfile = TRUE;
- }
+ }*/
if(options[ENCODING].doesOccur) {
encoding = options[ENCODING].value;
if(options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
+ /* Initialize ICU */
+ u_init(&status);
+ if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) {
+ /* Note: u_init() will try to open ICU property data.
+ * failures here are expected when building ICU from scratch.
+ * ignore them.
+ */
+ fprintf(stderr, "%s: can not initialize ICU. status = %s\n",
+ argv[0], u_errorName(status));
+ exit(1);
+ }
+ status = U_ZERO_ERROR;
if(options[WRITE_JAVA].doesOccur) {
write_java = TRUE;
outputEnc = options[WRITE_JAVA].value;
bundleName = options[BUNDLE_NAME].value;
}
- if(options[WRITE_XML].doesOccur) {
- write_xml = TRUE;
+ if(options[WRITE_XLIFF].doesOccur) {
+ write_xliff = TRUE;
+ if(options[WRITE_XLIFF].value != NULL){
+ xliffOutputFileName = options[WRITE_XLIFF].value;
+ }
}
- if(options[NO_BINARY_COLLATION].doesOccur) {
- initParser(FALSE);
- } else {
- initParser(TRUE);
- }
-
+ initParser(options[NO_BINARY_COLLATION].doesOccur, options[NO_COLLATION_RULES].doesOccur);
+
/*added by Jing*/
if(options[LANGUAGE].doesOccur) {
language = options[LANGUAGE].value;
}
+ if(options[WRITE_POOL_BUNDLE].doesOccur) {
+ newPoolBundle = bundle_open(NULL, TRUE, &status);
+ if(U_FAILURE(status)) {
+ fprintf(stderr, "unable to create an empty bundle for the pool keys: %s\n", u_errorName(status));
+ return status;
+ } else {
+ const char *poolResName = "pool.res";
+ char *nameWithoutSuffix = uprv_malloc(uprv_strlen(poolResName) + 1);
+ if (nameWithoutSuffix == NULL) {
+ fprintf(stderr, "out of memory error\n");
+ return U_MEMORY_ALLOCATION_ERROR;
+ }
+ uprv_strcpy(nameWithoutSuffix, poolResName);
+ *uprv_strrchr(nameWithoutSuffix, '.') = 0;
+ newPoolBundle->fLocale = nameWithoutSuffix;
+ }
+ }
+
+ if(options[USE_POOL_BUNDLE].doesOccur) {
+ const char *poolResName = "pool.res";
+ FileStream *poolFile;
+ int32_t poolFileSize;
+ int32_t indexLength;
+ /*
+ * TODO: Consolidate inputDir/filename handling from main() and processFile()
+ * into a common function, and use it here as well.
+ * Try to create toolutil functions for dealing with dir/filenames and
+ * loading ICU data files without udata_open().
+ * Share code with icupkg?
+ * Also, make_res_filename() seems to be unused. Review and remove.
+ */
+ if (options[USE_POOL_BUNDLE].value!=NULL) {
+ uprv_strcpy(theCurrentFileName, options[USE_POOL_BUNDLE].value);
+ uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
+ } else if (inputDir) {
+ uprv_strcpy(theCurrentFileName, inputDir);
+ uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
+ } else {
+ *theCurrentFileName = 0;
+ }
+ uprv_strcat(theCurrentFileName, poolResName);
+ poolFile = T_FileStream_open(theCurrentFileName, "rb");
+ if (poolFile == NULL) {
+ fprintf(stderr, "unable to open pool bundle file %s\n", theCurrentFileName);
+ return 1;
+ }
+ poolFileSize = T_FileStream_size(poolFile);
+ if (poolFileSize < 32) {
+ fprintf(stderr, "the pool bundle file %s is too small\n", theCurrentFileName);
+ return 1;
+ }
+ poolBundle.fBytes = (uint8_t *)uprv_malloc((poolFileSize + 15) & ~15);
+ if (poolFileSize > 0 && poolBundle.fBytes == NULL) {
+ fprintf(stderr, "unable to allocate memory for the pool bundle file %s\n", theCurrentFileName);
+ return U_MEMORY_ALLOCATION_ERROR;
+ } else {
+ UDataSwapper *ds;
+ const DataHeader *header;
+ int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize);
+ int32_t keysBottom;
+ if (bytesRead != poolFileSize) {
+ fprintf(stderr, "unable to read the pool bundle file %s\n", theCurrentFileName);
+ return 1;
+ }
+ /*
+ * Swap the pool bundle so that a single checked-in file can be used.
+ * The swapper functions also test that the data looks like
+ * a well-formed .res file.
+ */
+ ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead,
+ U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status);
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n",
+ theCurrentFileName, u_errorName(status));
+ return status;
+ }
+ ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status);
+ udata_closeSwapper(ds);
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n",
+ theCurrentFileName, u_errorName(status));
+ return status;
+ }
+ header = (const DataHeader *)poolBundle.fBytes;
+ if (header->info.formatVersion[0]!=2) {
+ fprintf(stderr, "invalid format of pool bundle file %s\n", theCurrentFileName);
+ return U_INVALID_FORMAT_ERROR;
+ }
+ poolBundle.fKeys = (const char *)header + header->dataHeader.headerSize;
+ poolBundle.fIndexes = (const int32_t *)poolBundle.fKeys + 1;
+ indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff;
+ if (indexLength <= URES_INDEX_POOL_CHECKSUM) {
+ fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", theCurrentFileName);
+ return U_INVALID_FORMAT_ERROR;
+ }
+ keysBottom = (1 + indexLength) * 4;
+ poolBundle.fKeys += keysBottom;
+ poolBundle.fKeysLength = (poolBundle.fIndexes[URES_INDEX_KEYS_TOP] * 4) - keysBottom;
+ poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM];
+ }
+ for (i = 0; i < poolBundle.fKeysLength; ++i) {
+ if (poolBundle.fKeys[i] == 0) {
+ ++poolBundle.fKeysCount;
+ }
+ }
+ T_FileStream_close(poolFile);
+ setUsePoolBundle(TRUE);
+ }
+
+ if(options[INCLUDE_UNIHAN_COLL].doesOccur) {
+ gIncludeUnihanColl = TRUE;
+ }
+
+ if((argc-1)!=1) {
+ printf("genrb number of files: %d\n", argc - 1);
+ }
/* generate the binary files */
for(i = 1; i < argc; ++i) {
status = U_ZERO_ERROR;
arg = getLongPathname(argv[i]);
-
+
if (inputDir) {
uprv_strcpy(theCurrentFileName, inputDir);
uprv_strcat(theCurrentFileName, U_FILE_SEP_STRING);
processFile(arg, encoding, inputDir, outputDir, gPackageName, &status);
}
+ uprv_free(poolBundle.fBytes);
+
+ if(options[WRITE_POOL_BUNDLE].doesOccur) {
+ char outputFileName[256];
+ bundle_write(newPoolBundle, outputDir, NULL, outputFileName, sizeof(outputFileName), &status);
+ bundle_close(newPoolBundle, &status);
+ if(U_FAILURE(status)) {
+ fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status));
+ }
+ }
+
+ /* Dont return warnings as a failure */
+ if (U_SUCCESS(status)) {
+ return 0;
+ }
+
return status;
}
/* Process a file */
-static void
+void
processFile(const char *filename, const char *cp, const char *inputDir, const char *outputDir, const char *packageName, UErrorCode *status) {
/*FileStream *in = NULL;*/
struct SRBRoot *data = NULL;
char *inputDirBuf = NULL;
char outputFileName[256];
-
+
int32_t dirlen = 0;
int32_t filelen = 0;
-
+
+
if (status==NULL || U_FAILURE(*status)) {
return;
}
openFileName[0] = '\0';
if (filenameBegin != NULL) {
/*
- * When a filename ../../../data/root.txt is specified,
+ * When a filename ../../../data/root.txt is specified,
* we presume that the input directory is ../../../data
* This is very important when the resource file includes
* another file, like UCARules.txt or thaidict.brk.
*/
- int32_t filenameSize = filenameBegin - filename + 1;
+ int32_t filenameSize = (int32_t)(filenameBegin - filename + 1);
inputDirBuf = uprv_strncpy((char *)uprv_malloc(filenameSize), filename, filenameSize);
/* test for NULL */
openFileName[0] = '\0';
/*
- * append the input dir to openFileName if the first char in
+ * append the input dir to openFileName if the first char in
* filename is not file seperation char and the last char input directory is not '.'.
* This is to support :
* genrb -s. /home/icu/data
}
uprv_strcpy(openFileName, inputDir);
-
+
}
}
uprv_strcat(openFileName, filename);
ucbuf = ucbuf_open(openFileName, &cp,getShowWarning(),TRUE, status);
-
if(*status == U_FILE_ACCESS_ERROR) {
-
+
fprintf(stderr, "couldn't open file %s\n", openFileName == NULL ? filename : openFileName);
goto finish;
}
goto finish;
}
/* auto detected popular encodings? */
- if (cp!=NULL) {
+ if (cp!=NULL && isVerbose()) {
printf("autodetected encoding %s\n", cp);
}
/* Parse the data into an SRBRoot */
- data = parse(ucbuf, inputDir, status);
+ data = parse(ucbuf, inputDir, outputDir, status);
if (data == NULL || U_FAILURE(*status)) {
fprintf(stderr, "couldn't parse the file %s. Error:%s\n", filename,u_errorName(*status));
goto finish;
}
-
- /* Determine the target rb filename */
- rbname = make_res_filename(filename, outputDir, packageName, status);
- if(touchfile == TRUE) {
- FileStream *q;
- char msg[1024];
- char *tfname = NULL;
-
- tfname = make_res_filename(filename, outputDir, NULL, status);
-
- if(U_FAILURE(*status))
- {
- fprintf(stderr, "Error writing touchfile for \"%s\"\n", filename);
- *status = U_FILE_ACCESS_ERROR;
- } else {
- uprv_strcat(tfname, ".res");
- sprintf(msg, "This empty file tells nmake that %s in package %s has been updated.\n",
- filename, packageName);
-
- q = T_FileStream_open(tfname, "w");
- if(q == NULL)
- {
- fprintf(stderr, "Error writing touchfile \"%s\"\n", tfname);
- *status = U_FILE_ACCESS_ERROR;
- }
- else
- {
- T_FileStream_write(q, msg, uprv_strlen(msg));
- T_FileStream_close(q);
+ if(options[WRITE_POOL_BUNDLE].doesOccur) {
+ int32_t newKeysLength;
+ const char *newKeys, *newKeysLimit;
+ bundle_compactKeys(data, status);
+ newKeys = bundle_getKeyBytes(data, &newKeysLength);
+ bundle_addKeyBytes(newPoolBundle, newKeys, newKeysLength, status);
+ if(U_FAILURE(*status)) {
+ fprintf(stderr, "bundle_compactKeys(%s) or bundle_getKeyBytes() failed: %s\n",
+ filename, u_errorName(*status));
+ goto finish;
+ }
+ /* count the number of just-added key strings */
+ for(newKeysLimit = newKeys + newKeysLength; newKeys < newKeysLimit; ++newKeys) {
+ if(*newKeys == 0) {
+ ++newPoolBundle->fKeysCount;
}
- uprv_free(tfname);
}
-
}
+
+ if(options[USE_POOL_BUNDLE].doesOccur) {
+ data->fPoolBundleKeys = poolBundle.fKeys;
+ data->fPoolBundleKeysLength = poolBundle.fKeysLength;
+ data->fPoolBundleKeysCount = poolBundle.fKeysCount;
+ data->fPoolChecksum = poolBundle.fChecksum;
+ }
+
+ /* Determine the target rb filename */
+ rbname = make_res_filename(filename, outputDir, packageName, status);
if(U_FAILURE(*status)) {
fprintf(stderr, "couldn't make the res fileName for bundle %s. Error:%s\n", filename,u_errorName(*status));
goto finish;
}
if(write_java== TRUE){
bundle_write_java(data,outputDir,outputEnc, outputFileName, sizeof(outputFileName),packageName,bundleName,status);
- }else if(write_xml ==TRUE){
- bundle_write_xml(data,outputDir,outputEnc, filename, outputFileName, sizeof(outputFileName),language, packageName,status);
+ }else if(write_xliff ==TRUE){
+ bundle_write_xml(data,outputDir,outputEnc, filename, outputFileName, sizeof(outputFileName),language, xliffOutputFileName,status);
}else{
/* Write the data to the file */
bundle_write(data, outputDir, packageName, outputFileName, sizeof(outputFileName), status);
if(packageName != NULL)
{
- pkgLen = 1 + uprv_strlen(packageName);
+ pkgLen = (int32_t)(1 + uprv_strlen(packageName));
}
/* setup */
projects / apple / icu.git / blobdiff