icuSources/tools/gensprep/gensprep.c

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2003-2004, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  gensprep.c
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2003-02-06
  14 *   created by: Ram Viswanadha
  15 *
  16 *   This program reads the Profile.txt files,
  17 *   parses them, and extracts the data for StringPrep profile.
  18 *   It then preprocesses it and writes a binary file for efficient use
  19 *   in various StringPrep conversion processes.
  20 */
  21
  22 #define USPREP_TYPE_NAMES_ARRAY 1
  23
  24 #include <stdio.h>
  25 #include <stdlib.h>
  26
  27 #include "cmemory.h"
  28 #include "cstring.h"
  29 #include "unewdata.h"
  30 #include "uoptions.h"
  31 #include "uparse.h"
  32 #include "sprpimpl.h"
  33
  34 #include "unicode/udata.h"
  35 #include "unicode/utypes.h"
  36 #include "unicode/putil.h"
  37
  38
  39 U_CDECL_BEGIN
  40 #include "gensprep.h"
  41 U_CDECL_END
  42
  43 #ifdef WIN32
  44 #   pragma warning(disable: 4100)
  45 #endif
  46
  47 UBool beVerbose=FALSE, haveCopyright=TRUE;
  48
  49 #define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt"
  50
  51 /* prototypes --------------------------------------------------------------- */
  52
  53 static void
  54 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode);
  55
  56 static void
  57 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);
  58
  59
  60 /* -------------------------------------------------------------------------- */
  61
  62 static UOption options[]={
  63     UOPTION_HELP_H,
  64     UOPTION_HELP_QUESTION_MARK,
  65     UOPTION_VERBOSE,
  66     UOPTION_COPYRIGHT,
  67     UOPTION_DESTDIR,
  68     UOPTION_SOURCEDIR,
  69     UOPTION_ICUDATADIR,
  70     UOPTION_PACKAGE_NAME,
  71     UOPTION_BUNDLE_NAME,
  72     { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 },
  73     { "check-bidi", NULL, NULL, NULL,  'k', UOPT_NO_ARG, 0},
  74     { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
  75 };
  76
  77 enum{
  78     HELP,
  79     HELP_QUESTION_MARK,
  80     VERBOSE,
  81     COPYRIGHT,
  82     DESTDIR,
  83     SOURCEDIR,
  84     ICUDATADIR,
  85     PACKAGE_NAME,
  86     BUNDLE_NAME,
  87     NORMALIZE,
  88     CHECK_BIDI,
  89     UNICODE_VERSION
  90 };
  91
  92 static int printHelp(int argc, char* argv[]){
  93     /*
  94      * Broken into chucks because the C89 standard says the minimum
  95      * required supported string length is 509 bytes.
  96      */
  97     fprintf(stderr,
  98         "Usage: %s [-options] [file_name]\n"
  99         "\n"
 100         "Read the files specified and\n"
 101         "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n"
 102         "\n",
 103         argv[0]);
 104     fprintf(stderr,
 105         "Options:\n"
 106         "\t-h or -? or --help       print this usage text\n"
 107         "\t-v or --verbose          verbose output\n"
 108         "\t-c or --copyright        include a copyright notice\n");
 109     fprintf(stderr,
 110         "\t-d or --destdir          destination directory, followed by the path\n"
 111         "\t-s or --sourcedir        source directory of ICU data, followed by the path\n"
 112         "\t-b or --bundle-name      generate the ouput data file with the name specified\n"
 113         "\t-i or --icudatadir       directory for locating any needed intermediate data files,\n"
 114         "\t                         followed by path, defaults to %s\n",
 115         u_getDataDirectory());
 116     fprintf(stderr,
 117         "\t-p or --package-name     prepend the output data file name with the package name specified\n"
 118         "\t-n or --normalize        turn on the option for normalization and include mappings\n"
 119         "\t                         from NormalizationCorrections.txt from the given path,\n"
 120         "\t                         e.g: /test/icu/source/data/unidata\n"
 121         "\t-k or --check-bidi       turn on the option for checking for BiDi in the profile\n"
 122         "\t-u or --unicode          version of Unicode to be used with this profile followed by the version\n"
 123         );
 124     return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
 125 }
 126
 127
 128 extern int
 129 main(int argc, char* argv[]) {
 130 #if !UCONFIG_NO_IDNA
 131     char* filename = NULL;
 132 #endif
 133     const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL;
 134     const char *packageName=NULL, *bundleName=NULL, *inputFileName = NULL;
 135     char *basename=NULL;
 136     int32_t sprepOptions = 0;
 137
 138     UErrorCode errorCode=U_ZERO_ERROR;
 139
 140     U_MAIN_INIT_ARGS(argc, argv);
 141
 142     /* preset then read command line options */
 143     options[DESTDIR].value=u_getDataDirectory();
 144     options[SOURCEDIR].value="";
 145     options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */
 146     options[BUNDLE_NAME].value = DATA_NAME;
 147     options[PACKAGE_NAME].value = NULL;
 148     options[NORMALIZE].value = "";
 149
 150     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
 151
 152     /* error handling, printing usage message */
 153     if(argc<0) {
 154         fprintf(stderr,
 155             "error in command line argument \"%s\"\n",
 156             argv[-argc]);
 157     }
 158     if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
 159         return printHelp(argc, argv);
 160
 161     }
 162
 163     /* get the options values */
 164     beVerbose=options[VERBOSE].doesOccur;
 165     haveCopyright=options[COPYRIGHT].doesOccur;
 166     srcDir=options[SOURCEDIR].value;
 167     destDir=options[DESTDIR].value;
 168     packageName = options[PACKAGE_NAME].value;
 169     bundleName = options[BUNDLE_NAME].value;
 170     icuUniDataDir = options[NORMALIZE].value;
 171
 172     if(argc<2) {
 173         /* print the help message */
 174         return printHelp(argc, argv);
 175     } else {
 176         inputFileName = argv[1];
 177     }
 178     if(!options[UNICODE_VERSION].doesOccur){
 179         return printHelp(argc, argv);
 180     }
 181     if(options[ICUDATADIR].doesOccur) {
 182         u_setDataDirectory(options[ICUDATADIR].value);
 183     }
 184 #if UCONFIG_NO_IDNA
 185
 186     fprintf(stderr,
 187         "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
 188         " because UCONFIG_NO_IDNA is set, \n"
 189         "see icu/source/common/unicode/uconfig.h\n");
 190     generateData(destDir, packageName, bundleName);
 191
 192 #else
 193
 194     setUnicodeVersion(options[UNICODE_VERSION].value);
 195     filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
 196
 197     /* prepare the filename beginning with the source dir */
 198     if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){
 199         filename[0] = '.';
 200         filename[1] = U_FILE_SEP_CHAR;
 201         uprv_strcpy(filename+2,srcDir);
 202     }else{
 203         uprv_strcpy(filename, srcDir);
 204     }
 205
 206     basename=filename+uprv_strlen(filename);
 207     if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
 208         *basename++=U_FILE_SEP_CHAR;
 209     }
 210
 211     /* initialize */
 212     init();
 213
 214     /* process the file */
 215     uprv_strcpy(basename,inputFileName);
 216     parseMappings(filename,FALSE, &errorCode);
 217     if(U_FAILURE(errorCode)) {
 218         fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode));
 219         return errorCode;
 220     }
 221
 222     if(options[NORMALIZE].doesOccur){
 223         /* set up directory for NormalizationCorrections.txt */
 224         uprv_strcpy(filename,icuUniDataDir);
 225         basename=filename+uprv_strlen(filename);
 226         if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
 227             *basename++=U_FILE_SEP_CHAR;
 228         }
 229
 230         *basename++=U_FILE_SEP_CHAR;
 231         uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME);
 232
 233         parseNormalizationCorrections(filename,&errorCode);
 234         if(U_FAILURE(errorCode)){
 235             fprintf(stderr,"Could not open file %s for reading \n", filename);
 236             return errorCode;
 237         }
 238         sprepOptions |= _SPREP_NORMALIZATION_ON;
 239     }
 240
 241     if(options[CHECK_BIDI].doesOccur){
 242         sprepOptions |= _SPREP_CHECK_BIDI_ON;
 243     }
 244
 245     setOptions(sprepOptions);
 246
 247     /* process parsed data */
 248     if(U_SUCCESS(errorCode)) {
 249         /* write the data file */
 250        generateData(destDir, packageName, bundleName);
 251
 252        cleanUpData();
 253     }
 254
 255     uprv_free(filename);
 256
 257 #endif
 258
 259     return errorCode;
 260 }
 261
 262 #if !UCONFIG_NO_IDNA
 263
 264 static void U_CALLCONV
 265 normalizationCorrectionsLineFn(void *context,
 266                     char *fields[][2], int32_t fieldCount,
 267                     UErrorCode *pErrorCode) {
 268     uint32_t mapping[40];
 269     char *end, *s;
 270     uint32_t code;
 271     int32_t length;
 272     UVersionInfo version;
 273     UVersionInfo thisVersion;
 274
 275     /* get the character code, field 0 */
 276     code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
 277     if(U_FAILURE(*pErrorCode)) {
 278         fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]);
 279         exit(*pErrorCode);
 280     }
 281     /* Original (erroneous) decomposition */
 282     s = fields[1][0];
 283
 284     /* parse the mapping string */
 285     length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
 286
 287     /* ignore corrected decomposition */
 288
 289     u_versionFromString(version,fields[3][0] );
 290     u_versionFromString(thisVersion, "3.2.0");
 291
 292
 293
 294     if(U_FAILURE(*pErrorCode)) {
 295         fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n",
 296                 (long)code, u_errorName(*pErrorCode));
 297         exit(*pErrorCode);
 298     }
 299
 300     /* store the mapping */
 301     if( version[0] > thisVersion[0] ||
 302         ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
 303         ){
 304         storeMapping(code,mapping, length, USPREP_MAP, pErrorCode);
 305     }
 306     setUnicodeVersionNC(version);
 307 }
 308
 309 static void
 310 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
 311     char *fields[4][2];
 312
 313     if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
 314         return;
 315     }
 316
 317     u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);
 318
 319     /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */
 320
 321     if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
 322         fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
 323         exit(*pErrorCode);
 324     }
 325 }
 326
 327 static void U_CALLCONV
 328 strprepProfileLineFn(void *context,
 329               char *fields[][2], int32_t fieldCount,
 330               UErrorCode *pErrorCode) {
 331     uint32_t mapping[40];
 332     char *end, *map;
 333     uint32_t code;
 334     int32_t length;
 335    /*UBool* mapWithNorm = (UBool*) context;*/
 336     const char* typeName;
 337     uint32_t rangeStart=0,rangeEnd =0;
 338     const char* filename = (const char*) context;
 339
 340     typeName = fields[2][0];
 341     map = fields[1][0];
 342
 343     if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
 344
 345         u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
 346         if(U_FAILURE(*pErrorCode)){
 347             fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
 348             return;
 349         }
 350
 351         /* store the range */
 352         storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode);
 353
 354     }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
 355
 356         u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
 357         if(U_FAILURE(*pErrorCode)){
 358             fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
 359             return;
 360         }
 361
 362         /* store the range */
 363         storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode);
 364
 365     }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
 366
 367         /* get the character code, field 0 */
 368         code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
 369         if(end<=fields[0][0] || end!=fields[0][1]) {
 370             fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]);
 371             *pErrorCode=U_PARSE_ERROR;
 372             exit(U_PARSE_ERROR);
 373         }
 374
 375         /* parse the mapping string */
 376         length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
 377
 378         /* store the mapping */
 379         storeMapping(code,mapping, length,USPREP_MAP, pErrorCode);
 380
 381     }else{
 382         *pErrorCode = U_INVALID_FORMAT_ERROR;
 383     }
 384
 385     if(U_FAILURE(*pErrorCode)) {
 386         fprintf(stderr, "gensprep error parsing  %s line %s at %s. Error: %s\n",filename,
 387                fields[0][0],fields[2][0],u_errorName(*pErrorCode));
 388         exit(*pErrorCode);
 389     }
 390
 391 }
 392
 393 static void
 394 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) {
 395     char *fields[3][2];
 396
 397     if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
 398         return;
 399     }
 400
 401     u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
 402
 403     /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
 404
 405     if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
 406         fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
 407         exit(*pErrorCode);
 408     }
 409 }
 410
 411
 412 #endif /* #if !UCONFIG_NO_IDNA */
 413
 414 /*
 415  * Hey, Emacs, please set the following:
 416  *
 417  * Local Variables:
 418  * indent-tabs-mode: nil
 419  * End:
 420  *
 421  */