icuSources/tools/icupkg/icupkg.cpp

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2005-2007, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  icupkg.cpp
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2005jul29
  14 *   created by: Markus W. Scherer
  15 *
  16 *   This tool operates on ICU data (.dat package) files.
  17 *   It takes one as input, or creates an empty one, and can remove, add, and
  18 *   extract data pieces according to command-line options.
  19 *   At the same time, it swaps each piece to a consistent set of platform
  20 *   properties as desired.
  21 *   Useful as an install-time tool for shipping only one flavor of ICU data
  22 *   and preparing data files for the target platform.
  23 *   Also for customizing ICU data (pruning, augmenting, replacing) and for
  24 *   taking it apart.
  25 *   Subsumes functionality and implementation code from
  26 *   gencmn, decmn, and icuswap tools.
  27 *   Will not work with data DLLs (shared libraries).
  28 */
  29
  30 #include "unicode/utypes.h"
  31 #include "unicode/putil.h"
  32 #include "cstring.h"
  33 #include "toolutil.h"
  34 #include "uoptions.h"
  35 #include "uparse.h"
  36 #include "package.h"
  37
  38 #include <stdio.h>
  39 #include <stdlib.h>
  40 #include <string.h>
  41
  42 U_NAMESPACE_USE
  43
  44 // TODO: add --matchmode=regex for using the ICU regex engine for item name pattern matching?
  45
  46 // general definitions ----------------------------------------------------- ***
  47
  48 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  49
  50 // read a file list -------------------------------------------------------- ***
  51
  52 static const char *reservedChars="\"%&'()*+,-./:;<=>?_";
  53
  54 static const struct {
  55     const char *suffix;
  56     int32_t length;
  57 } listFileSuffixes[]={
  58     { ".txt", 4 },
  59     { ".lst", 4 },
  60     { ".tmp", 4 }
  61 };
  62
  63 /* check for multiple text file suffixes to see if this list name is a text file name */
  64 static UBool
  65 isListTextFile(const char *listname) {
  66     const char *listNameEnd=strchr(listname, 0);
  67     const char *suffix;
  68     int32_t i, length;
  69     for(i=0; i<LENGTHOF(listFileSuffixes); ++i) {
  70         suffix=listFileSuffixes[i].suffix;
  71         length=listFileSuffixes[i].length;
  72         if((listNameEnd-listname)>length && 0==memcmp(listNameEnd-length, suffix, length)) {
  73             return TRUE;
  74         }
  75     }
  76     return FALSE;
  77 }
  78
  79 /*
  80  * Read a file list.
  81  * If the listname ends with ".txt", then read the list file
  82  * (in the system/ invariant charset).
  83  * If the listname ends with ".dat", then read the ICU .dat package file.
  84  * Otherwise, read the file itself as a single-item list.
  85  */
  86 static Package *
  87 readList(const char *filesPath, const char *listname, UBool readContents) {
  88     Package *listPkg;
  89     FILE *file;
  90     const char *listNameEnd;
  91
  92     if(listname==NULL || listname[0]==0) {
  93         fprintf(stderr, "missing list file\n");
  94         return NULL;
  95     }
  96
  97     listPkg=new Package();
  98     if(listPkg==NULL) {
  99         fprintf(stderr, "icupkg: not enough memory\n");
 100         exit(U_MEMORY_ALLOCATION_ERROR);
 101     }
 102
 103     listNameEnd=strchr(listname, 0);
 104     if(isListTextFile(listname)) {
 105         // read the list file
 106         char line[1024];
 107         char *end;
 108         const char *start;
 109
 110         file=fopen(listname, "r");
 111         if(file==NULL) {
 112             fprintf(stderr, "icupkg: unable to open list file \"%s\"\n", listname);
 113             delete listPkg;
 114             exit(U_FILE_ACCESS_ERROR);
 115         }
 116
 117         while(fgets(line, sizeof(line), file)) {
 118             // remove comments
 119             end=strchr(line, '#');
 120             if(end!=NULL) {
 121                 *end=0;
 122             } else {
 123                 // remove trailing CR LF
 124                 end=strchr(line, 0);
 125                 while(line<end && (*(end-1)=='\r' || *(end-1)=='\n')) {
 126                     *--end=0;
 127                 }
 128             }
 129
 130             // check first non-whitespace character and
 131             // skip empty lines and
 132             // skip lines starting with reserved characters
 133             start=u_skipWhitespace(line);
 134             if(*start==0 || NULL!=strchr(reservedChars, *start)) {
 135                 continue;
 136             }
 137
 138             // take whitespace-separated items from the line
 139             for(;;) {
 140                 // find whitespace after the item or the end of the line
 141                 for(end=(char *)start; *end!=0 && *end!=' ' && *end!='\t'; ++end) {}
 142                 if(*end==0) {
 143                     // this item is the last one on the line
 144                     end=NULL;
 145                 } else {
 146                     // the item is terminated by whitespace, terminate it with NUL
 147                     *end=0;
 148                 }
 149                 if(readContents) {
 150                     listPkg->addFile(filesPath, start);
 151                 } else {
 152                     listPkg->addItem(start);
 153                 }
 154
 155                 // find the start of the next item or exit the loop
 156                 if(end==NULL || *(start=u_skipWhitespace(end+1))==0) {
 157                     break;
 158                 }
 159             }
 160         }
 161         fclose(file);
 162     } else if((listNameEnd-listname)>4 && 0==memcmp(listNameEnd-4, ".dat", 4)) {
 163         // read the ICU .dat package
 164         listPkg->readPackage(listname);
 165     } else {
 166         // list the single file itself
 167         if(readContents) {
 168             listPkg->addFile(filesPath, listname);
 169         } else {
 170             listPkg->addItem(listname);
 171         }
 172     }
 173
 174     return listPkg;
 175 }
 176
 177 // main() ------------------------------------------------------------------ ***
 178
 179 static void
 180 printUsage(const char *pname, UBool isHelp) {
 181     FILE *where=isHelp ? stdout : stderr;
 182
 183     fprintf(where,
 184             "%csage: %s [-h|-?|--help ] [-tl|-tb|-te] [-c] [-C comment]\n"
 185             "\t[-a list] [-r list] [-x list] [-l]\n"
 186             "\t[-s path] [-d path] [-w] [-m mode]\n"
 187             "\tinfilename [outfilename]\n",
 188             isHelp ? 'U' : 'u', pname);
 189     if(isHelp) {
 190         fprintf(where,
 191             "\n"
 192             "Read the input ICU .dat package file, modify it according to the options,\n"
 193             "swap it to the desired platform properties (charset & endianness),\n"
 194             "and optionally write the resulting ICU .dat package to the output file.\n"
 195             "Items are removed, then added, then extracted and listed.\n"
 196             "An ICU .dat package is written if items are removed or added,\n"
 197             "or if the input and output filenames differ,\n"
 198             "or if the --writepkg (-w) option is set.\n");
 199         fprintf(where,
 200             "\n"
 201             "If the input filename is \"new\" then an empty package is created.\n"
 202             "If the output filename is missing, then it is automatically generated\n"
 203             "from the input filename: If the input filename ends with an l, b, or e\n"
 204             "matching its platform properties, then the output filename will\n"
 205             "contain the letter from the -t (--type) option.\n");
 206         fprintf(where,
 207             "\n"
 208             "This tool can also be used to just swap a single ICU data file, replacing the\n"
 209             "former icuswap tool. For this mode, provide the infilename (and optional\n"
 210             "outfilename) for a non-package ICU data file.\n"
 211             "Allowed options include -t, -w, -s and -d.\n"
 212             "The filenames can be absolute, or relative to the source/dest dir paths.\n"
 213             "Other options are not allowed in this mode.\n");
 214         fprintf(where,
 215             "\n"
 216             "Options:\n"
 217             "\t(Only the last occurrence of an option is used.)\n"
 218             "\n"
 219             "\t-h or -? or --help    print this message and exit\n");
 220         fprintf(where,
 221             "\n"
 222             "\t-tl or --type l   output for little-endian/ASCII charset family\n"
 223             "\t-tb or --type b   output for big-endian/ASCII charset family\n"
 224             "\t-te or --type e   output for big-endian/EBCDIC charset family\n"
 225             "\t                  The output type defaults to the input type.\n"
 226             "\n"
 227             "\t-c or --copyright include the ICU copyright notice\n"
 228             "\t-C comment or --comment comment   include a comment string\n");
 229         fprintf(where,
 230             "\n"
 231             "\t-a list or --add list      add items to the package\n"
 232             "\t-r list or --remove list   remove items from the package\n"
 233             "\t-x list or --extract list  extract items from the package\n"
 234             "\tThe list can be a single item's filename,\n"
 235             "\tor a .txt filename with a list of item filenames,\n"
 236             "\tor an ICU .dat package filename.\n");
 237         fprintf(where,
 238             "\n"
 239             "\t-w or --writepkg  write the output package even if no items are removed\n"
 240             "\t                  or added (e.g., for only swapping the data)\n");
 241         fprintf(where,
 242             "\n"
 243             "\t-m mode or --matchmode mode  set the matching mode for item names with\n"
 244             "\t                             wildcards\n"
 245             "\t        noslash: the '*' wildcard does not match the '/' tree separator\n");
 246         /*
 247          * Usage text columns, starting after the initial TAB.
 248          *      1         2         3         4         5         6         7         8
 249          *     901234567890123456789012345678901234567890123456789012345678901234567890
 250          */
 251         fprintf(where,
 252             "\n"
 253             "\tList file syntax: Items are listed on one or more lines and separated\n"
 254             "\tby whitespace (space+tab).\n"
 255             "\tComments begin with # and are ignored. Empty lines are ignored.\n"
 256             "\tLines where the first non-whitespace character is one of %s\n"
 257             "\tare also ignored, to reserve for future syntax.\n",
 258             reservedChars);
 259         fprintf(where,
 260             "\tItems for removal or extraction may contain a single '*' wildcard\n"
 261             "\tcharacter. The '*' matches zero or more characters.\n"
 262             "\tIf --matchmode noslash (-m noslash) is set, then the '*'\n"
 263             "\tdoes not match '/'.\n");
 264         fprintf(where,
 265             "\n"
 266             "\tItems must be listed relative to the package, and the --sourcedir or\n"
 267             "\tthe --destdir path will be prepended.\n"
 268             "\tThe paths are only prepended to item filenames while adding or\n"
 269             "\textracting items, not to ICU .dat package or list filenames.\n"
 270             "\t\n"
 271             "\tPaths may contain '/' instead of the platform's\n"
 272             "\tfile separator character, and are converted as appropriate.\n");
 273         fprintf(where,
 274             "\n"
 275             "\t-s path or --sourcedir path  directory for the --add items\n"
 276             "\t-d path or --destdir path    directory for the --extract items\n"
 277             "\n"
 278             "\t-l or --list                 list the package items to stdout\n"
 279             "\t                             (after modifying the package)\n");
 280     }
 281 }
 282
 283 static UOption options[]={
 284     UOPTION_HELP_H,
 285     UOPTION_HELP_QUESTION_MARK,
 286     UOPTION_DEF("type", 't', UOPT_REQUIRES_ARG),
 287
 288     UOPTION_COPYRIGHT,
 289     UOPTION_DEF("comment", 'C', UOPT_REQUIRES_ARG),
 290
 291     UOPTION_SOURCEDIR,
 292     UOPTION_DESTDIR,
 293
 294     UOPTION_DEF("writepkg", 'w', UOPT_NO_ARG),
 295
 296     UOPTION_DEF("matchmode", 'm', UOPT_REQUIRES_ARG),
 297
 298     UOPTION_DEF("add", 'a', UOPT_REQUIRES_ARG),
 299     UOPTION_DEF("remove", 'r', UOPT_REQUIRES_ARG),
 300     UOPTION_DEF("extract", 'x', UOPT_REQUIRES_ARG),
 301
 302     UOPTION_DEF("list", 'l', UOPT_NO_ARG)
 303 };
 304
 305 enum {
 306     OPT_HELP_H,
 307     OPT_HELP_QUESTION_MARK,
 308     OPT_OUT_TYPE,
 309
 310     OPT_COPYRIGHT,
 311     OPT_COMMENT,
 312
 313     OPT_SOURCEDIR,
 314     OPT_DESTDIR,
 315
 316     OPT_WRITEPKG,
 317
 318     OPT_MATCHMODE,
 319
 320     OPT_ADD_LIST,
 321     OPT_REMOVE_LIST,
 322     OPT_EXTRACT_LIST,
 323
 324     OPT_LIST_ITEMS,
 325
 326     OPT_COUNT
 327 };
 328
 329 static UBool
 330 isPackageName(const char *filename) {
 331     int32_t len;
 332
 333     len=(int32_t)strlen(filename)-4; /* -4: subtract the length of ".dat" */
 334     return (UBool)(len>0 && 0==strcmp(filename+len, ".dat"));
 335 }
 336
 337 /*
 338 This line is required by MinGW because it incorrectly globs the arguments.
 339 So when \* is used, it turns into a list of files instead of a literal "*"
 340 */
 341 int _CRT_glob = 0;
 342
 343 extern int
 344 main(int argc, char *argv[]) {
 345     const char *pname, *sourcePath, *destPath, *inFilename, *outFilename, *outComment;
 346     char outType;
 347     UBool isHelp, isModified, isPackage;
 348
 349     Package *pkg, *listPkg, *addListPkg;
 350
 351     U_MAIN_INIT_ARGS(argc, argv);
 352
 353     /* get the program basename */
 354     pname=findBasename(argv[0]);
 355
 356     argc=u_parseArgs(argc, argv, LENGTHOF(options), options);
 357     isHelp=options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur;
 358     if(isHelp) {
 359         printUsage(pname, TRUE);
 360         return U_ZERO_ERROR;
 361     }
 362     if(argc<2 || 3<argc) {
 363         printUsage(pname, FALSE);
 364         return U_ILLEGAL_ARGUMENT_ERROR;
 365     }
 366
 367     pkg=new Package;
 368     if(pkg==NULL) {
 369         fprintf(stderr, "icupkg: not enough memory\n");
 370         return U_MEMORY_ALLOCATION_ERROR;
 371     }
 372     isModified=FALSE;
 373
 374     if(options[OPT_SOURCEDIR].doesOccur) {
 375         sourcePath=options[OPT_SOURCEDIR].value;
 376     } else {
 377         // work relative to the current working directory
 378         sourcePath=NULL;
 379     }
 380     if(options[OPT_DESTDIR].doesOccur) {
 381         destPath=options[OPT_DESTDIR].value;
 382     } else {
 383         // work relative to the current working directory
 384         destPath=NULL;
 385     }
 386
 387     if(0==strcmp(argv[1], "new")) {
 388         inFilename=NULL;
 389         isPackage=TRUE;
 390     } else {
 391         inFilename=argv[1];
 392         if(isPackageName(inFilename)) {
 393             pkg->readPackage(inFilename);
 394             isPackage=TRUE;
 395         } else {
 396             /* swap a single file (icuswap replacement) rather than work on a package */
 397             pkg->addFile(sourcePath, inFilename);
 398             isPackage=FALSE;
 399         }
 400     }
 401
 402     if(argc>=3) {
 403         outFilename=argv[2];
 404         if(0!=strcmp(argv[1], argv[2])) {
 405             isModified=TRUE;
 406         }
 407     } else if(isPackage) {
 408         outFilename=NULL;
 409     } else /* !isPackage */ {
 410         outFilename=inFilename;
 411         isModified=(UBool)(sourcePath!=destPath);
 412     }
 413
 414     /* parse the output type option */
 415     if(options[OPT_OUT_TYPE].doesOccur) {
 416         const char *type=options[OPT_OUT_TYPE].value;
 417         if(type[0]==0 || type[1]!=0) {
 418             /* the type must be exactly one letter */
 419             printUsage(pname, FALSE);
 420             return U_ILLEGAL_ARGUMENT_ERROR;
 421         }
 422         outType=type[0];
 423         switch(outType) {
 424         case 'l':
 425         case 'b':
 426         case 'e':
 427             break;
 428         default:
 429             printUsage(pname, FALSE);
 430             return U_ILLEGAL_ARGUMENT_ERROR;
 431         }
 432
 433         /*
 434          * Set the isModified flag if the output type differs from the
 435          * input package type.
 436          * If we swap a single file, just assume that we are modifying it.
 437          * The Package class does not give us access to the item and its type.
 438          */
 439         isModified=(UBool)(!isPackage || outType!=pkg->getInType());
 440     } else if(isPackage) {
 441         outType=pkg->getInType(); // default to input type
 442     } else /* !isPackage: swap single file */ {
 443         outType=0; /* tells extractItem() to not swap */
 444     }
 445
 446     if(options[OPT_WRITEPKG].doesOccur) {
 447         isModified=TRUE;
 448     }
 449
 450     if(!isPackage) {
 451         /*
 452          * icuswap tool replacement: Only swap a single file.
 453          * Check that irrelevant options are not set.
 454          */
 455         if( options[OPT_COMMENT].doesOccur ||
 456             options[OPT_COPYRIGHT].doesOccur ||
 457             options[OPT_MATCHMODE].doesOccur ||
 458             options[OPT_REMOVE_LIST].doesOccur ||
 459             options[OPT_ADD_LIST].doesOccur ||
 460             options[OPT_EXTRACT_LIST].doesOccur ||
 461             options[OPT_LIST_ITEMS].doesOccur
 462         ) {
 463             printUsage(pname, FALSE);
 464             return U_ILLEGAL_ARGUMENT_ERROR;
 465         }
 466         if(isModified) {
 467             pkg->extractItem(destPath, outFilename, 0, outType);
 468         }
 469
 470         delete pkg;
 471         return 0;
 472     }
 473
 474     /* Work with a package. */
 475
 476     if(options[OPT_COMMENT].doesOccur) {
 477         outComment=options[OPT_COMMENT].value;
 478     } else if(options[OPT_COPYRIGHT].doesOccur) {
 479         outComment=U_COPYRIGHT_STRING;
 480     } else {
 481         outComment=NULL;
 482     }
 483
 484     if(options[OPT_MATCHMODE].doesOccur) {
 485         if(0==strcmp(options[OPT_MATCHMODE].value, "noslash")) {
 486             pkg->setMatchMode(Package::MATCH_NOSLASH);
 487         } else {
 488             printUsage(pname, FALSE);
 489             return U_ILLEGAL_ARGUMENT_ERROR;
 490         }
 491     }
 492
 493     /* remove items */
 494     if(options[OPT_REMOVE_LIST].doesOccur) {
 495         listPkg=readList(NULL, options[OPT_REMOVE_LIST].value, FALSE);
 496         if(listPkg!=NULL) {
 497             pkg->removeItems(*listPkg);
 498             delete listPkg;
 499             isModified=TRUE;
 500         } else {
 501             printUsage(pname, FALSE);
 502             return U_ILLEGAL_ARGUMENT_ERROR;
 503         }
 504     }
 505
 506     /*
 507      * add items
 508      * use a separate Package so that its memory and items stay around
 509      * as long as the main Package
 510      */
 511     addListPkg=NULL;
 512     if(options[OPT_ADD_LIST].doesOccur) {
 513         addListPkg=readList(sourcePath, options[OPT_ADD_LIST].value, TRUE);
 514         if(addListPkg!=NULL) {
 515             pkg->addItems(*addListPkg);
 516             // delete addListPkg; deferred until after writePackage()
 517             isModified=TRUE;
 518         } else {
 519             printUsage(pname, FALSE);
 520             return U_ILLEGAL_ARGUMENT_ERROR;
 521         }
 522     }
 523
 524     /* extract items */
 525     if(options[OPT_EXTRACT_LIST].doesOccur) {
 526         listPkg=readList(NULL, options[OPT_EXTRACT_LIST].value, FALSE);
 527         if(listPkg!=NULL) {
 528             pkg->extractItems(destPath, *listPkg, outType);
 529             delete listPkg;
 530         } else {
 531             printUsage(pname, FALSE);
 532             return U_ILLEGAL_ARGUMENT_ERROR;
 533         }
 534     }
 535
 536     /* list items */
 537     if(options[OPT_LIST_ITEMS].doesOccur) {
 538         int32_t i;
 539
 540         for(i=0; i<pkg->getItemCount(); ++i) {
 541             fprintf(stdout, "%s\n", pkg->getItem(i)->name);
 542         }
 543     }
 544
 545     /* check dependencies between items */
 546     if(!pkg->checkDependencies()) {
 547         /* some dependencies are not fulfilled */
 548         return U_MISSING_RESOURCE_ERROR;
 549     }
 550
 551     /* write the output .dat package if there are any modifications */
 552     if(isModified) {
 553         char outFilenameBuffer[1024]; // for auto-generated output filename, if necessary
 554
 555         if(outFilename==NULL || outFilename[0]==0) {
 556             if(inFilename==NULL || inFilename[0]==0) {
 557                 fprintf(stderr, "icupkg: unable to auto-generate an output filename if there is no input filename\n");
 558                 exit(U_ILLEGAL_ARGUMENT_ERROR);
 559             }
 560
 561             /*
 562              * auto-generate a filename:
 563              * copy the inFilename,
 564              * and if the last basename character matches the input file's type,
 565              * then replace it with the output file's type
 566              */
 567             char suffix[6]="?.dat";
 568             char *s;
 569
 570             suffix[0]=pkg->getInType();
 571             strcpy(outFilenameBuffer, inFilename);
 572             s=strchr(outFilenameBuffer, 0);
 573             if((s-outFilenameBuffer)>5 && 0==memcmp(s-5, suffix, 5)) {
 574                 *(s-5)=outType;
 575             }
 576             outFilename=outFilenameBuffer;
 577         }
 578         pkg->writePackage(outFilename, outType, outComment);
 579     }
 580
 581     delete addListPkg;
 582     delete pkg;
 583     return 0;
 584 }
 585
 586 /*
 587  * Hey, Emacs, please set the following:
 588  *
 589  * Local Variables:
 590  * indent-tabs-mode: nil
 591  * End:
 592  *
 593  */