icuSources/tools/icupkg/icupkg.cpp

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2005-2014, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  icupkg.cpp
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2005jul29
  14 *   created by: Markus W. Scherer
  15 *
  16 *   This tool operates on ICU data (.dat package) files.
  17 *   It takes one as input, or creates an empty one, and can remove, add, and
  18 *   extract data pieces according to command-line options.
  19 *   At the same time, it swaps each piece to a consistent set of platform
  20 *   properties as desired.
  21 *   Useful as an install-time tool for shipping only one flavor of ICU data
  22 *   and preparing data files for the target platform.
  23 *   Also for customizing ICU data (pruning, augmenting, replacing) and for
  24 *   taking it apart.
  25 *   Subsumes functionality and implementation code from
  26 *   gencmn, decmn, and icuswap tools.
  27 *   Will not work with data DLLs (shared libraries).
  28 */
  29
  30 #include "unicode/utypes.h"
  31 #include "unicode/putil.h"
  32 #include "cstring.h"
  33 #include "toolutil.h"
  34 #include "uoptions.h"
  35 #include "uparse.h"
  36 #include "filestrm.h"
  37 #include "package.h"
  38 #include "pkg_icu.h"
  39
  40 #include <stdio.h>
  41 #include <stdlib.h>
  42 #include <string.h>
  43
  44 U_NAMESPACE_USE
  45
  46 // TODO: add --matchmode=regex for using the ICU regex engine for item name pattern matching?
  47
  48 // general definitions ----------------------------------------------------- ***
  49
  50 // main() ------------------------------------------------------------------ ***
  51
  52 static void
  53 printUsage(const char *pname, UBool isHelp) {
  54     FILE *where=isHelp ? stdout : stderr;
  55
  56     fprintf(where,
  57             "%csage: %s [-h|-?|--help ] [-tl|-tb|-te] [-c] [-C comment]\n"
  58             "\t[-a list] [-r list] [-x list] [-l [-o outputListFileName]]\n"
  59             "\t[-s path] [-d path] [-w] [-m mode]\n"
  60             "\t[--auto_toc_prefix] [--auto_toc_prefix_with_type] [--toc_prefix]\n"
  61             "\tinfilename [outfilename]\n",
  62             isHelp ? 'U' : 'u', pname);
  63     if(isHelp) {
  64         fprintf(where,
  65             "\n"
  66             "Read the input ICU .dat package file, modify it according to the options,\n"
  67             "swap it to the desired platform properties (charset & endianness),\n"
  68             "and optionally write the resulting ICU .dat package to the output file.\n"
  69             "Items are removed, then added, then extracted and listed.\n"
  70             "An ICU .dat package is written if items are removed or added,\n"
  71             "or if the input and output filenames differ,\n"
  72             "or if the --writepkg (-w) option is set.\n");
  73         fprintf(where,
  74             "\n"
  75             "If the input filename is \"new\" then an empty package is created.\n"
  76             "If the output filename is missing, then it is automatically generated\n"
  77             "from the input filename: If the input filename ends with an l, b, or e\n"
  78             "matching its platform properties, then the output filename will\n"
  79             "contain the letter from the -t (--type) option.\n");
  80         fprintf(where,
  81             "\n"
  82             "This tool can also be used to just swap a single ICU data file, replacing the\n"
  83             "former icuswap tool. For this mode, provide the infilename (and optional\n"
  84             "outfilename) for a non-package ICU data file.\n"
  85             "Allowed options include -t, -w, -s and -d.\n"
  86             "The filenames can be absolute, or relative to the source/dest dir paths.\n"
  87             "Other options are not allowed in this mode.\n");
  88         fprintf(where,
  89             "\n"
  90             "Options:\n"
  91             "\t(Only the last occurrence of an option is used.)\n"
  92             "\n"
  93             "\t-h or -? or --help    print this message and exit\n");
  94         fprintf(where,
  95             "\n"
  96             "\t-tl or --type l   output for little-endian/ASCII charset family\n"
  97             "\t-tb or --type b   output for big-endian/ASCII charset family\n"
  98             "\t-te or --type e   output for big-endian/EBCDIC charset family\n"
  99             "\t                  The output type defaults to the input type.\n"
 100             "\n"
 101             "\t-c or --copyright include the ICU copyright notice\n"
 102             "\t-C comment or --comment comment   include a comment string\n");
 103         fprintf(where,
 104             "\n"
 105             "\t-a list or --add list      add items to the package\n"
 106             "\t-r list or --remove list   remove items from the package\n"
 107             "\t-x list or --extract list  extract items from the package\n"
 108             "\tThe list can be a single item's filename,\n"
 109             "\tor a .txt filename with a list of item filenames,\n"
 110             "\tor an ICU .dat package filename.\n");
 111         fprintf(where,
 112             "\n"
 113             "\t-w or --writepkg  write the output package even if no items are removed\n"
 114             "\t                  or added (e.g., for only swapping the data)\n");
 115         fprintf(where,
 116             "\n"
 117             "\t-m mode or --matchmode mode  set the matching mode for item names with\n"
 118             "\t                             wildcards\n"
 119             "\t        noslash: the '*' wildcard does not match the '/' tree separator\n");
 120         fprintf(where,
 121             "\n"
 122             "\tIn the .dat package, the Table of Contents (ToC) contains an entry\n"
 123             "\tfor each item of the form prefix/tree/itemname .\n"
 124             "\tThe prefix normally matches the package basename, and icupkg checks that,\n"
 125             "\tbut this is not necessary when ICU need not find and load the package by filename.\n"
 126             "\tICU package names end with the platform type letter, and thus differ\n"
 127             "\tbetween platform types. This is not required for user data packages.\n");
 128         fprintf(where,
 129             "\n"
 130             "\t--auto_toc_prefix            automatic ToC entries prefix\n"
 131             "\t                             Uses the prefix of the first entry of the\n"
 132             "\t                             input package, rather than its basename.\n"
 133             "\t                             Requires a non-empty input package.\n"
 134             "\t--auto_toc_prefix_with_type  auto_toc_prefix + adjust platform type\n"
 135             "\t                             Same as auto_toc_prefix but also checks that\n"
 136             "\t                             the prefix ends with the input platform\n"
 137             "\t                             type letter, and modifies it to the output\n"
 138             "\t                             platform type letter.\n"
 139             "\t                At most one of the auto_toc_prefix options\n"
 140             "\t                can be used at a time.\n"
 141             "\t--toc_prefix prefix          ToC prefix to be used in the output package\n"
 142             "\t                             Overrides the package basename\n"
 143             "\t                             and --auto_toc_prefix.\n"
 144             "\t                             Cannot be combined with --auto_toc_prefix_with_type.\n");
 145         /*
 146          * Usage text columns, starting after the initial TAB.
 147          *      1         2         3         4         5         6         7         8
 148          *     901234567890123456789012345678901234567890123456789012345678901234567890
 149          */
 150         fprintf(where,
 151             "\n"
 152             "\tList file syntax: Items are listed on one or more lines and separated\n"
 153             "\tby whitespace (space+tab).\n"
 154             "\tComments begin with # and are ignored. Empty lines are ignored.\n"
 155             "\tLines where the first non-whitespace character is one of %s\n"
 156             "\tare also ignored, to reserve for future syntax.\n",
 157             U_PKG_RESERVED_CHARS);
 158         fprintf(where,
 159             "\tItems for removal or extraction may contain a single '*' wildcard\n"
 160             "\tcharacter. The '*' matches zero or more characters.\n"
 161             "\tIf --matchmode noslash (-m noslash) is set, then the '*'\n"
 162             "\tdoes not match '/'.\n");
 163         fprintf(where,
 164             "\n"
 165             "\tItems must be listed relative to the package, and the --sourcedir or\n"
 166             "\tthe --destdir path will be prepended.\n"
 167             "\tThe paths are only prepended to item filenames while adding or\n"
 168             "\textracting items, not to ICU .dat package or list filenames.\n"
 169             "\t\n"
 170             "\tPaths may contain '/' instead of the platform's\n"
 171             "\tfile separator character, and are converted as appropriate.\n");
 172         fprintf(where,
 173             "\n"
 174             "\t-s path or --sourcedir path  directory for the --add items\n"
 175             "\t-d path or --destdir path    directory for the --extract items\n"
 176             "\n"
 177             "\t-l or --list                 list the package items\n"
 178             "\t                             (after modifying the package)\n"
 179             "\t                             to stdout or to output list file\n"
 180             "\t-o path or --outlist path    path/filename for the --list output\n");
 181     }
 182 }
 183
 184 static UOption options[]={
 185     UOPTION_HELP_H,
 186     UOPTION_HELP_QUESTION_MARK,
 187     UOPTION_DEF("type", 't', UOPT_REQUIRES_ARG),
 188
 189     UOPTION_COPYRIGHT,
 190     UOPTION_DEF("comment", 'C', UOPT_REQUIRES_ARG),
 191
 192     UOPTION_SOURCEDIR,
 193     UOPTION_DESTDIR,
 194
 195     UOPTION_DEF("writepkg", 'w', UOPT_NO_ARG),
 196
 197     UOPTION_DEF("matchmode", 'm', UOPT_REQUIRES_ARG),
 198
 199     UOPTION_DEF("add", 'a', UOPT_REQUIRES_ARG),
 200     UOPTION_DEF("remove", 'r', UOPT_REQUIRES_ARG),
 201     UOPTION_DEF("extract", 'x', UOPT_REQUIRES_ARG),
 202
 203     UOPTION_DEF("list", 'l', UOPT_NO_ARG),
 204     UOPTION_DEF("outlist", 'o', UOPT_REQUIRES_ARG),
 205
 206     UOPTION_DEF("auto_toc_prefix", '\1', UOPT_NO_ARG),
 207     UOPTION_DEF("auto_toc_prefix_with_type", '\1', UOPT_NO_ARG),
 208     UOPTION_DEF("toc_prefix", '\1', UOPT_REQUIRES_ARG)
 209 };
 210
 211 enum {
 212     OPT_HELP_H,
 213     OPT_HELP_QUESTION_MARK,
 214     OPT_OUT_TYPE,
 215
 216     OPT_COPYRIGHT,
 217     OPT_COMMENT,
 218
 219     OPT_SOURCEDIR,
 220     OPT_DESTDIR,
 221
 222     OPT_WRITEPKG,
 223
 224     OPT_MATCHMODE,
 225
 226     OPT_ADD_LIST,
 227     OPT_REMOVE_LIST,
 228     OPT_EXTRACT_LIST,
 229
 230     OPT_LIST_ITEMS,
 231     OPT_LIST_FILE,
 232
 233     OPT_AUTO_TOC_PREFIX,
 234     OPT_AUTO_TOC_PREFIX_WITH_TYPE,
 235     OPT_TOC_PREFIX,
 236
 237     OPT_COUNT
 238 };
 239
 240 static UBool
 241 isPackageName(const char *filename) {
 242     int32_t len;
 243
 244     len=(int32_t)strlen(filename)-4; /* -4: subtract the length of ".dat" */
 245     return (UBool)(len>0 && 0==strcmp(filename+len, ".dat"));
 246 }
 247 /*
 248 This line is required by MinGW because it incorrectly globs the arguments.
 249 So when \* is used, it turns into a list of files instead of a literal "*"
 250 */
 251 int _CRT_glob = 0;
 252
 253 extern int
 254 main(int argc, char *argv[]) {
 255     const char *pname, *sourcePath, *destPath, *inFilename, *outFilename, *outComment;
 256     char outType;
 257     UBool isHelp, isModified, isPackage;
 258     int result = 0;
 259
 260     Package *pkg, *listPkg, *addListPkg;
 261
 262     U_MAIN_INIT_ARGS(argc, argv);
 263
 264     /* get the program basename */
 265     pname=findBasename(argv[0]);
 266
 267     argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
 268     isHelp=options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur;
 269     if(isHelp) {
 270         printUsage(pname, TRUE);
 271         return U_ZERO_ERROR;
 272     }
 273
 274     pkg=new Package;
 275     if(pkg==NULL) {
 276         fprintf(stderr, "icupkg: not enough memory\n");
 277         return U_MEMORY_ALLOCATION_ERROR;
 278     }
 279     isModified=FALSE;
 280
 281     int autoPrefix=0;
 282     if(options[OPT_AUTO_TOC_PREFIX].doesOccur) {
 283         pkg->setAutoPrefix();
 284         ++autoPrefix;
 285     }
 286     if(options[OPT_AUTO_TOC_PREFIX_WITH_TYPE].doesOccur) {
 287         if(options[OPT_TOC_PREFIX].doesOccur) {
 288             fprintf(stderr, "icupkg: --auto_toc_prefix_with_type and also --toc_prefix\n");
 289             printUsage(pname, FALSE);
 290             return U_ILLEGAL_ARGUMENT_ERROR;
 291         }
 292         pkg->setAutoPrefixWithType();
 293         ++autoPrefix;
 294     }
 295     if(argc<2 || 3<argc || autoPrefix>1) {
 296         printUsage(pname, FALSE);
 297         return U_ILLEGAL_ARGUMENT_ERROR;
 298     }
 299
 300     if(options[OPT_SOURCEDIR].doesOccur) {
 301         sourcePath=options[OPT_SOURCEDIR].value;
 302     } else {
 303         // work relative to the current working directory
 304         sourcePath=NULL;
 305     }
 306     if(options[OPT_DESTDIR].doesOccur) {
 307         destPath=options[OPT_DESTDIR].value;
 308     } else {
 309         // work relative to the current working directory
 310         destPath=NULL;
 311     }
 312
 313     if(0==strcmp(argv[1], "new")) {
 314         if(autoPrefix) {
 315             fprintf(stderr, "icupkg: --auto_toc_prefix[_with_type] but no input package\n");
 316             printUsage(pname, FALSE);
 317             return U_ILLEGAL_ARGUMENT_ERROR;
 318         }
 319         inFilename=NULL;
 320         isPackage=TRUE;
 321     } else {
 322         inFilename=argv[1];
 323         if(isPackageName(inFilename)) {
 324             pkg->readPackage(inFilename);
 325             isPackage=TRUE;
 326         } else {
 327             /* swap a single file (icuswap replacement) rather than work on a package */
 328             pkg->addFile(sourcePath, inFilename);
 329             isPackage=FALSE;
 330         }
 331     }
 332
 333     if(argc>=3) {
 334         outFilename=argv[2];
 335         if(0!=strcmp(argv[1], argv[2])) {
 336             isModified=TRUE;
 337         }
 338     } else if(isPackage) {
 339         outFilename=NULL;
 340     } else /* !isPackage */ {
 341         outFilename=inFilename;
 342         isModified=(UBool)(sourcePath!=destPath);
 343     }
 344
 345     /* parse the output type option */
 346     if(options[OPT_OUT_TYPE].doesOccur) {
 347         const char *type=options[OPT_OUT_TYPE].value;
 348         if(type[0]==0 || type[1]!=0) {
 349             /* the type must be exactly one letter */
 350             printUsage(pname, FALSE);
 351             return U_ILLEGAL_ARGUMENT_ERROR;
 352         }
 353         outType=type[0];
 354         switch(outType) {
 355         case 'l':
 356         case 'b':
 357         case 'e':
 358             break;
 359         default:
 360             printUsage(pname, FALSE);
 361             return U_ILLEGAL_ARGUMENT_ERROR;
 362         }
 363
 364         /*
 365          * Set the isModified flag if the output type differs from the
 366          * input package type.
 367          * If we swap a single file, just assume that we are modifying it.
 368          * The Package class does not give us access to the item and its type.
 369          */
 370         isModified|=(UBool)(!isPackage || outType!=pkg->getInType());
 371     } else if(isPackage) {
 372         outType=pkg->getInType(); // default to input type
 373     } else /* !isPackage: swap single file */ {
 374         outType=0; /* tells extractItem() to not swap */
 375     }
 376
 377     if(options[OPT_WRITEPKG].doesOccur) {
 378         isModified=TRUE;
 379     }
 380
 381     if(!isPackage) {
 382         /*
 383          * icuswap tool replacement: Only swap a single file.
 384          * Check that irrelevant options are not set.
 385          */
 386         if( options[OPT_COMMENT].doesOccur ||
 387             options[OPT_COPYRIGHT].doesOccur ||
 388             options[OPT_MATCHMODE].doesOccur ||
 389             options[OPT_REMOVE_LIST].doesOccur ||
 390             options[OPT_ADD_LIST].doesOccur ||
 391             options[OPT_EXTRACT_LIST].doesOccur ||
 392             options[OPT_LIST_ITEMS].doesOccur
 393         ) {
 394             printUsage(pname, FALSE);
 395             return U_ILLEGAL_ARGUMENT_ERROR;
 396         }
 397         if(isModified) {
 398             pkg->extractItem(destPath, outFilename, 0, outType);
 399         }
 400
 401         delete pkg;
 402         return result;
 403     }
 404
 405     /* Work with a package. */
 406
 407     if(options[OPT_COMMENT].doesOccur) {
 408         outComment=options[OPT_COMMENT].value;
 409     } else if(options[OPT_COPYRIGHT].doesOccur) {
 410         outComment=U_COPYRIGHT_STRING;
 411     } else {
 412         outComment=NULL;
 413     }
 414
 415     if(options[OPT_MATCHMODE].doesOccur) {
 416         if(0==strcmp(options[OPT_MATCHMODE].value, "noslash")) {
 417             pkg->setMatchMode(Package::MATCH_NOSLASH);
 418         } else {
 419             printUsage(pname, FALSE);
 420             return U_ILLEGAL_ARGUMENT_ERROR;
 421         }
 422     }
 423
 424     /* remove items */
 425     if(options[OPT_REMOVE_LIST].doesOccur) {
 426         listPkg=new Package();
 427         if(listPkg==NULL) {
 428             fprintf(stderr, "icupkg: not enough memory\n");
 429             exit(U_MEMORY_ALLOCATION_ERROR);
 430         }
 431         if(readList(NULL, options[OPT_REMOVE_LIST].value, FALSE, listPkg)) {
 432             pkg->removeItems(*listPkg);
 433             delete listPkg;
 434             isModified=TRUE;
 435         } else {
 436             printUsage(pname, FALSE);
 437             return U_ILLEGAL_ARGUMENT_ERROR;
 438         }
 439     }
 440
 441     /*
 442      * add items
 443      * use a separate Package so that its memory and items stay around
 444      * as long as the main Package
 445      */
 446     addListPkg=NULL;
 447     if(options[OPT_ADD_LIST].doesOccur) {
 448         addListPkg=new Package();
 449         if(addListPkg==NULL) {
 450             fprintf(stderr, "icupkg: not enough memory\n");
 451             exit(U_MEMORY_ALLOCATION_ERROR);
 452         }
 453         if(readList(sourcePath, options[OPT_ADD_LIST].value, TRUE, addListPkg)) {
 454             pkg->addItems(*addListPkg);
 455             // delete addListPkg; deferred until after writePackage()
 456             isModified=TRUE;
 457         } else {
 458             printUsage(pname, FALSE);
 459             return U_ILLEGAL_ARGUMENT_ERROR;
 460         }
 461     }
 462
 463     /* extract items */
 464     if(options[OPT_EXTRACT_LIST].doesOccur) {
 465         listPkg=new Package();
 466         if(listPkg==NULL) {
 467             fprintf(stderr, "icupkg: not enough memory\n");
 468             exit(U_MEMORY_ALLOCATION_ERROR);
 469         }
 470         if(readList(NULL, options[OPT_EXTRACT_LIST].value, FALSE, listPkg)) {
 471             pkg->extractItems(destPath, *listPkg, outType);
 472             delete listPkg;
 473         } else {
 474             printUsage(pname, FALSE);
 475             return U_ILLEGAL_ARGUMENT_ERROR;
 476         }
 477     }
 478
 479     /* list items */
 480     if(options[OPT_LIST_ITEMS].doesOccur) {
 481         int32_t i;
 482         if (options[OPT_LIST_FILE].doesOccur) {
 483             FileStream *out;
 484             out = T_FileStream_open(options[OPT_LIST_FILE].value, "w");
 485             if (out != NULL) {
 486                 for(i=0; i<pkg->getItemCount(); ++i) {
 487                     T_FileStream_writeLine(out, pkg->getItem(i)->name);
 488                     T_FileStream_writeLine(out, "\n");
 489                 }
 490                 T_FileStream_close(out);
 491             } else {
 492                 return U_ILLEGAL_ARGUMENT_ERROR;
 493             }
 494         } else {
 495             for(i=0; i<pkg->getItemCount(); ++i) {
 496                 fprintf(stdout, "%s\n", pkg->getItem(i)->name);
 497             }
 498         }
 499     }
 500
 501     /* check dependencies between items */
 502     if(!pkg->checkDependencies()) {
 503         /* some dependencies are not fulfilled */
 504         return U_MISSING_RESOURCE_ERROR;
 505     }
 506
 507     /* write the output .dat package if there are any modifications */
 508     if(isModified) {
 509         char outFilenameBuffer[1024]; // for auto-generated output filename, if necessary
 510
 511         if(outFilename==NULL || outFilename[0]==0) {
 512             if(inFilename==NULL || inFilename[0]==0) {
 513                 fprintf(stderr, "icupkg: unable to auto-generate an output filename if there is no input filename\n");
 514                 exit(U_ILLEGAL_ARGUMENT_ERROR);
 515             }
 516
 517             /*
 518              * auto-generate a filename:
 519              * copy the inFilename,
 520              * and if the last basename character matches the input file's type,
 521              * then replace it with the output file's type
 522              */
 523             char suffix[6]="?.dat";
 524             char *s;
 525
 526             suffix[0]=pkg->getInType();
 527             strcpy(outFilenameBuffer, inFilename);
 528             s=strchr(outFilenameBuffer, 0);
 529             if((s-outFilenameBuffer)>5 && 0==memcmp(s-5, suffix, 5)) {
 530                 *(s-5)=outType;
 531             }
 532             outFilename=outFilenameBuffer;
 533         }
 534         if(options[OPT_TOC_PREFIX].doesOccur) {
 535             pkg->setPrefix(options[OPT_TOC_PREFIX].value);
 536         }
 537         result = writePackageDatFile(outFilename, outComment, NULL, NULL, pkg, outType);
 538     }
 539
 540     delete addListPkg;
 541     delete pkg;
 542     return result;
 543 }
 544
 545 /*
 546  * Hey, Emacs, please set the following:
 547  *
 548  * Local Variables:
 549  * indent-tabs-mode: nil
 550  * End:
 551  *
 552  */