icuSources/tools/icupkg/icupkg.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 2005-2014, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *   file name:  icupkg.cpp
  11 *   encoding:   UTF-8
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2005jul29
  16 *   created by: Markus W. Scherer
  17 *
  18 *   This tool operates on ICU data (.dat package) files.
  19 *   It takes one as input, or creates an empty one, and can remove, add, and
  20 *   extract data pieces according to command-line options.
  21 *   At the same time, it swaps each piece to a consistent set of platform
  22 *   properties as desired.
  23 *   Useful as an install-time tool for shipping only one flavor of ICU data
  24 *   and preparing data files for the target platform.
  25 *   Also for customizing ICU data (pruning, augmenting, replacing) and for
  26 *   taking it apart.
  27 *   Subsumes functionality and implementation code from
  28 *   gencmn, decmn, and icuswap tools.
  29 *   Will not work with data DLLs (shared libraries).
  30 */
  31
  32 #include "unicode/utypes.h"
  33 #include "unicode/putil.h"
  34 #include "cstring.h"
  35 #include "toolutil.h"
  36 #include "uoptions.h"
  37 #include "uparse.h"
  38 #include "filestrm.h"
  39 #include "package.h"
  40 #include "pkg_icu.h"
  41
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <string.h>
  45
  46 U_NAMESPACE_USE
  47
  48 // TODO: add --matchmode=regex for using the ICU regex engine for item name pattern matching?
  49
  50 // general definitions ----------------------------------------------------- ***
  51
  52 // main() ------------------------------------------------------------------ ***
  53
  54 static void
  55 printUsage(const char *pname, UBool isHelp) {
  56     FILE *where=isHelp ? stdout : stderr;
  57
  58     fprintf(where,
  59             "%csage: %s [-h|-?|--help ] [-tl|-tb|-te] [-c] [-C comment]\n"
  60             "\t[-a list] [-r list] [-x list] [-l [-o outputListFileName]]\n"
  61             "\t[-s path] [-d path] [-w] [-m mode]\n"
  62             "\t[--auto_toc_prefix] [--auto_toc_prefix_with_type] [--toc_prefix]\n"
  63             "\tinfilename [outfilename]\n",
  64             isHelp ? 'U' : 'u', pname);
  65     if(isHelp) {
  66         fprintf(where,
  67             "\n"
  68             "Read the input ICU .dat package file, modify it according to the options,\n"
  69             "swap it to the desired platform properties (charset & endianness),\n"
  70             "and optionally write the resulting ICU .dat package to the output file.\n"
  71             "Items are removed, then added, then extracted and listed.\n"
  72             "An ICU .dat package is written if items are removed or added,\n"
  73             "or if the input and output filenames differ,\n"
  74             "or if the --writepkg (-w) option is set.\n");
  75         fprintf(where,
  76             "\n"
  77             "If the input filename is \"new\" then an empty package is created.\n"
  78             "If the output filename is missing, then it is automatically generated\n"
  79             "from the input filename: If the input filename ends with an l, b, or e\n"
  80             "matching its platform properties, then the output filename will\n"
  81             "contain the letter from the -t (--type) option.\n");
  82         fprintf(where,
  83             "\n"
  84             "This tool can also be used to just swap a single ICU data file, replacing the\n"
  85             "former icuswap tool. For this mode, provide the infilename (and optional\n"
  86             "outfilename) for a non-package ICU data file.\n"
  87             "Allowed options include -t, -w, -s and -d.\n"
  88             "The filenames can be absolute, or relative to the source/dest dir paths.\n"
  89             "Other options are not allowed in this mode.\n");
  90         fprintf(where,
  91             "\n"
  92             "Options:\n"
  93             "\t(Only the last occurrence of an option is used.)\n"
  94             "\n"
  95             "\t-h or -? or --help    print this message and exit\n");
  96         fprintf(where,
  97             "\n"
  98             "\t-tl or --type l   output for little-endian/ASCII charset family\n"
  99             "\t-tb or --type b   output for big-endian/ASCII charset family\n"
 100             "\t-te or --type e   output for big-endian/EBCDIC charset family\n"
 101             "\t                  The output type defaults to the input type.\n"
 102             "\n"
 103             "\t-c or --copyright include the ICU copyright notice\n"
 104             "\t-C comment or --comment comment   include a comment string\n");
 105         fprintf(where,
 106             "\n"
 107             "\t-a list or --add list      add items to the package\n"
 108             "\t-r list or --remove list   remove items from the package\n"
 109             "\t-x list or --extract list  extract items from the package\n"
 110             "\tThe list can be a single item's filename,\n"
 111             "\tor a .txt filename with a list of item filenames,\n"
 112             "\tor an ICU .dat package filename.\n");
 113         fprintf(where,
 114             "\n"
 115             "\t-w or --writepkg  write the output package even if no items are removed\n"
 116             "\t                  or added (e.g., for only swapping the data)\n");
 117         fprintf(where,
 118             "\n"
 119             "\t-m mode or --matchmode mode  set the matching mode for item names with\n"
 120             "\t                             wildcards\n"
 121             "\t        noslash: the '*' wildcard does not match the '/' tree separator\n");
 122         fprintf(where,
 123             "\n"
 124             "\tIn the .dat package, the Table of Contents (ToC) contains an entry\n"
 125             "\tfor each item of the form prefix/tree/itemname .\n"
 126             "\tThe prefix normally matches the package basename, and icupkg checks that,\n"
 127             "\tbut this is not necessary when ICU need not find and load the package by filename.\n"
 128             "\tICU package names end with the platform type letter, and thus differ\n"
 129             "\tbetween platform types. This is not required for user data packages.\n");
 130         fprintf(where,
 131             "\n"
 132             "\t--auto_toc_prefix            automatic ToC entries prefix\n"
 133             "\t                             Uses the prefix of the first entry of the\n"
 134             "\t                             input package, rather than its basename.\n"
 135             "\t                             Requires a non-empty input package.\n"
 136             "\t--auto_toc_prefix_with_type  auto_toc_prefix + adjust platform type\n"
 137             "\t                             Same as auto_toc_prefix but also checks that\n"
 138             "\t                             the prefix ends with the input platform\n"
 139             "\t                             type letter, and modifies it to the output\n"
 140             "\t                             platform type letter.\n"
 141             "\t                At most one of the auto_toc_prefix options\n"
 142             "\t                can be used at a time.\n"
 143             "\t--toc_prefix prefix          ToC prefix to be used in the output package\n"
 144             "\t                             Overrides the package basename\n"
 145             "\t                             and --auto_toc_prefix.\n"
 146             "\t                             Cannot be combined with --auto_toc_prefix_with_type.\n");
 147         /*
 148          * Usage text columns, starting after the initial TAB.
 149          *      1         2         3         4         5         6         7         8
 150          *     901234567890123456789012345678901234567890123456789012345678901234567890
 151          */
 152         fprintf(where,
 153             "\n"
 154             "\tList file syntax: Items are listed on one or more lines and separated\n"
 155             "\tby whitespace (space+tab).\n"
 156             "\tComments begin with # and are ignored. Empty lines are ignored.\n"
 157             "\tLines where the first non-whitespace character is one of %s\n"
 158             "\tare also ignored, to reserve for future syntax.\n",
 159             U_PKG_RESERVED_CHARS);
 160         fprintf(where,
 161             "\tItems for removal or extraction may contain a single '*' wildcard\n"
 162             "\tcharacter. The '*' matches zero or more characters.\n"
 163             "\tIf --matchmode noslash (-m noslash) is set, then the '*'\n"
 164             "\tdoes not match '/'.\n");
 165         fprintf(where,
 166             "\n"
 167             "\tItems must be listed relative to the package, and the --sourcedir or\n"
 168             "\tthe --destdir path will be prepended.\n"
 169             "\tThe paths are only prepended to item filenames while adding or\n"
 170             "\textracting items, not to ICU .dat package or list filenames.\n"
 171             "\t\n"
 172             "\tPaths may contain '/' instead of the platform's\n"
 173             "\tfile separator character, and are converted as appropriate.\n");
 174         fprintf(where,
 175             "\n"
 176             "\t-s path or --sourcedir path  directory for the --add items\n"
 177             "\t-d path or --destdir path    directory for the --extract items\n"
 178             "\n"
 179             "\t-l or --list                 list the package items\n"
 180             "\t                             (after modifying the package)\n"
 181             "\t                             to stdout or to output list file\n"
 182             "\t-o path or --outlist path    path/filename for the --list output\n");
 183     }
 184 }
 185
 186 static UOption options[]={
 187     UOPTION_HELP_H,
 188     UOPTION_HELP_QUESTION_MARK,
 189     UOPTION_DEF("type", 't', UOPT_REQUIRES_ARG),
 190
 191     UOPTION_COPYRIGHT,
 192     UOPTION_DEF("comment", 'C', UOPT_REQUIRES_ARG),
 193
 194     UOPTION_SOURCEDIR,
 195     UOPTION_DESTDIR,
 196
 197     UOPTION_DEF("writepkg", 'w', UOPT_NO_ARG),
 198
 199     UOPTION_DEF("matchmode", 'm', UOPT_REQUIRES_ARG),
 200
 201     UOPTION_DEF("add", 'a', UOPT_REQUIRES_ARG),
 202     UOPTION_DEF("remove", 'r', UOPT_REQUIRES_ARG),
 203     UOPTION_DEF("extract", 'x', UOPT_REQUIRES_ARG),
 204
 205     UOPTION_DEF("list", 'l', UOPT_NO_ARG),
 206     UOPTION_DEF("outlist", 'o', UOPT_REQUIRES_ARG),
 207
 208     UOPTION_DEF("auto_toc_prefix", '\1', UOPT_NO_ARG),
 209     UOPTION_DEF("auto_toc_prefix_with_type", '\1', UOPT_NO_ARG),
 210     UOPTION_DEF("toc_prefix", '\1', UOPT_REQUIRES_ARG)
 211 };
 212
 213 enum {
 214     OPT_HELP_H,
 215     OPT_HELP_QUESTION_MARK,
 216     OPT_OUT_TYPE,
 217
 218     OPT_COPYRIGHT,
 219     OPT_COMMENT,
 220
 221     OPT_SOURCEDIR,
 222     OPT_DESTDIR,
 223
 224     OPT_WRITEPKG,
 225
 226     OPT_MATCHMODE,
 227
 228     OPT_ADD_LIST,
 229     OPT_REMOVE_LIST,
 230     OPT_EXTRACT_LIST,
 231
 232     OPT_LIST_ITEMS,
 233     OPT_LIST_FILE,
 234
 235     OPT_AUTO_TOC_PREFIX,
 236     OPT_AUTO_TOC_PREFIX_WITH_TYPE,
 237     OPT_TOC_PREFIX,
 238
 239     OPT_COUNT
 240 };
 241
 242 static UBool
 243 isPackageName(const char *filename) {
 244     int32_t len;
 245
 246     len=(int32_t)strlen(filename)-4; /* -4: subtract the length of ".dat" */
 247     return (UBool)(len>0 && 0==strcmp(filename+len, ".dat"));
 248 }
 249 /*
 250 This line is required by MinGW because it incorrectly globs the arguments.
 251 So when \* is used, it turns into a list of files instead of a literal "*"
 252 */
 253 int _CRT_glob = 0;
 254
 255 extern int
 256 main(int argc, char *argv[]) {
 257     const char *pname, *sourcePath, *destPath, *inFilename, *outFilename, *outComment;
 258     char outType;
 259     UBool isHelp, isModified, isPackage;
 260     int result = 0;
 261
 262     Package *pkg, *listPkg, *addListPkg;
 263
 264     U_MAIN_INIT_ARGS(argc, argv);
 265
 266     /* get the program basename */
 267     pname=findBasename(argv[0]);
 268
 269     argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
 270     isHelp=options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur;
 271     if(isHelp) {
 272         printUsage(pname, TRUE);
 273         return U_ZERO_ERROR;
 274     }
 275
 276     pkg=new Package;
 277     if(pkg==NULL) {
 278         fprintf(stderr, "icupkg: not enough memory\n");
 279         return U_MEMORY_ALLOCATION_ERROR;
 280     }
 281     isModified=FALSE;
 282
 283     int autoPrefix=0;
 284     if(options[OPT_AUTO_TOC_PREFIX].doesOccur) {
 285         pkg->setAutoPrefix();
 286         ++autoPrefix;
 287     }
 288     if(options[OPT_AUTO_TOC_PREFIX_WITH_TYPE].doesOccur) {
 289         if(options[OPT_TOC_PREFIX].doesOccur) {
 290             fprintf(stderr, "icupkg: --auto_toc_prefix_with_type and also --toc_prefix\n");
 291             printUsage(pname, FALSE);
 292             return U_ILLEGAL_ARGUMENT_ERROR;
 293         }
 294         pkg->setAutoPrefixWithType();
 295         ++autoPrefix;
 296     }
 297     if(argc<2 || 3<argc || autoPrefix>1) {
 298         printUsage(pname, FALSE);
 299         return U_ILLEGAL_ARGUMENT_ERROR;
 300     }
 301
 302     if(options[OPT_SOURCEDIR].doesOccur) {
 303         sourcePath=options[OPT_SOURCEDIR].value;
 304     } else {
 305         // work relative to the current working directory
 306         sourcePath=NULL;
 307     }
 308     if(options[OPT_DESTDIR].doesOccur) {
 309         destPath=options[OPT_DESTDIR].value;
 310     } else {
 311         // work relative to the current working directory
 312         destPath=NULL;
 313     }
 314
 315     if(0==strcmp(argv[1], "new")) {
 316         if(autoPrefix) {
 317             fprintf(stderr, "icupkg: --auto_toc_prefix[_with_type] but no input package\n");
 318             printUsage(pname, FALSE);
 319             return U_ILLEGAL_ARGUMENT_ERROR;
 320         }
 321         inFilename=NULL;
 322         isPackage=TRUE;
 323     } else {
 324         inFilename=argv[1];
 325         if(isPackageName(inFilename)) {
 326             pkg->readPackage(inFilename);
 327             isPackage=TRUE;
 328         } else {
 329             /* swap a single file (icuswap replacement) rather than work on a package */
 330             pkg->addFile(sourcePath, inFilename);
 331             isPackage=FALSE;
 332         }
 333     }
 334
 335     if(argc>=3) {
 336         outFilename=argv[2];
 337         if(0!=strcmp(argv[1], argv[2])) {
 338             isModified=TRUE;
 339         }
 340     } else if(isPackage) {
 341         outFilename=NULL;
 342     } else /* !isPackage */ {
 343         outFilename=inFilename;
 344         isModified=(UBool)(sourcePath!=destPath);
 345     }
 346
 347     /* parse the output type option */
 348     if(options[OPT_OUT_TYPE].doesOccur) {
 349         const char *type=options[OPT_OUT_TYPE].value;
 350         if(type[0]==0 || type[1]!=0) {
 351             /* the type must be exactly one letter */
 352             printUsage(pname, FALSE);
 353             return U_ILLEGAL_ARGUMENT_ERROR;
 354         }
 355         outType=type[0];
 356         switch(outType) {
 357         case 'l':
 358         case 'b':
 359         case 'e':
 360             break;
 361         default:
 362             printUsage(pname, FALSE);
 363             return U_ILLEGAL_ARGUMENT_ERROR;
 364         }
 365
 366         /*
 367          * Set the isModified flag if the output type differs from the
 368          * input package type.
 369          * If we swap a single file, just assume that we are modifying it.
 370          * The Package class does not give us access to the item and its type.
 371          */
 372         isModified|=(UBool)(!isPackage || outType!=pkg->getInType());
 373     } else if(isPackage) {
 374         outType=pkg->getInType(); // default to input type
 375     } else /* !isPackage: swap single file */ {
 376         outType=0; /* tells extractItem() to not swap */
 377     }
 378
 379     if(options[OPT_WRITEPKG].doesOccur) {
 380         isModified=TRUE;
 381     }
 382
 383     if(!isPackage) {
 384         /*
 385          * icuswap tool replacement: Only swap a single file.
 386          * Check that irrelevant options are not set.
 387          */
 388         if( options[OPT_COMMENT].doesOccur ||
 389             options[OPT_COPYRIGHT].doesOccur ||
 390             options[OPT_MATCHMODE].doesOccur ||
 391             options[OPT_REMOVE_LIST].doesOccur ||
 392             options[OPT_ADD_LIST].doesOccur ||
 393             options[OPT_EXTRACT_LIST].doesOccur ||
 394             options[OPT_LIST_ITEMS].doesOccur
 395         ) {
 396             printUsage(pname, FALSE);
 397             return U_ILLEGAL_ARGUMENT_ERROR;
 398         }
 399         if(isModified) {
 400             pkg->extractItem(destPath, outFilename, 0, outType);
 401         }
 402
 403         delete pkg;
 404         return result;
 405     }
 406
 407     /* Work with a package. */
 408
 409     if(options[OPT_COMMENT].doesOccur) {
 410         outComment=options[OPT_COMMENT].value;
 411     } else if(options[OPT_COPYRIGHT].doesOccur) {
 412         outComment=U_COPYRIGHT_STRING;
 413     } else {
 414         outComment=NULL;
 415     }
 416
 417     if(options[OPT_MATCHMODE].doesOccur) {
 418         if(0==strcmp(options[OPT_MATCHMODE].value, "noslash")) {
 419             pkg->setMatchMode(Package::MATCH_NOSLASH);
 420         } else {
 421             printUsage(pname, FALSE);
 422             return U_ILLEGAL_ARGUMENT_ERROR;
 423         }
 424     }
 425
 426     /* remove items */
 427     if(options[OPT_REMOVE_LIST].doesOccur) {
 428         listPkg=new Package();
 429         if(listPkg==NULL) {
 430             fprintf(stderr, "icupkg: not enough memory\n");
 431             exit(U_MEMORY_ALLOCATION_ERROR);
 432         }
 433         if(readList(NULL, options[OPT_REMOVE_LIST].value, FALSE, listPkg)) {
 434             pkg->removeItems(*listPkg);
 435             delete listPkg;
 436             isModified=TRUE;
 437         } else {
 438             printUsage(pname, FALSE);
 439             return U_ILLEGAL_ARGUMENT_ERROR;
 440         }
 441     }
 442
 443     /*
 444      * add items
 445      * use a separate Package so that its memory and items stay around
 446      * as long as the main Package
 447      */
 448     addListPkg=NULL;
 449     if(options[OPT_ADD_LIST].doesOccur) {
 450         addListPkg=new Package();
 451         if(addListPkg==NULL) {
 452             fprintf(stderr, "icupkg: not enough memory\n");
 453             exit(U_MEMORY_ALLOCATION_ERROR);
 454         }
 455         if(readList(sourcePath, options[OPT_ADD_LIST].value, TRUE, addListPkg)) {
 456             pkg->addItems(*addListPkg);
 457             // delete addListPkg; deferred until after writePackage()
 458             isModified=TRUE;
 459         } else {
 460             printUsage(pname, FALSE);
 461             return U_ILLEGAL_ARGUMENT_ERROR;
 462         }
 463     }
 464
 465     /* extract items */
 466     if(options[OPT_EXTRACT_LIST].doesOccur) {
 467         listPkg=new Package();
 468         if(listPkg==NULL) {
 469             fprintf(stderr, "icupkg: not enough memory\n");
 470             exit(U_MEMORY_ALLOCATION_ERROR);
 471         }
 472         if(readList(NULL, options[OPT_EXTRACT_LIST].value, FALSE, listPkg)) {
 473             pkg->extractItems(destPath, *listPkg, outType);
 474             delete listPkg;
 475         } else {
 476             printUsage(pname, FALSE);
 477             return U_ILLEGAL_ARGUMENT_ERROR;
 478         }
 479     }
 480
 481     /* list items */
 482     if(options[OPT_LIST_ITEMS].doesOccur) {
 483         int32_t i;
 484         if (options[OPT_LIST_FILE].doesOccur) {
 485             FileStream *out;
 486             out = T_FileStream_open(options[OPT_LIST_FILE].value, "w");
 487             if (out != NULL) {
 488                 for(i=0; i<pkg->getItemCount(); ++i) {
 489                     T_FileStream_writeLine(out, pkg->getItem(i)->name);
 490                     T_FileStream_writeLine(out, "\n");
 491                 }
 492                 T_FileStream_close(out);
 493             } else {
 494                 return U_ILLEGAL_ARGUMENT_ERROR;
 495             }
 496         } else {
 497             for(i=0; i<pkg->getItemCount(); ++i) {
 498                 fprintf(stdout, "%s\n", pkg->getItem(i)->name);
 499             }
 500         }
 501     }
 502
 503     /* check dependencies between items */
 504     if(!pkg->checkDependencies()) {
 505         /* some dependencies are not fulfilled */
 506         return U_MISSING_RESOURCE_ERROR;
 507     }
 508
 509     /* write the output .dat package if there are any modifications */
 510     if(isModified) {
 511         char outFilenameBuffer[1024]; // for auto-generated output filename, if necessary
 512
 513         if(outFilename==NULL || outFilename[0]==0) {
 514             if(inFilename==NULL || inFilename[0]==0) {
 515                 fprintf(stderr, "icupkg: unable to auto-generate an output filename if there is no input filename\n");
 516                 exit(U_ILLEGAL_ARGUMENT_ERROR);
 517             }
 518
 519             /*
 520              * auto-generate a filename:
 521              * copy the inFilename,
 522              * and if the last basename character matches the input file's type,
 523              * then replace it with the output file's type
 524              */
 525             char suffix[6]="?.dat";
 526             char *s;
 527
 528             suffix[0]=pkg->getInType();
 529             strcpy(outFilenameBuffer, inFilename);
 530             s=strchr(outFilenameBuffer, 0);
 531             if((s-outFilenameBuffer)>5 && 0==memcmp(s-5, suffix, 5)) {
 532                 *(s-5)=outType;
 533             }
 534             outFilename=outFilenameBuffer;
 535         }
 536         if(options[OPT_TOC_PREFIX].doesOccur) {
 537             pkg->setPrefix(options[OPT_TOC_PREFIX].value);
 538         }
 539         result = writePackageDatFile(outFilename, outComment, NULL, NULL, pkg, outType);
 540     }
 541
 542     delete addListPkg;
 543     delete pkg;
 544     return result;
 545 }
 546
 547 /*
 548  * Hey, Emacs, please set the following:
 549  *
 550  * Local Variables:
 551  * indent-tabs-mode: nil
 552  * End:
 553  *
 554  */