icuSources/tools/toolutil/package.cpp

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 1999-2006, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  package.cpp
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2005aug25
  14 *   created by: Markus W. Scherer
  15 *
  16 *   Read, modify, and write ICU .dat data package files.
  17 *   This is an integral part of the icupkg tool, moved to the toolutil library
  18 *   because parts of tool implementations tend to be later shared by
  19 *   other tools.
  20 *   Subsumes functionality and implementation code from
  21 *   gencmn, decmn, and icuswap tools.
  22 */
  23
  24 #include "unicode/utypes.h"
  25 #include "unicode/putil.h"
  26 #include "unicode/udata.h"
  27 #include "cstring.h"
  28 #include "uarrsort.h"
  29 #include "ucmndata.h"
  30 #include "udataswp.h"
  31 #include "swapimpl.h"
  32 #include "toolutil.h"
  33 #include "package.h"
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38
  39 // general definitions ----------------------------------------------------- ***
  40
  41 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  42
  43 /* UDataInfo cf. udata.h */
  44 static const UDataInfo dataInfo={
  45     (uint16_t)sizeof(UDataInfo),
  46     0,
  47
  48     U_IS_BIG_ENDIAN,
  49     U_CHARSET_FAMILY,
  50     (uint8_t)sizeof(UChar),
  51     0,
  52
  53     {0x43, 0x6d, 0x6e, 0x44},     /* dataFormat="CmnD" */
  54     {1, 0, 0, 0},                 /* formatVersion */
  55     {3, 0, 0, 0}                  /* dataVersion */
  56 };
  57
  58 U_CDECL_BEGIN
  59 static void U_CALLCONV
  60 printPackageError(void *context, const char *fmt, va_list args) {
  61     vfprintf((FILE *)context, fmt, args);
  62 }
  63 U_CDECL_END
  64
  65 static uint16_t
  66 readSwapUInt16(uint16_t x) {
  67     return (uint16_t)((x<<8)|(x>>8));
  68 }
  69
  70 // platform types ---------------------------------------------------------- ***
  71
  72 static const char *types="lb?e";
  73
  74 enum { TYPE_L, TYPE_B, TYPE_LE, TYPE_E, TYPE_COUNT };
  75
  76 static inline int32_t
  77 makeTypeEnum(uint8_t charset, UBool isBigEndian) {
  78     return 2*(int32_t)charset+isBigEndian;
  79 }
  80
  81 static inline int32_t
  82 makeTypeEnum(char type) {
  83     return
  84         type == 'l' ? TYPE_L :
  85         type == 'b' ? TYPE_B :
  86         type == 'e' ? TYPE_E :
  87                -1;
  88 }
  89
  90 static inline char
  91 makeTypeLetter(uint8_t charset, UBool isBigEndian) {
  92     return types[makeTypeEnum(charset, isBigEndian)];
  93 }
  94
  95 static inline char
  96 makeTypeLetter(int32_t typeEnum) {
  97     return types[typeEnum];
  98 }
  99
 100 static void
 101 makeTypeProps(char type, uint8_t &charset, UBool &isBigEndian) {
 102     int32_t typeEnum=makeTypeEnum(type);
 103     charset=(uint8_t)(typeEnum>>1);
 104     isBigEndian=(UBool)(typeEnum&1);
 105 }
 106
 107 U_CFUNC const UDataInfo *
 108 getDataInfo(const uint8_t *data, int32_t length,
 109             int32_t &infoLength, int32_t &headerLength,
 110             UErrorCode *pErrorCode) {
 111     const DataHeader *pHeader;
 112     const UDataInfo *pInfo;
 113
 114     if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
 115         return NULL;
 116     }
 117     if( data==NULL ||
 118         (length>=0 && length<(int32_t)sizeof(DataHeader))
 119     ) {
 120         *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
 121         return NULL;
 122     }
 123
 124     pHeader=(const DataHeader *)data;
 125     pInfo=&pHeader->info;
 126     if( (length>=0 && length<(int32_t)sizeof(DataHeader)) ||
 127         pHeader->dataHeader.magic1!=0xda ||
 128         pHeader->dataHeader.magic2!=0x27 ||
 129         pInfo->sizeofUChar!=2
 130     ) {
 131         *pErrorCode=U_UNSUPPORTED_ERROR;
 132         return NULL;
 133     }
 134
 135     if(pInfo->isBigEndian==U_IS_BIG_ENDIAN) {
 136         headerLength=pHeader->dataHeader.headerSize;
 137         infoLength=pInfo->size;
 138     } else {
 139         headerLength=readSwapUInt16(pHeader->dataHeader.headerSize);
 140         infoLength=readSwapUInt16(pInfo->size);
 141     }
 142
 143     if( headerLength<(int32_t)sizeof(DataHeader) ||
 144         infoLength<(int32_t)sizeof(UDataInfo) ||
 145         headerLength<(int32_t)(sizeof(pHeader->dataHeader)+infoLength) ||
 146         (length>=0 && length<headerLength)
 147     ) {
 148         *pErrorCode=U_UNSUPPORTED_ERROR;
 149         return NULL;
 150     }
 151
 152     return pInfo;
 153 }
 154
 155 static int32_t
 156 getTypeEnumForInputData(const uint8_t *data, int32_t length,
 157                         UErrorCode *pErrorCode) {
 158     const UDataInfo *pInfo;
 159     int32_t infoLength, headerLength;
 160
 161     /* getDataInfo() checks for illegal arguments */
 162     pInfo=getDataInfo(data, length, infoLength, headerLength, pErrorCode);
 163     if(pInfo==NULL) {
 164         return -1;
 165     }
 166
 167     return makeTypeEnum(pInfo->charsetFamily, (UBool)pInfo->isBigEndian);
 168 }
 169
 170 // file handling ----------------------------------------------------------- ***
 171
 172 static void
 173 extractPackageName(const char *filename,
 174                    char pkg[], int32_t capacity) {
 175     const char *basename;
 176     int32_t len;
 177
 178     basename=findBasename(filename);
 179     len=(int32_t)strlen(basename)-4; /* -4: subtract the length of ".dat" */
 180
 181     if(len<=0 || 0!=strcmp(basename+len, ".dat")) {
 182         fprintf(stderr, "icupkg: \"%s\" is not recognized as a package filename (must end with .dat)\n",
 183                          basename);
 184         exit(U_ILLEGAL_ARGUMENT_ERROR);
 185     }
 186
 187     if(len>=capacity) {
 188         fprintf(stderr, "icupkg: the package name \"%s\" is too long (>=%ld)\n",
 189                          basename, (long)capacity);
 190         exit(U_ILLEGAL_ARGUMENT_ERROR);
 191     }
 192
 193     memcpy(pkg, basename, len);
 194     pkg[len]=0;
 195 }
 196
 197 static int32_t
 198 getFileLength(FILE *f) {
 199     int32_t length;
 200
 201     fseek(f, 0, SEEK_END);
 202     length=(int32_t)ftell(f);
 203     fseek(f, 0, SEEK_SET);
 204     return length;
 205 }
 206
 207 /*
 208  * Turn tree separators and alternate file separators into normal file separators.
 209  */
 210 #if U_TREE_ENTRY_SEP_CHAR==U_FILE_SEP_CHAR && U_FILE_ALT_SEP_CHAR==U_FILE_SEP_CHAR
 211 #define treeToPath(s)
 212 #else
 213 static void
 214 treeToPath(char *s) {
 215     char *t;
 216
 217     for(t=s; *t!=0; ++t) {
 218         if(*t==U_TREE_ENTRY_SEP_CHAR || *t==U_FILE_ALT_SEP_CHAR) {
 219             *t=U_FILE_SEP_CHAR;
 220         }
 221     }
 222 }
 223 #endif
 224
 225 /*
 226  * Turn file separators into tree separators.
 227  */
 228 #if U_TREE_ENTRY_SEP_CHAR==U_FILE_SEP_CHAR && U_FILE_ALT_SEP_CHAR==U_FILE_SEP_CHAR
 229 #define pathToTree(s)
 230 #else
 231 static void
 232 pathToTree(char *s) {
 233     char *t;
 234
 235     for(t=s; *t!=0; ++t) {
 236         if(*t==U_FILE_SEP_CHAR || *t==U_FILE_ALT_SEP_CHAR) {
 237             *t=U_TREE_ENTRY_SEP_CHAR;
 238         }
 239     }
 240 }
 241 #endif
 242
 243 /*
 244  * Prepend the path (if any) to the name and run the name through treeToName().
 245  */
 246 static void
 247 makeFullFilename(const char *path, const char *name,
 248                  char *filename, int32_t capacity) {
 249     char *s;
 250
 251     // prepend the path unless NULL or empty
 252     if(path!=NULL && path[0]!=0) {
 253         if((int32_t)(strlen(path)+1)>=capacity) {
 254             fprintf(stderr, "pathname too long: \"%s\"\n", path);
 255             exit(U_BUFFER_OVERFLOW_ERROR);
 256         }
 257         strcpy(filename, path);
 258
 259         // make sure the path ends with a file separator
 260         s=strchr(filename, 0);
 261         if(*(s-1)!=U_FILE_SEP_CHAR && *(s-1)!=U_FILE_ALT_SEP_CHAR) {
 262             *s++=U_FILE_SEP_CHAR;
 263         }
 264     } else {
 265         s=filename;
 266     }
 267
 268     // turn the name into a filename, turn tree separators into file separators
 269     if((int32_t)((s-filename)+strlen(name))>=capacity) {
 270         fprintf(stderr, "path/filename too long: \"%s%s\"\n", filename, name);
 271         exit(U_BUFFER_OVERFLOW_ERROR);
 272     }
 273     strcpy(s, name);
 274     treeToPath(s);
 275 }
 276
 277 static void
 278 makeFullFilenameAndDirs(const char *path, const char *name,
 279                         char *filename, int32_t capacity) {
 280     char *sep;
 281     UErrorCode errorCode;
 282
 283     makeFullFilename(path, name, filename, capacity);
 284
 285     // make tree directories
 286     errorCode=U_ZERO_ERROR;
 287     sep=strchr(filename, 0)-strlen(name);
 288     while((sep=strchr(sep, U_FILE_SEP_CHAR))!=NULL) {
 289         if(sep!=filename) {
 290             *sep=0;                 // truncate temporarily
 291             uprv_mkdir(filename, &errorCode);
 292             if(U_FAILURE(errorCode)) {
 293                 fprintf(stderr, "icupkg: unable to create tree directory \"%s\"\n", filename);
 294                 exit(U_FILE_ACCESS_ERROR);
 295             }
 296         }
 297         *sep++=U_FILE_SEP_CHAR; // restore file separator character
 298     }
 299 }
 300
 301 static uint8_t *
 302 readFile(const char *path, const char *name, int32_t &length, char &type) {
 303     char filename[1024];
 304     FILE *file;
 305     uint8_t *data;
 306     UErrorCode errorCode;
 307     int32_t fileLength, typeEnum;
 308
 309     makeFullFilename(path, name, filename, (int32_t)sizeof(filename));
 310
 311     /* open the input file, get its length, allocate memory for it, read the file */
 312     file=fopen(filename, "rb");
 313     if(file==NULL) {
 314         fprintf(stderr, "icupkg: unable to open input file \"%s\"\n", filename);
 315         exit(U_FILE_ACCESS_ERROR);
 316     }
 317
 318     /* get the file length */
 319     fileLength=getFileLength(file);
 320     if(ferror(file) || fileLength<=0) {
 321         fprintf(stderr, "icupkg: empty input file \"%s\"\n", filename);
 322         fclose(file);
 323         exit(U_FILE_ACCESS_ERROR);
 324     }
 325
 326     /* allocate the buffer, pad to multiple of 16 */
 327     length=(fileLength+0xf)&~0xf;
 328     data=(uint8_t *)malloc(length);
 329     if(data==NULL) {
 330         fclose(file);
 331         exit(U_MEMORY_ALLOCATION_ERROR);
 332     }
 333
 334     /* read the file */
 335     if(fileLength!=(int32_t)fread(data, 1, fileLength, file)) {
 336         fprintf(stderr, "icupkg: error reading \"%s\"\n", filename);
 337         fclose(file);
 338         free(data);
 339         exit(U_FILE_ACCESS_ERROR);
 340     }
 341
 342     /* pad the file to a multiple of 16 using the usual padding byte */
 343     if(fileLength<length) {
 344         memset(data+fileLength, 0xaa, length-fileLength);
 345     }
 346
 347     fclose(file);
 348
 349     // minimum check for ICU-format data
 350     errorCode=U_ZERO_ERROR;
 351     typeEnum=getTypeEnumForInputData(data, length, &errorCode);
 352     if(typeEnum<0 || U_FAILURE(errorCode)) {
 353         fprintf(stderr, "icupkg: not an ICU data file: \"%s\"\n", filename);
 354         free(data);
 355         exit(U_INVALID_FORMAT_ERROR);
 356     }
 357     type=makeTypeLetter(typeEnum);
 358
 359     return data;
 360 }
 361
 362 // .dat package file representation ---------------------------------------- ***
 363
 364 U_CDECL_BEGIN
 365
 366 static int32_t U_CALLCONV
 367 compareItems(const void * /*context*/, const void *left, const void *right) {
 368     return (int32_t)strcmp(((Item *)left)->name, ((Item *)right)->name);
 369 }
 370
 371 U_CDECL_END
 372
 373 Package::Package() {
 374     inPkgName[0]=0;
 375     inData=NULL;
 376     inLength=0;
 377     inCharset=U_CHARSET_FAMILY;
 378     inIsBigEndian=U_IS_BIG_ENDIAN;
 379
 380     itemCount=0;
 381     inStringTop=outStringTop=0;
 382
 383     matchMode=0;
 384     findPrefix=findSuffix=NULL;
 385     findPrefixLength=findSuffixLength=0;
 386     findNextIndex=-1;
 387
 388     // create a header for an empty package
 389     DataHeader *pHeader;
 390     pHeader=(DataHeader *)header;
 391     pHeader->dataHeader.magic1=0xda;
 392     pHeader->dataHeader.magic2=0x27;
 393     memcpy(&pHeader->info, &dataInfo, sizeof(dataInfo));
 394     headerLength=(int32_t)(4+sizeof(dataInfo));
 395     if(headerLength&0xf) {
 396         /* NUL-pad the header to a multiple of 16 */
 397         int32_t length=(headerLength+0xf)&~0xf;
 398         memset(header+headerLength, 0, length-headerLength);
 399         headerLength=length;
 400     }
 401     pHeader->dataHeader.headerSize=(uint16_t)headerLength;
 402 }
 403
 404 Package::~Package() {
 405     int32_t index;
 406
 407     free(inData);
 408
 409     for(index=0; index<itemCount; ++index) {
 410         if(items[index].isDataOwned) {
 411             free(items[index].data);
 412         }
 413     }
 414 }
 415
 416 void
 417 Package::readPackage(const char *filename) {
 418     UDataSwapper *ds;
 419     const UDataInfo *pInfo;
 420     UErrorCode errorCode;
 421
 422     const uint8_t *inBytes;
 423
 424     int32_t length, offset, i;
 425     int32_t itemLength, typeEnum;
 426     char type;
 427
 428     const UDataOffsetTOCEntry *inEntries;
 429
 430     extractPackageName(filename, inPkgName, (int32_t)sizeof(inPkgName));
 431
 432     /* read the file */
 433     inData=readFile(NULL, filename, inLength, type);
 434     length=inLength;
 435
 436     /*
 437      * swap the header - even if the swapping itself is a no-op
 438      * because it tells us the header length
 439      */
 440     errorCode=U_ZERO_ERROR;
 441     makeTypeProps(type, inCharset, inIsBigEndian);
 442     ds=udata_openSwapper(inIsBigEndian, inCharset, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &errorCode);
 443     if(U_FAILURE(errorCode)) {
 444         fprintf(stderr, "icupkg: udata_openSwapper(\"%s\") failed - %s\n",
 445                 filename, u_errorName(errorCode));
 446         exit(errorCode);
 447     }
 448
 449     ds->printError=printPackageError;
 450     ds->printErrorContext=stderr;
 451
 452     headerLength=sizeof(header);
 453     if(length<headerLength) {
 454         headerLength=length;
 455     }
 456     headerLength=udata_swapDataHeader(ds, inData, headerLength, header, &errorCode);
 457     if(U_FAILURE(errorCode)) {
 458         exit(errorCode);
 459     }
 460
 461     /* check data format and format version */
 462     pInfo=(const UDataInfo *)((const char *)inData+4);
 463     if(!(
 464         pInfo->dataFormat[0]==0x43 &&   /* dataFormat="CmnD" */
 465         pInfo->dataFormat[1]==0x6d &&
 466         pInfo->dataFormat[2]==0x6e &&
 467         pInfo->dataFormat[3]==0x44 &&
 468         pInfo->formatVersion[0]==1
 469     )) {
 470         fprintf(stderr, "icupkg: data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as an ICU .dat package\n",
 471                 pInfo->dataFormat[0], pInfo->dataFormat[1],
 472                 pInfo->dataFormat[2], pInfo->dataFormat[3],
 473                 pInfo->formatVersion[0]);
 474         exit(U_UNSUPPORTED_ERROR);
 475     }
 476     inIsBigEndian=(UBool)pInfo->isBigEndian;
 477     inCharset=pInfo->charsetFamily;
 478
 479     inBytes=(const uint8_t *)inData+headerLength;
 480     inEntries=(const UDataOffsetTOCEntry *)(inBytes+4);
 481
 482     /* check that the itemCount fits, then the ToC table, then at least the header of the last item */
 483     length-=headerLength;
 484     if(length<4) {
 485         /* itemCount does not fit */
 486         offset=0x7fffffff;
 487     } else {
 488         itemCount=udata_readInt32(ds, *(const int32_t *)inBytes);
 489         if(itemCount==0) {
 490             offset=4;
 491         } else if(length<(4+8*itemCount)) {
 492             /* ToC table does not fit */
 493             offset=0x7fffffff;
 494         } else {
 495             /* offset of the last item plus at least 20 bytes for its header */
 496             offset=20+(int32_t)ds->readUInt32(inEntries[itemCount-1].dataOffset);
 497         }
 498     }
 499     if(length<offset) {
 500         fprintf(stderr, "icupkg: too few bytes (%ld after header) for a .dat package\n",
 501                         (long)length);
 502         exit(U_INDEX_OUTOFBOUNDS_ERROR);
 503     }
 504     /* do not modify the package length variable until the last item's length is set */
 505
 506     if(itemCount>0) {
 507         char prefix[MAX_PKG_NAME_LENGTH+4];
 508         char *s, *inItemStrings;
 509         int32_t inPkgNameLength, prefixLength, stringsOffset;
 510
 511         if(itemCount>MAX_FILE_COUNT) {
 512             fprintf(stderr, "icupkg: too many items, maximum is %d\n", MAX_FILE_COUNT);
 513             exit(U_BUFFER_OVERFLOW_ERROR);
 514         }
 515
 516         /* swap the item name strings */
 517         stringsOffset=4+8*itemCount;
 518         itemLength=(int32_t)(ds->readUInt32(inEntries[0].dataOffset))-stringsOffset;
 519
 520         // don't include padding bytes at the end of the item names
 521         while(itemLength>0 && inBytes[stringsOffset+itemLength-1]!=0) {
 522             --itemLength;
 523         }
 524
 525         if((inStringTop+itemLength)>STRING_STORE_SIZE) {
 526             fprintf(stderr, "icupkg: total length of item name strings too long\n");
 527             exit(U_BUFFER_OVERFLOW_ERROR);
 528         }
 529
 530         inItemStrings=inStrings+inStringTop;
 531         ds->swapInvChars(ds, inBytes+stringsOffset, itemLength, inItemStrings, &errorCode);
 532         if(U_FAILURE(errorCode)) {
 533             fprintf(stderr, "icupkg failed to swap the input .dat package item name strings\n");
 534             exit(U_INVALID_FORMAT_ERROR);
 535         }
 536         inStringTop+=itemLength;
 537
 538         // reset the Item entries
 539         memset(items, 0, itemCount*sizeof(Item));
 540
 541         inPkgNameLength=strlen(inPkgName);
 542         memcpy(prefix, inPkgName, inPkgNameLength);
 543         prefixLength=inPkgNameLength;
 544
 545         /*
 546          * Get the common prefix of the items.
 547          * New-style ICU .dat packages use tree separators ('/') between package names,
 548          * tree names, and item names,
 549          * while old-style ICU .dat packages (before multi-tree support)
 550          * use an underscore ('_') between package and item names.
 551          */
 552         offset=(int32_t)ds->readUInt32(inEntries[0].nameOffset)-stringsOffset;
 553         s=inItemStrings+offset;
 554         if( (int32_t)strlen(s)>=(inPkgNameLength+2) &&
 555             0==memcmp(s, inPkgName, inPkgNameLength) &&
 556             s[inPkgNameLength]=='_'
 557         ) {
 558             // old-style .dat package
 559             prefix[prefixLength++]='_';
 560         } else {
 561             // new-style .dat package
 562             prefix[prefixLength++]=U_TREE_ENTRY_SEP_CHAR;
 563             // if it turns out to not contain U_TREE_ENTRY_SEP_CHAR
 564             // then the test in the loop below will fail
 565         }
 566         prefix[prefixLength]=0;
 567
 568         /* read the ToC table */
 569         for(i=0; i<itemCount; ++i) {
 570             // skip the package part of the item name, error if it does not match the actual package name
 571             // or if nothing follows the package name
 572             offset=(int32_t)ds->readUInt32(inEntries[i].nameOffset)-stringsOffset;
 573             s=inItemStrings+offset;
 574             if(0!=strncmp(s, prefix, prefixLength) || s[prefixLength]==0) {
 575                 fprintf(stderr, "icupkg: input .dat item name \"%s\" does not start with \"%s\"\n",
 576                         s, prefix);
 577                 exit(U_UNSUPPORTED_ERROR);
 578             }
 579             items[i].name=s+prefixLength;
 580
 581             // set the item's data
 582             items[i].data=(uint8_t *)inBytes+ds->readUInt32(inEntries[i].dataOffset);
 583             if(i>0) {
 584                 items[i-1].length=(int32_t)(items[i].data-items[i-1].data);
 585
 586                 // set the previous item's platform type
 587                 typeEnum=getTypeEnumForInputData(items[i-1].data, items[i-1].length, &errorCode);
 588                 if(typeEnum<0 || U_FAILURE(errorCode)) {
 589                     fprintf(stderr, "icupkg: not an ICU data file: item \"%s\" in \"%s\"\n", items[i-1].name, filename);
 590                     exit(U_INVALID_FORMAT_ERROR);
 591                 }
 592                 items[i-1].type=makeTypeLetter(typeEnum);
 593             }
 594             items[i].isDataOwned=FALSE;
 595         }
 596         // set the last item's length
 597         items[itemCount-1].length=length-ds->readUInt32(inEntries[itemCount-1].dataOffset);
 598
 599         // set the last item's platform type
 600         typeEnum=getTypeEnumForInputData(items[itemCount-1].data, items[itemCount-1].length, &errorCode);
 601         if(typeEnum<0 || U_FAILURE(errorCode)) {
 602             fprintf(stderr, "icupkg: not an ICU data file: item \"%s\" in \"%s\"\n", items[i-1].name, filename);
 603             exit(U_INVALID_FORMAT_ERROR);
 604         }
 605         items[itemCount-1].type=makeTypeLetter(typeEnum);
 606
 607         if(type!=U_ICUDATA_TYPE_LETTER[0]) {
 608             // sort the item names for the local charset
 609             sortItems();
 610         }
 611     }
 612
 613     udata_closeSwapper(ds);
 614 }
 615
 616 char
 617 Package::getInType() {
 618     return makeTypeLetter(inCharset, inIsBigEndian);
 619 }
 620
 621 void
 622 Package::writePackage(const char *filename, char outType, const char *comment) {
 623     char prefix[MAX_PKG_NAME_LENGTH+4];
 624     UDataOffsetTOCEntry entry;
 625     UDataSwapper *dsLocalToOut, *ds[TYPE_COUNT];
 626     FILE *file;
 627     Item *pItem;
 628     char *name;
 629     UErrorCode errorCode;
 630     int32_t i, length, prefixLength, maxItemLength, basenameOffset, offset, outInt32;
 631     uint8_t outCharset;
 632     UBool outIsBigEndian;
 633
 634     extractPackageName(filename, prefix, MAX_PKG_NAME_LENGTH);
 635
 636     // if there is an explicit comment, then use it, else use what's in the current header
 637     if(comment!=NULL) {
 638         /* get the header size minus the current comment */
 639         DataHeader *pHeader;
 640         int32_t length;
 641
 642         pHeader=(DataHeader *)header;
 643         headerLength=4+pHeader->info.size;
 644         length=(int32_t)strlen(comment);
 645         if((int32_t)(headerLength+length)>=(int32_t)sizeof(header)) {
 646             fprintf(stderr, "icupkg: comment too long\n");
 647             exit(U_BUFFER_OVERFLOW_ERROR);
 648         }
 649         memcpy(header+headerLength, comment, length+1);
 650         headerLength+=length;
 651         if(headerLength&0xf) {
 652             /* NUL-pad the header to a multiple of 16 */
 653             length=(headerLength+0xf)&~0xf;
 654             memset(header+headerLength, 0, length-headerLength);
 655             headerLength=length;
 656         }
 657         pHeader->dataHeader.headerSize=(uint16_t)headerLength;
 658     }
 659
 660     makeTypeProps(outType, outCharset, outIsBigEndian);
 661
 662     // open (TYPE_COUNT-2) swappers
 663     // one is a no-op for local type==outType
 664     // one type (TYPE_LE) is bogus
 665     errorCode=U_ZERO_ERROR;
 666     i=makeTypeEnum(outType);
 667     ds[TYPE_B]= i==TYPE_B ? NULL : udata_openSwapper(TRUE, U_ASCII_FAMILY, outIsBigEndian, outCharset, &errorCode);
 668     ds[TYPE_L]= i==TYPE_L ? NULL : udata_openSwapper(FALSE, U_ASCII_FAMILY, outIsBigEndian, outCharset, &errorCode);
 669     ds[TYPE_LE]=NULL;
 670     ds[TYPE_E]= i==TYPE_E ? NULL : udata_openSwapper(TRUE, U_EBCDIC_FAMILY, outIsBigEndian, outCharset, &errorCode);
 671     if(U_FAILURE(errorCode)) {
 672         fprintf(stderr, "icupkg: udata_openSwapper() failed - %s\n", u_errorName(errorCode));
 673         exit(errorCode);
 674     }
 675     for(i=0; i<TYPE_COUNT; ++i) {
 676         if(ds[i]!=NULL) {
 677             ds[i]->printError=printPackageError;
 678             ds[i]->printErrorContext=stderr;
 679         }
 680     }
 681
 682     dsLocalToOut=ds[makeTypeEnum(U_CHARSET_FAMILY, U_IS_BIG_ENDIAN)];
 683
 684     // create the file and write its contents
 685     file=fopen(filename, "wb");
 686     if(file==NULL) {
 687         fprintf(stderr, "icupkg: unable to create file \"%s\"\n", filename);
 688         exit(U_FILE_ACCESS_ERROR);
 689     }
 690
 691     // swap and write the header
 692     if(dsLocalToOut!=NULL) {
 693         udata_swapDataHeader(dsLocalToOut, header, headerLength, header, &errorCode);
 694         if(U_FAILURE(errorCode)) {
 695             fprintf(stderr, "icupkg: udata_swapDataHeader(local to out) failed - %s\n", u_errorName(errorCode));
 696             exit(errorCode);
 697         }
 698     }
 699     length=(int32_t)fwrite(header, 1, headerLength, file);
 700     if(length!=headerLength) {
 701         fprintf(stderr, "icupkg: unable to write complete header to file \"%s\"\n", filename);
 702         exit(U_FILE_ACCESS_ERROR);
 703     }
 704
 705     // prepare and swap the package name with a tree separator
 706     // for prepending to item names
 707     strcat(prefix, U_TREE_ENTRY_SEP_STRING);
 708     prefixLength=(int32_t)strlen(prefix);
 709     if(dsLocalToOut!=NULL) {
 710         dsLocalToOut->swapInvChars(dsLocalToOut, prefix, prefixLength, prefix, &errorCode);
 711         if(U_FAILURE(errorCode)) {
 712             fprintf(stderr, "icupkg: swapInvChars(output package name) failed - %s\n", u_errorName(errorCode));
 713             exit(errorCode);
 714         }
 715
 716         // swap and sort the item names (sorting needs to be done in the output charset)
 717         dsLocalToOut->swapInvChars(dsLocalToOut, inStrings, inStringTop, inStrings, &errorCode);
 718         if(U_FAILURE(errorCode)) {
 719             fprintf(stderr, "icupkg: swapInvChars(item names) failed - %s\n", u_errorName(errorCode));
 720             exit(errorCode);
 721         }
 722         sortItems();
 723     }
 724
 725     // create the output item names in sorted order, with the package name prepended to each
 726     for(i=0; i<itemCount; ++i) {
 727         length=(int32_t)strlen(items[i].name);
 728         name=allocString(FALSE, length+prefixLength);
 729         memcpy(name, prefix, prefixLength);
 730         memcpy(name+prefixLength, items[i].name, length+1);
 731         items[i].name=name;
 732     }
 733
 734     // calculate offsets for item names and items, pad to 16-align items
 735     // align only the first item; each item's length is a multiple of 16
 736     basenameOffset=4+8*itemCount;
 737     offset=basenameOffset+outStringTop;
 738     if((length=(offset&15))!=0) {
 739         length=16-length;
 740         memset(allocString(FALSE, length-1), 0xaa, length);
 741         offset+=length;
 742     }
 743
 744     // write the table of contents
 745     // first the itemCount
 746     outInt32=itemCount;
 747     if(dsLocalToOut!=NULL) {
 748         dsLocalToOut->swapArray32(dsLocalToOut, &outInt32, 4, &outInt32, &errorCode);
 749         if(U_FAILURE(errorCode)) {
 750             fprintf(stderr, "icupkg: swapArray32(item count) failed - %s\n", u_errorName(errorCode));
 751             exit(errorCode);
 752         }
 753     }
 754     length=(int32_t)fwrite(&outInt32, 1, 4, file);
 755     if(length!=4) {
 756         fprintf(stderr, "icupkg: unable to write complete item count to file \"%s\"\n", filename);
 757         exit(U_FILE_ACCESS_ERROR);
 758     }
 759
 760     // then write the item entries (and collect the maxItemLength)
 761     maxItemLength=0;
 762     for(i=0; i<itemCount; ++i) {
 763         entry.nameOffset=(uint32_t)(basenameOffset+(items[i].name-outStrings));
 764         entry.dataOffset=(uint32_t)offset;
 765         if(dsLocalToOut!=NULL) {
 766             dsLocalToOut->swapArray32(dsLocalToOut, &entry, 8, &entry, &errorCode);
 767             if(U_FAILURE(errorCode)) {
 768                 fprintf(stderr, "icupkg: swapArray32(item entry %ld) failed - %s\n", (long)i, u_errorName(errorCode));
 769                 exit(errorCode);
 770             }
 771         }
 772         length=(int32_t)fwrite(&entry, 1, 8, file);
 773         if(length!=8) {
 774             fprintf(stderr, "icupkg: unable to write complete item entry %ld to file \"%s\"\n", (long)i, filename);
 775             exit(U_FILE_ACCESS_ERROR);
 776         }
 777
 778         length=items[i].length;
 779         if(length>maxItemLength) {
 780             maxItemLength=length;
 781         }
 782         offset+=length;
 783     }
 784
 785     // write the item names
 786     length=(int32_t)fwrite(outStrings, 1, outStringTop, file);
 787     if(length!=outStringTop) {
 788         fprintf(stderr, "icupkg: unable to write complete item names to file \"%s\"\n", filename);
 789         exit(U_FILE_ACCESS_ERROR);
 790     }
 791
 792     // write the items
 793     for(pItem=items, i=0; i<itemCount; ++pItem, ++i) {
 794         int32_t type=makeTypeEnum(pItem->type);
 795         if(ds[type]!=NULL) {
 796             // swap each item from its platform properties to the desired ones
 797             udata_swap(
 798                 ds[type],
 799                 pItem->data, pItem->length, pItem->data,
 800                 &errorCode);
 801             if(U_FAILURE(errorCode)) {
 802                 fprintf(stderr, "icupkg: udata_swap(item %ld) failed - %s\n", (long)i, u_errorName(errorCode));
 803                 exit(errorCode);
 804             }
 805         }
 806         length=(int32_t)fwrite(pItem->data, 1, pItem->length, file);
 807         if(length!=pItem->length) {
 808             fprintf(stderr, "icupkg: unable to write complete item %ld to file \"%s\"\n", (long)i, filename);
 809             exit(U_FILE_ACCESS_ERROR);
 810         }
 811     }
 812
 813     if(ferror(file)) {
 814         fprintf(stderr, "icupkg: unable to write complete file \"%s\"\n", filename);
 815         exit(U_FILE_ACCESS_ERROR);
 816     }
 817
 818     fclose(file);
 819     for(i=0; i<TYPE_COUNT; ++i) {
 820         udata_closeSwapper(ds[i]);
 821     }
 822 }
 823
 824 int32_t
 825 Package::findItem(const char *name, int32_t length) {
 826     int32_t i, start, limit;
 827     int result;
 828
 829     /* do a binary search for the string */
 830     start=0;
 831     limit=itemCount;
 832     while(start<limit) {
 833         i=(start+limit)/2;
 834         if(length>=0) {
 835             result=strncmp(name, items[i].name, length);
 836         } else {
 837             result=strcmp(name, items[i].name);
 838         }
 839
 840         if(result==0) {
 841             /* found */
 842             if(length>=0) {
 843                 /*
 844                  * if we compared just prefixes, then we may need to back up
 845                  * to the first item with this prefix
 846                  */
 847                 while(i>0 && 0==strncmp(name, items[i-1].name, length)) {
 848                     --i;
 849                 }
 850             }
 851             return i;
 852         } else if(result<0) {
 853             limit=i;
 854         } else /* result>0 */ {
 855             start=i+1;
 856         }
 857     }
 858
 859     return ~start; /* not found, return binary-not of the insertion point */
 860 }
 861
 862 void
 863 Package::findItems(const char *pattern) {
 864     const char *wild;
 865
 866     if(pattern==NULL || *pattern==0) {
 867         findNextIndex=-1;
 868         return;
 869     }
 870
 871     findPrefix=pattern;
 872     findSuffix=NULL;
 873     findSuffixLength=0;
 874
 875     wild=strchr(pattern, '*');
 876     if(wild==NULL) {
 877         // no wildcard
 878         findPrefixLength=(int32_t)strlen(pattern);
 879     } else {
 880         // one wildcard
 881         findPrefixLength=(int32_t)(wild-pattern);
 882         findSuffix=wild+1;
 883         findSuffixLength=(int32_t)strlen(findSuffix);
 884         if(NULL!=strchr(findSuffix, '*')) {
 885             // two or more wildcards
 886             fprintf(stderr, "icupkg: syntax error (more than one '*') in item pattern \"%s\"\n", pattern);
 887             exit(U_PARSE_ERROR);
 888         }
 889     }
 890
 891     if(findPrefixLength==0) {
 892         findNextIndex=0;
 893     } else {
 894         findNextIndex=findItem(findPrefix, findPrefixLength);
 895     }
 896 }
 897
 898 int32_t
 899 Package::findNextItem() {
 900     const char *name, *middle, *treeSep;
 901     int32_t index, nameLength, middleLength;
 902
 903     if(findNextIndex<0) {
 904         return -1;
 905     }
 906
 907     while(findNextIndex<itemCount) {
 908         index=findNextIndex++;
 909         name=items[index].name;
 910         nameLength=(int32_t)strlen(name);
 911         if(nameLength<(findPrefixLength+findSuffixLength)) {
 912             // item name too short for prefix & suffix
 913             continue;
 914         }
 915         if(findPrefixLength>0 && 0!=memcmp(findPrefix, name, findPrefixLength)) {
 916             // left the range of names with this prefix
 917             break;
 918         }
 919         middle=name+findPrefixLength;
 920         middleLength=nameLength-findPrefixLength-findSuffixLength;
 921         if(findSuffixLength>0 && 0!=memcmp(findSuffix, name+(nameLength-findSuffixLength), findSuffixLength)) {
 922             // suffix does not match
 923             continue;
 924         }
 925         // prefix & suffix match
 926
 927         if(matchMode&MATCH_NOSLASH) {
 928             treeSep=strchr(middle, U_TREE_ENTRY_SEP_CHAR);
 929             if(treeSep!=NULL && (treeSep-middle)<middleLength) {
 930                 // the middle (matching the * wildcard) contains a tree separator /
 931                 continue;
 932             }
 933         }
 934
 935         // found a matching item
 936         return index;
 937     }
 938
 939     // no more items
 940     findNextIndex=-1;
 941     return -1;
 942 }
 943
 944 void
 945 Package::setMatchMode(uint32_t mode) {
 946     matchMode=mode;
 947 }
 948
 949 void
 950 Package::addItem(const char *name) {
 951     addItem(name, NULL, 0, FALSE, U_ICUDATA_TYPE_LETTER[0]);
 952 }
 953
 954 void
 955 Package::addItem(const char *name, uint8_t *data, int32_t length, UBool isDataOwned, char type) {
 956     int32_t index;
 957
 958     index=findItem(name);
 959     if(index<0) {
 960         // new item, make space at the insertion point
 961         if(itemCount>=MAX_FILE_COUNT) {
 962             fprintf(stderr, "icupkg: too many items, maximum is %d\n", MAX_FILE_COUNT);
 963             exit(U_BUFFER_OVERFLOW_ERROR);
 964         }
 965         // move the following items down
 966         index=~index;
 967         if(index<itemCount) {
 968             memmove(items+index+1, items+index, (itemCount-index)*sizeof(Item));
 969         }
 970         ++itemCount;
 971
 972         // reset this Item entry
 973         memset(items+index, 0, sizeof(Item));
 974
 975         // copy the item's name
 976         items[index].name=allocString(TRUE, strlen(name));
 977         strcpy(items[index].name, name);
 978         pathToTree(items[index].name);
 979     } else {
 980         // same-name item found, replace it
 981         if(items[index].isDataOwned) {
 982             free(items[index].data);
 983         }
 984
 985         // keep the item's name since it is the same
 986     }
 987
 988     // set the item's data
 989     items[index].data=data;
 990     items[index].length=length;
 991     items[index].isDataOwned=isDataOwned;
 992     items[index].type=type;
 993 }
 994
 995 void
 996 Package::addFile(const char *filesPath, const char *name) {
 997     uint8_t *data;
 998     int32_t length;
 999     char type;
1000
1001     data=readFile(filesPath, name, length, type);
1002     // readFile() exits the tool if it fails
1003     addItem(name, data, length, TRUE, type);
1004 }
1005
1006 void
1007 Package::addItems(const Package &listPkg) {
1008     const Item *pItem;
1009     int32_t i;
1010
1011     for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) {
1012         addItem(pItem->name, pItem->data, pItem->length, FALSE, pItem->type);
1013     }
1014 }
1015
1016 void
1017 Package::removeItem(int32_t index) {
1018     if(index>=0) {
1019         // remove the item
1020         if(items[index].isDataOwned) {
1021             free(items[index].data);
1022         }
1023
1024         // move the following items up
1025         if((index+1)<itemCount) {
1026             memmove(items+index, items+index+1, (itemCount-(index+1))*sizeof(Item));
1027         }
1028         --itemCount;
1029
1030         if(index<=findNextIndex) {
1031             --findNextIndex;
1032         }
1033     }
1034 }
1035
1036 void
1037 Package::removeItems(const char *pattern) {
1038     int32_t index;
1039
1040     findItems(pattern);
1041     while((index=findNextItem())>=0) {
1042         removeItem(index);
1043     }
1044 }
1045
1046 void
1047 Package::removeItems(const Package &listPkg) {
1048     const Item *pItem;
1049     int32_t i;
1050
1051     for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) {
1052         removeItems(pItem->name);
1053     }
1054 }
1055
1056 void
1057 Package::extractItem(const char *filesPath, const char *outName, int32_t index, char outType) {
1058     char filename[1024];
1059     UDataSwapper *ds;
1060     FILE *file;
1061     Item *pItem;
1062     int32_t fileLength;
1063     uint8_t itemCharset, outCharset;
1064     UBool itemIsBigEndian, outIsBigEndian;
1065
1066     if(index<0 || itemCount<=index) {
1067         return;
1068     }
1069     pItem=items+index;
1070
1071     // swap the data to the outType
1072     // outType==0: don't swap
1073     if(outType!=0 && pItem->type!=outType) {
1074         // open the swapper
1075         UErrorCode errorCode=U_ZERO_ERROR;
1076         makeTypeProps(pItem->type, itemCharset, itemIsBigEndian);
1077         makeTypeProps(outType, outCharset, outIsBigEndian);
1078         ds=udata_openSwapper(itemIsBigEndian, itemCharset, outIsBigEndian, outCharset, &errorCode);
1079         if(U_FAILURE(errorCode)) {
1080             fprintf(stderr, "icupkg: udata_openSwapper(item %ld) failed - %s\n",
1081                     (long)index, u_errorName(errorCode));
1082             exit(errorCode);
1083         }
1084
1085         ds->printError=printPackageError;
1086         ds->printErrorContext=stderr;
1087
1088         // swap the item from its platform properties to the desired ones
1089         udata_swap(ds, pItem->data, pItem->length, pItem->data, &errorCode);
1090         if(U_FAILURE(errorCode)) {
1091             fprintf(stderr, "icupkg: udata_swap(item %ld) failed - %s\n", (long)index, u_errorName(errorCode));
1092             exit(errorCode);
1093         }
1094         udata_closeSwapper(ds);
1095     }
1096
1097     // create the file and write its contents
1098     makeFullFilenameAndDirs(filesPath, outName, filename, (int32_t)sizeof(filename));
1099     file=fopen(filename, "wb");
1100     if(file==NULL) {
1101         fprintf(stderr, "icupkg: unable to create file \"%s\"\n", filename);
1102         exit(U_FILE_ACCESS_ERROR);
1103     }
1104     fileLength=(int32_t)fwrite(pItem->data, 1, pItem->length, file);
1105
1106     if(ferror(file) || fileLength!=pItem->length) {
1107         fprintf(stderr, "icupkg: unable to write complete file \"%s\"\n", filename);
1108         exit(U_FILE_ACCESS_ERROR);
1109     }
1110     fclose(file);
1111 }
1112
1113 void
1114 Package::extractItem(const char *filesPath, int32_t index, char outType) {
1115     extractItem(filesPath, items[index].name, index, outType);
1116 }
1117
1118 void
1119 Package::extractItems(const char *filesPath, const char *pattern, char outType) {
1120     int32_t index;
1121
1122     findItems(pattern);
1123     while((index=findNextItem())>=0) {
1124         extractItem(filesPath, index, outType);
1125     }
1126 }
1127
1128 void
1129 Package::extractItems(const char *filesPath, const Package &listPkg, char outType) {
1130     const Item *pItem;
1131     int32_t i;
1132
1133     for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) {
1134         extractItems(filesPath, pItem->name, outType);
1135     }
1136 }
1137
1138 void
1139 Package::listItems(FILE *file) {
1140     int32_t i;
1141
1142     for(i=0; i<itemCount; ++i) {
1143         fprintf(file, "%s\n", items[i].name);
1144     }
1145 }
1146
1147 void
1148 Package::checkDependency(void *context, const char *itemName, const char *targetName) {
1149     // check dependency: make sure the target item is in the package
1150     Package *me=(Package *)context;
1151     if(me->findItem(targetName)<0) {
1152         me->isMissingItems=TRUE;
1153         fprintf(stderr, "Item %s depends on missing item %s\n", itemName, targetName);
1154     }
1155 }
1156
1157 UBool
1158 Package::checkDependencies() {
1159     int32_t i;
1160
1161     isMissingItems=FALSE;
1162     for(i=0; i<itemCount; ++i) {
1163         enumDependencies(items+i);
1164     }
1165     return (UBool)!isMissingItems;
1166 }
1167
1168 char *
1169 Package::allocString(UBool in, int32_t length) {
1170     char *p;
1171     int32_t top;
1172
1173     if(in) {
1174         top=inStringTop;
1175         p=inStrings+top;
1176     } else {
1177         top=outStringTop;
1178         p=outStrings+top;
1179     }
1180     top+=length+1;
1181
1182     if(top>STRING_STORE_SIZE) {
1183         fprintf(stderr, "icupkg: string storage overflow\n");
1184         exit(U_BUFFER_OVERFLOW_ERROR);
1185     }
1186     if(in) {
1187         inStringTop=top;
1188     } else {
1189         outStringTop=top;
1190     }
1191     return p;
1192 }
1193
1194 void
1195 Package::sortItems() {
1196     UErrorCode errorCode=U_ZERO_ERROR;
1197     uprv_sortArray(items, itemCount, (int32_t)sizeof(Item), compareItems, NULL, FALSE, &errorCode);
1198     if(U_FAILURE(errorCode)) {
1199         fprintf(stderr, "icupkg: sorting item names failed - %s\n", u_errorName(errorCode));
1200         exit(errorCode);
1201     }
1202 }