icuSources/tools/toolutil/pkg_gencmn.c

   1 /******************************************************************************
   2  *   Copyright (C) 2008-2012, International Business Machines
   3  *   Corporation and others.  All Rights Reserved.
   4  *******************************************************************************
   5  */
   6 #include "unicode/utypes.h"
   7
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include "unicode/utypes.h"
  11 #include "unicode/putil.h"
  12 #include "cmemory.h"
  13 #include "cstring.h"
  14 #include "filestrm.h"
  15 #include "toolutil.h"
  16 #include "unicode/uclean.h"
  17 #include "unewdata.h"
  18 #include "putilimp.h"
  19 #include "pkg_gencmn.h"
  20
  21 #define STRING_STORE_SIZE 200000
  22
  23 #define COMMON_DATA_NAME U_ICUDATA_NAME
  24 #define DATA_TYPE "dat"
  25
  26 /* ICU package data file format (.dat files) ------------------------------- ***
  27
  28 Description of the data format after the usual ICU data file header
  29 (UDataInfo etc.).
  30
  31 Format version 1
  32
  33 A .dat package file contains a simple Table of Contents of item names,
  34 followed by the items themselves:
  35
  36 1. ToC table
  37
  38 uint32_t count; - number of items
  39 UDataOffsetTOCEntry entry[count]; - pair of uint32_t values per item:
  40     uint32_t nameOffset; - offset of the item name
  41     uint32_t dataOffset; - offset of the item data
  42 both are byte offsets from the beginning of the data
  43
  44 2. item name strings
  45
  46 All item names are stored as char * strings in one block between the ToC table
  47 and the data items.
  48
  49 3. data items
  50
  51 The data items are stored following the item names block.
  52 Each data item is 16-aligned.
  53 The data items are stored in the sorted order of their names.
  54
  55 Therefore, the top of the name strings block is the offset of the first item,
  56 the length of the last item is the difference between its offset and
  57 the .dat file length, and the length of all previous items is the difference
  58 between its offset and the next one.
  59
  60 ----------------------------------------------------------------------------- */
  61
  62 /* UDataInfo cf. udata.h */
  63 static const UDataInfo dataInfo={
  64     sizeof(UDataInfo),
  65     0,
  66
  67     U_IS_BIG_ENDIAN,
  68     U_CHARSET_FAMILY,
  69     sizeof(UChar),
  70     0,
  71
  72     {0x43, 0x6d, 0x6e, 0x44},     /* dataFormat="CmnD" */
  73     {1, 0, 0, 0},                 /* formatVersion */
  74     {3, 0, 0, 0}                  /* dataVersion */
  75 };
  76
  77 static uint32_t maxSize;
  78
  79 static char stringStore[STRING_STORE_SIZE];
  80 static uint32_t stringTop=0, basenameTotal=0;
  81
  82 typedef struct {
  83     char *pathname, *basename;
  84     uint32_t basenameLength, basenameOffset, fileSize, fileOffset;
  85 } File;
  86
  87 #define CHUNK_FILE_COUNT 256
  88 static File *files = NULL;
  89 static uint32_t fileCount=0;
  90 static uint32_t fileMax = 0;
  91
  92
  93 static char *symPrefix = NULL;
  94
  95 #define LINE_BUFFER_SIZE 512
  96 /* prototypes --------------------------------------------------------------- */
  97
  98 static void
  99 addFile(const char *filename, const char *name, const char *source, UBool sourceTOC, UBool verbose);
 100
 101 static char *
 102 allocString(uint32_t length);
 103
 104 static int
 105 compareFiles(const void *file1, const void *file2);
 106
 107 static char *
 108 pathToFullPath(const char *path, const char *source);
 109
 110 /* map non-tree separator (such as '\') to tree separator ('/') inplace. */
 111 static void
 112 fixDirToTreePath(char *s);
 113 /* -------------------------------------------------------------------------- */
 114
 115 U_CAPI void U_EXPORT2
 116 createCommonDataFile(const char *destDir, const char *name, const char *entrypointName, const char *type, const char *source, const char *copyRight,
 117                      const char *dataFile, uint32_t max_size, UBool sourceTOC, UBool verbose, char *gencmnFileName) {
 118     static char buffer[4096];
 119     char *line;
 120     char *linePtr;
 121     char *s = NULL;
 122     UErrorCode errorCode=U_ZERO_ERROR;
 123     uint32_t i, fileOffset, basenameOffset, length, nread;
 124     FileStream *in, *file;
 125
 126     line = (char *)uprv_malloc(sizeof(char) * LINE_BUFFER_SIZE);
 127     if (line == NULL) {
 128         fprintf(stderr, "gencmn: unable to allocate memory for line buffer of size %d\n", LINE_BUFFER_SIZE);
 129         exit(U_MEMORY_ALLOCATION_ERROR);
 130     }
 131
 132     linePtr = line;
 133
 134     maxSize = max_size;
 135
 136     if (destDir == NULL) {
 137         destDir = u_getDataDirectory();
 138     }
 139     if (name == NULL) {
 140         name = COMMON_DATA_NAME;
 141     }
 142     if (type == NULL) {
 143         type = DATA_TYPE;
 144     }
 145     if (source == NULL) {
 146         source = ".";
 147     }
 148
 149     if (dataFile == NULL) {
 150         in = T_FileStream_stdin();
 151     } else {
 152         in = T_FileStream_open(dataFile, "r");
 153         if(in == NULL) {
 154             fprintf(stderr, "gencmn: unable to open input file %s\n", dataFile);
 155             exit(U_FILE_ACCESS_ERROR);
 156         }
 157     }
 158
 159     if (verbose) {
 160         if(sourceTOC) {
 161             printf("generating %s_%s.c (table of contents source file)\n", name, type);
 162         } else {
 163             printf("generating %s.%s (common data file with table of contents)\n", name, type);
 164         }
 165     }
 166
 167     /* read the list of files and get their lengths */
 168     while((s != NULL && *s != 0) || (s=T_FileStream_readLine(in, (line=linePtr),
 169                                                              LINE_BUFFER_SIZE))!=NULL) {
 170         /* remove trailing newline characters and parse space separated items */
 171         if (s != NULL && *s != 0) {
 172             line=s;
 173         } else {
 174             s=line;
 175         }
 176         while(*s!=0) {
 177             if(*s==' ') {
 178                 *s=0;
 179                 ++s;
 180                 break;
 181             } else if(*s=='\r' || *s=='\n') {
 182                 *s=0;
 183                 break;
 184             }
 185             ++s;
 186         }
 187
 188         /* check for comment */
 189
 190         if (*line == '#') {
 191             continue;
 192         }
 193
 194         /* add the file */
 195 #if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
 196         {
 197           char *t;
 198           while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) {
 199             *t = U_FILE_SEP_CHAR;
 200           }
 201         }
 202 #endif
 203         addFile(getLongPathname(line), name, source, sourceTOC, verbose);
 204     }
 205
 206     uprv_free(linePtr);
 207
 208     if(in!=T_FileStream_stdin()) {
 209         T_FileStream_close(in);
 210     }
 211
 212     if(fileCount==0) {
 213         fprintf(stderr, "gencmn: no files listed in %s\n", dataFile == NULL ? "<stdin>" : dataFile);
 214         return;
 215     }
 216
 217     /* sort the files by basename */
 218     qsort(files, fileCount, sizeof(File), compareFiles);
 219
 220     if(!sourceTOC) {
 221         UNewDataMemory *out;
 222
 223         /* determine the offsets of all basenames and files in this common one */
 224         basenameOffset=4+8*fileCount;
 225         fileOffset=(basenameOffset+(basenameTotal+15))&~0xf;
 226         for(i=0; i<fileCount; ++i) {
 227             files[i].fileOffset=fileOffset;
 228             fileOffset+=(files[i].fileSize+15)&~0xf;
 229             files[i].basenameOffset=basenameOffset;
 230             basenameOffset+=files[i].basenameLength;
 231         }
 232
 233         /* create the output file */
 234         out=udata_create(destDir, type, name,
 235                          &dataInfo,
 236                          copyRight == NULL ? U_COPYRIGHT_STRING : copyRight,
 237                          &errorCode);
 238         if(U_FAILURE(errorCode)) {
 239             fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n",
 240                 destDir, name, type,
 241                 u_errorName(errorCode));
 242             exit(errorCode);
 243         }
 244
 245         /* write the table of contents */
 246         udata_write32(out, fileCount);
 247         for(i=0; i<fileCount; ++i) {
 248             udata_write32(out, files[i].basenameOffset);
 249             udata_write32(out, files[i].fileOffset);
 250         }
 251
 252         /* write the basenames */
 253         for(i=0; i<fileCount; ++i) {
 254             udata_writeString(out, files[i].basename, files[i].basenameLength);
 255         }
 256         length=4+8*fileCount+basenameTotal;
 257
 258         /* copy the files */
 259         for(i=0; i<fileCount; ++i) {
 260             /* pad to 16-align the next file */
 261             length&=0xf;
 262             if(length!=0) {
 263                 udata_writePadding(out, 16-length);
 264             }
 265
 266             if (verbose) {
 267                 printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
 268             }
 269
 270             /* copy the next file */
 271             file=T_FileStream_open(files[i].pathname, "rb");
 272             if(file==NULL) {
 273                 fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname);
 274                 exit(U_FILE_ACCESS_ERROR);
 275             }
 276             for(nread = 0;;) {
 277                 length=T_FileStream_read(file, buffer, sizeof(buffer));
 278                 if(length <= 0) {
 279                     break;
 280                 }
 281                 nread += length;
 282                 udata_writeBlock(out, buffer, length);
 283             }
 284             T_FileStream_close(file);
 285             length=files[i].fileSize;
 286
 287             if (nread != files[i].fileSize) {
 288               fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname,  (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
 289                 exit(U_FILE_ACCESS_ERROR);
 290             }
 291         }
 292
 293         /* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */
 294         length&=0xf;
 295         if(length!=0) {
 296             udata_writePadding(out, 16-length);
 297         }
 298
 299         /* finish */
 300         udata_finish(out, &errorCode);
 301         if(U_FAILURE(errorCode)) {
 302             fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode));
 303             exit(errorCode);
 304         }
 305     } else {
 306         /* write a .c source file with the table of contents */
 307         char *filename;
 308         FileStream *out;
 309
 310         /* create the output filename */
 311         filename=s=buffer;
 312         uprv_strcpy(filename, destDir);
 313         s=filename+uprv_strlen(filename);
 314         if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) {
 315             *s++=U_FILE_SEP_CHAR;
 316         }
 317         uprv_strcpy(s, name);
 318         if(*(type)!=0) {
 319             s+=uprv_strlen(s);
 320             *s++='_';
 321             uprv_strcpy(s, type);
 322         }
 323         s+=uprv_strlen(s);
 324         uprv_strcpy(s, ".c");
 325
 326         /* open the output file */
 327         out=T_FileStream_open(filename, "w");
 328         if (gencmnFileName != NULL) {
 329             uprv_strcpy(gencmnFileName, filename);
 330         }
 331         if(out==NULL) {
 332             fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename);
 333             exit(U_FILE_ACCESS_ERROR);
 334         }
 335
 336         /* write the source file */
 337         sprintf(buffer,
 338             "/*\n"
 339             " * ICU common data table of contents for %s.%s\n"
 340             " * Automatically generated by icu/source/tools/gencmn/gencmn .\n"
 341             " */\n\n"
 342             "#include \"unicode/utypes.h\"\n"
 343             "#include \"unicode/udata.h\"\n"
 344             "\n"
 345             "/* external symbol declarations for data (%d files) */\n",
 346                 name, type, fileCount);
 347         T_FileStream_writeLine(out, buffer);
 348
 349         sprintf(buffer, "extern const char\n    %s%s[]", symPrefix?symPrefix:"", files[0].pathname);
 350         T_FileStream_writeLine(out, buffer);
 351         for(i=1; i<fileCount; ++i) {
 352             sprintf(buffer, ",\n    %s%s[]", symPrefix?symPrefix:"", files[i].pathname);
 353             T_FileStream_writeLine(out, buffer);
 354         }
 355         T_FileStream_writeLine(out, ";\n\n");
 356
 357         sprintf(
 358             buffer,
 359             "U_EXPORT struct {\n"
 360             "    uint16_t headerSize;\n"
 361             "    uint8_t magic1, magic2;\n"
 362             "    UDataInfo info;\n"
 363             "    char padding[%lu];\n"
 364             "    uint32_t count, reserved;\n"
 365             "    struct {\n"
 366             "        const char *name;\n"
 367             "        const void *data;\n"
 368             "    } toc[%lu];\n"
 369             "} U_EXPORT2 %s_dat = {\n"
 370             "    32, 0xda, 0x27, {\n"
 371             "        %lu, 0,\n"
 372             "        %u, %u, %u, 0,\n"
 373             "        {0x54, 0x6f, 0x43, 0x50},\n"
 374             "        {1, 0, 0, 0},\n"
 375             "        {0, 0, 0, 0}\n"
 376             "    },\n"
 377             "    \"\", %lu, 0, {\n",
 378             (unsigned long)32-4-sizeof(UDataInfo),
 379             (unsigned long)fileCount,
 380             entrypointName,
 381             (unsigned long)sizeof(UDataInfo),
 382             U_IS_BIG_ENDIAN,
 383             U_CHARSET_FAMILY,
 384             U_SIZEOF_UCHAR,
 385             (unsigned long)fileCount
 386         );
 387         T_FileStream_writeLine(out, buffer);
 388
 389         sprintf(buffer, "        { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname);
 390         T_FileStream_writeLine(out, buffer);
 391         for(i=1; i<fileCount; ++i) {
 392             sprintf(buffer, ",\n        { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname);
 393             T_FileStream_writeLine(out, buffer);
 394         }
 395
 396         T_FileStream_writeLine(out, "\n    }\n};\n");
 397         T_FileStream_close(out);
 398
 399         uprv_free(symPrefix);
 400     }
 401 }
 402
 403 static void
 404 addFile(const char *filename, const char *name, const char *source, UBool sourceTOC, UBool verbose) {
 405     char *s;
 406     uint32_t length;
 407     char *fullPath = NULL;
 408
 409     if(fileCount==fileMax) {
 410       fileMax += CHUNK_FILE_COUNT;
 411       files = uprv_realloc(files, fileMax*sizeof(files[0])); /* note: never freed. */
 412       if(files==NULL) {
 413         fprintf(stderr, "pkgdata/gencmn: Could not allocate %u bytes for %d files\n", (unsigned int)(fileMax*sizeof(files[0])), fileCount);
 414         exit(U_MEMORY_ALLOCATION_ERROR);
 415       }
 416     }
 417
 418     if(!sourceTOC) {
 419         FileStream *file;
 420
 421         if(uprv_pathIsAbsolute(filename)) {
 422             fprintf(stderr, "gencmn: Error: absolute path encountered. Old style paths are not supported. Use relative paths such as 'fur.res' or 'translit%cfur.res'.\n\tBad path: '%s'\n", U_FILE_SEP_CHAR, filename);
 423             exit(U_ILLEGAL_ARGUMENT_ERROR);
 424         }
 425         fullPath = pathToFullPath(filename, source);
 426         /* store the pathname */
 427         length = (uint32_t)(uprv_strlen(filename) + 1 + uprv_strlen(name) + 1);
 428         s=allocString(length);
 429         uprv_strcpy(s, name);
 430         uprv_strcat(s, U_TREE_ENTRY_SEP_STRING);
 431         uprv_strcat(s, filename);
 432
 433         /* get the basename */
 434         fixDirToTreePath(s);
 435         files[fileCount].basename=s;
 436         files[fileCount].basenameLength=length;
 437
 438         files[fileCount].pathname=fullPath;
 439
 440         basenameTotal+=length;
 441
 442         /* try to open the file */
 443         file=T_FileStream_open(fullPath, "rb");
 444         if(file==NULL) {
 445             fprintf(stderr, "gencmn: unable to open listed file %s\n", fullPath);
 446             exit(U_FILE_ACCESS_ERROR);
 447         }
 448
 449         /* get the file length */
 450         length=T_FileStream_size(file);
 451         if(T_FileStream_error(file) || length<=20) {
 452             fprintf(stderr, "gencmn: unable to get length of listed file %s\n", fullPath);
 453             exit(U_FILE_ACCESS_ERROR);
 454         }
 455
 456         T_FileStream_close(file);
 457
 458         /* do not add files that are longer than maxSize */
 459         if(maxSize && length>maxSize) {
 460             if (verbose) {
 461                 printf("%s ignored (size %ld > %ld)\n", fullPath, (long)length, (long)maxSize);
 462             }
 463             return;
 464         }
 465         files[fileCount].fileSize=length;
 466     } else {
 467         char *t;
 468         /* get and store the basename */
 469         /* need to include the package name */
 470         length = (uint32_t)(uprv_strlen(filename) + 1 + uprv_strlen(name) + 1);
 471         s=allocString(length);
 472         uprv_strcpy(s, name);
 473         uprv_strcat(s, U_TREE_ENTRY_SEP_STRING);
 474         uprv_strcat(s, filename);
 475         fixDirToTreePath(s);
 476         files[fileCount].basename=s;
 477         /* turn the basename into an entry point name and store in the pathname field */
 478         t=files[fileCount].pathname=allocString(length);
 479         while(--length>0) {
 480             if(*s=='.' || *s=='-' || *s=='/') {
 481                 *t='_';
 482             } else {
 483                 *t=*s;
 484             }
 485             ++s;
 486             ++t;
 487         }
 488         *t=0;
 489     }
 490     ++fileCount;
 491 }
 492
 493 static char *
 494 allocString(uint32_t length) {
 495     uint32_t top=stringTop+length;
 496     char *p;
 497
 498     if(top>STRING_STORE_SIZE) {
 499         fprintf(stderr, "gencmn: out of memory\n");
 500         exit(U_MEMORY_ALLOCATION_ERROR);
 501     }
 502     p=stringStore+stringTop;
 503     stringTop=top;
 504     return p;
 505 }
 506
 507 static char *
 508 pathToFullPath(const char *path, const char *source) {
 509     int32_t length;
 510     int32_t newLength;
 511     char *fullPath;
 512     int32_t n;
 513
 514     length = (uint32_t)(uprv_strlen(path) + 1);
 515     newLength = (length + 1 + (int32_t)uprv_strlen(source));
 516     fullPath = uprv_malloc(newLength);
 517     if(source != NULL) {
 518         uprv_strcpy(fullPath, source);
 519         uprv_strcat(fullPath, U_FILE_SEP_STRING);
 520     } else {
 521         fullPath[0] = 0;
 522     }
 523     n = (int32_t)uprv_strlen(fullPath);
 524     fullPath[n] = 0;       /* Suppress compiler warning for unused variable n    */
 525                            /*  when conditional code below is not compiled.      */
 526     uprv_strcat(fullPath, path);
 527
 528 #if (U_FILE_ALT_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
 529 #if (U_FILE_ALT_SEP_CHAR != U_FILE_SEP_CHAR)
 530     /* replace tree separator (such as '/') with file sep char (such as ':' or '\\') */
 531     for(;fullPath[n];n++) {
 532         if(fullPath[n] == U_FILE_ALT_SEP_CHAR) {
 533             fullPath[n] = U_FILE_SEP_CHAR;
 534         }
 535     }
 536 #endif
 537 #endif
 538 #if (U_FILE_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
 539     /* replace tree separator (such as '/') with file sep char (such as ':' or '\\') */
 540     for(;fullPath[n];n++) {
 541         if(fullPath[n] == U_TREE_ENTRY_SEP_CHAR) {
 542             fullPath[n] = U_FILE_SEP_CHAR;
 543         }
 544     }
 545 #endif
 546     return fullPath;
 547 }
 548
 549 static int
 550 compareFiles(const void *file1, const void *file2) {
 551     /* sort by basename */
 552     return uprv_strcmp(((File *)file1)->basename, ((File *)file2)->basename);
 553 }
 554
 555 static void
 556 fixDirToTreePath(char *s)
 557 {
 558 #if (U_FILE_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR) || ((U_FILE_ALT_SEP_CHAR != U_FILE_SEP_CHAR) && (U_FILE_ALT_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR))
 559     char *t;
 560 #endif
 561 #if (U_FILE_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
 562     for(t=s;t=uprv_strchr(t,U_FILE_SEP_CHAR);) {
 563         *t = U_TREE_ENTRY_SEP_CHAR;
 564     }
 565 #endif
 566 #if (U_FILE_ALT_SEP_CHAR != U_FILE_SEP_CHAR) && (U_FILE_ALT_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
 567     for(t=s;t=uprv_strchr(t,U_FILE_ALT_SEP_CHAR);) {
 568         *t = U_TREE_ENTRY_SEP_CHAR;
 569     }
 570 #endif
 571 }