icuSources/tools/makeconv/makeconv.c

   1 /*
   2  ********************************************************************************
   3  *
   4  *   Copyright (C) 1998-2012, International Business Machines
   5  *   Corporation and others.  All Rights Reserved.
   6  *
   7  ********************************************************************************
   8  *
   9  *
  10  *  makeconv.c:
  11  *  tool creating a binary (compressed) representation of the conversion mapping
  12  *  table (IBM NLTC ucmap format).
  13  *
  14  *  05/04/2000    helena     Added fallback mapping into the picture...
  15  *  06/29/2000  helena      Major rewrite of the callback APIs.
  16  */
  17
  18 #include <stdio.h>
  19 #include "unicode/putil.h"
  20 #include "unicode/ucnv_err.h"
  21 #include "ucnv_bld.h"
  22 #include "ucnv_imp.h"
  23 #include "ucnv_cnv.h"
  24 #include "cstring.h"
  25 #include "cmemory.h"
  26 #include "uinvchar.h"
  27 #include "filestrm.h"
  28 #include "toolutil.h"
  29 #include "uoptions.h"
  30 #include "unicode/udata.h"
  31 #include "unewdata.h"
  32 #include "uparse.h"
  33 #include "ucm.h"
  34 #include "makeconv.h"
  35 #include "genmbcs.h"
  36
  37 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  38
  39 #define DEBUG 0
  40
  41 typedef struct ConvData {
  42     UCMFile *ucm;
  43     NewConverter *cnvData, *extData;
  44     UConverterSharedData sharedData;
  45     UConverterStaticData staticData;
  46 } ConvData;
  47
  48 static void
  49 initConvData(ConvData *data) {
  50     uprv_memset(data, 0, sizeof(ConvData));
  51     data->sharedData.structSize=sizeof(UConverterSharedData);
  52     data->staticData.structSize=sizeof(UConverterStaticData);
  53     data->sharedData.staticData=&data->staticData;
  54 }
  55
  56 static void
  57 cleanupConvData(ConvData *data) {
  58     if(data!=NULL) {
  59         if(data->cnvData!=NULL) {
  60             data->cnvData->close(data->cnvData);
  61             data->cnvData=NULL;
  62         }
  63         if(data->extData!=NULL) {
  64             data->extData->close(data->extData);
  65             data->extData=NULL;
  66         }
  67         ucm_close(data->ucm);
  68         data->ucm=NULL;
  69     }
  70 }
  71
  72 /*
  73  * from ucnvstat.c - static prototypes of data-based converters
  74  */
  75 extern const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES];
  76
  77 /*
  78  * Global - verbosity
  79  */
  80 UBool VERBOSE = FALSE;
  81 UBool SMALL = FALSE;
  82 UBool IGNORE_SISO_CHECK = FALSE;
  83
  84 static void
  85 createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode);
  86
  87 /*
  88  * Set up the UNewData and write the converter..
  89  */
  90 static void
  91 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status);
  92
  93 UBool haveCopyright=TRUE;
  94
  95 static UDataInfo dataInfo={
  96     sizeof(UDataInfo),
  97     0,
  98
  99     U_IS_BIG_ENDIAN,
 100     U_CHARSET_FAMILY,
 101     sizeof(UChar),
 102     0,
 103
 104     {0x63, 0x6e, 0x76, 0x74},     /* dataFormat="cnvt" */
 105     {6, 2, 0, 0},                 /* formatVersion */
 106     {0, 0, 0, 0}                  /* dataVersion (calculated at runtime) */
 107 };
 108
 109 static void
 110 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status)
 111 {
 112     UNewDataMemory *mem = NULL;
 113     uint32_t sz2;
 114     uint32_t size = 0;
 115     int32_t tableType;
 116
 117     if(U_FAILURE(*status))
 118       {
 119         return;
 120       }
 121
 122     tableType=TABLE_NONE;
 123     if(data->cnvData!=NULL) {
 124         tableType|=TABLE_BASE;
 125     }
 126     if(data->extData!=NULL) {
 127         tableType|=TABLE_EXT;
 128     }
 129
 130     mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status);
 131
 132     if(U_FAILURE(*status))
 133       {
 134         fprintf(stderr, "Couldn't create the udata %s.%s: %s\n",
 135                 cnvName,
 136                 "cnv",
 137                 u_errorName(*status));
 138         return;
 139       }
 140
 141     if(VERBOSE)
 142       {
 143         printf("- Opened udata %s.%s\n", cnvName, "cnv");
 144       }
 145
 146
 147     /* all read only, clean, platform independent data.  Mmmm. :)  */
 148     udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData));
 149     size += sizeof(UConverterStaticData); /* Is 4-aligned  - by size */
 150     /* Now, write the table */
 151     if(tableType&TABLE_BASE) {
 152         size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType);
 153     }
 154     if(tableType&TABLE_EXT) {
 155         size += data->extData->write(data->extData, &data->staticData, mem, tableType);
 156     }
 157
 158     sz2 = udata_finish(mem, status);
 159     if(size != sz2)
 160     {
 161         fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size);
 162         *status=U_INTERNAL_PROGRAM_ERROR;
 163     }
 164     if(VERBOSE)
 165     {
 166       printf("- Wrote %u bytes to the udata.\n", (int)sz2);
 167     }
 168 }
 169
 170 enum {
 171     OPT_HELP_H,
 172     OPT_HELP_QUESTION_MARK,
 173     OPT_COPYRIGHT,
 174     OPT_VERSION,
 175     OPT_DESTDIR,
 176     OPT_VERBOSE,
 177     OPT_SMALL,
 178     OPT_IGNORE_SISO_CHECK,
 179     OPT_COUNT
 180 };
 181
 182 static UOption options[]={
 183     UOPTION_HELP_H,
 184     UOPTION_HELP_QUESTION_MARK,
 185     UOPTION_COPYRIGHT,
 186     UOPTION_VERSION,
 187     UOPTION_DESTDIR,
 188     UOPTION_VERBOSE,
 189     { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 },
 190     { "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }
 191 };
 192
 193 int main(int argc, char* argv[])
 194 {
 195     ConvData data;
 196     UErrorCode err = U_ZERO_ERROR, localError;
 197     char outFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 198     const char* destdir, *arg;
 199     size_t destdirlen;
 200     char* dot = NULL, *outBasename;
 201     char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 202     char cnvNameWithPkg[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 203     UVersionInfo icuVersion;
 204     UBool printFilename;
 205
 206     err = U_ZERO_ERROR;
 207
 208     U_MAIN_INIT_ARGS(argc, argv);
 209
 210     /* Set up the ICU version number */
 211     u_getVersion(icuVersion);
 212     uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));
 213
 214     /* preset then read command line options */
 215     options[OPT_DESTDIR].value=u_getDataDirectory();
 216     argc=u_parseArgs(argc, argv, LENGTHOF(options), options);
 217
 218     /* error handling, printing usage message */
 219     if(argc<0) {
 220         fprintf(stderr,
 221             "error in command line argument \"%s\"\n",
 222             argv[-argc]);
 223     } else if(argc<2) {
 224         argc=-1;
 225     }
 226     if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) {
 227         FILE *stdfile=argc<0 ? stderr : stdout;
 228         fprintf(stdfile,
 229             "usage: %s [-options] files...\n"
 230             "\tread .ucm codepage mapping files and write .cnv files\n"
 231             "options:\n"
 232             "\t-h or -? or --help  this usage text\n"
 233             "\t-V or --version     show a version message\n"
 234             "\t-c or --copyright   include a copyright notice\n"
 235             "\t-d or --destdir     destination directory, followed by the path\n"
 236             "\t-v or --verbose     Turn on verbose output\n",
 237             argv[0]);
 238         fprintf(stdfile,
 239             "\t      --small       Generate smaller .cnv files. They will be\n"
 240             "\t                    significantly smaller but may not be compatible with\n"
 241             "\t                    older versions of ICU and will require heap memory\n"
 242             "\t                    allocation when loaded.\n"
 243             "\t      --ignore-siso-check         Use SI/SO other than 0xf/0xe.\n");
 244         return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
 245     }
 246
 247     if(options[OPT_VERSION].doesOccur) {
 248         printf("makeconv version %u.%u, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
 249                dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
 250         printf("%s\n", U_COPYRIGHT_STRING);
 251         exit(0);
 252     }
 253
 254     /* get the options values */
 255     haveCopyright = options[OPT_COPYRIGHT].doesOccur;
 256     destdir = options[OPT_DESTDIR].value;
 257     VERBOSE = options[OPT_VERBOSE].doesOccur;
 258     SMALL = options[OPT_SMALL].doesOccur;
 259
 260     if (options[OPT_IGNORE_SISO_CHECK].doesOccur) {
 261         IGNORE_SISO_CHECK = TRUE;
 262     }
 263
 264     if (destdir != NULL && *destdir != 0) {
 265         uprv_strcpy(outFileName, destdir);
 266         destdirlen = uprv_strlen(destdir);
 267         outBasename = outFileName + destdirlen;
 268         if (*(outBasename - 1) != U_FILE_SEP_CHAR) {
 269             *outBasename++ = U_FILE_SEP_CHAR;
 270             ++destdirlen;
 271         }
 272     } else {
 273         destdirlen = 0;
 274         outBasename = outFileName;
 275     }
 276
 277 #if DEBUG
 278     {
 279       int i;
 280       printf("makeconv: processing %d files...\n", argc - 1);
 281       for(i=1; i<argc; ++i) {
 282         printf("%s ", argv[i]);
 283       }
 284       printf("\n");
 285       fflush(stdout);
 286     }
 287 #endif
 288
 289     err = U_ZERO_ERROR;
 290     printFilename = (UBool) (argc > 2 || VERBOSE);
 291     for (++argv; --argc; ++argv)
 292     {
 293         arg = getLongPathname(*argv);
 294
 295         /* Check for potential buffer overflow */
 296         if(strlen(arg) >= UCNV_MAX_FULL_FILE_NAME_LENGTH)
 297         {
 298             fprintf(stderr, "%s\n", u_errorName(U_BUFFER_OVERFLOW_ERROR));
 299             return U_BUFFER_OVERFLOW_ERROR;
 300         }
 301
 302         /*produces the right destination path for display*/
 303         if (destdirlen != 0)
 304         {
 305             const char *basename;
 306
 307             /* find the last file sepator */
 308             basename = findBasename(arg);
 309             uprv_strcpy(outBasename, basename);
 310         }
 311         else
 312         {
 313             uprv_strcpy(outFileName, arg);
 314         }
 315
 316         /*removes the extension if any is found*/
 317         dot = uprv_strrchr(outBasename, '.');
 318         if (dot)
 319         {
 320             *dot = '\0';
 321         }
 322
 323         /* the basename without extension is the converter name */
 324         uprv_strcpy(cnvName, outBasename);
 325
 326         /*Adds the target extension*/
 327         uprv_strcat(outBasename, CONVERTER_FILE_EXTENSION);
 328
 329 #if DEBUG
 330         printf("makeconv: processing %s  ...\n", arg);
 331         fflush(stdout);
 332 #endif
 333         localError = U_ZERO_ERROR;
 334         initConvData(&data);
 335         createConverter(&data, arg, &localError);
 336
 337         if (U_FAILURE(localError))
 338         {
 339             /* if an error is found, print out an error msg and keep going */
 340             fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
 341                 u_errorName(localError));
 342             if(U_SUCCESS(err)) {
 343                 err = localError;
 344             }
 345         }
 346         else
 347         {
 348             /* Insure the static data name matches the  file name */
 349             /* Changed to ignore directory and only compare base name
 350              LDH 1/2/08*/
 351             char *p;
 352             p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */
 353
 354             if(p == NULL)            /* OK, try alternate */
 355             {
 356                 p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR);
 357                 if(p == NULL)
 358                 {
 359                     p=cnvName; /* If no separators, no problem */
 360                 }
 361             }
 362             else
 363             {
 364                 p++;   /* If found separtor, don't include it in compare */
 365             }
 366             if(uprv_stricmp(p,data.staticData.name))
 367             {
 368                 fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
 369                     cnvName,  CONVERTER_FILE_EXTENSION,
 370                     data.staticData.name);
 371             }
 372
 373             uprv_strcpy((char*)data.staticData.name, cnvName);
 374
 375             if(!uprv_isInvariantString((char*)data.staticData.name, -1)) {
 376                 fprintf(stderr,
 377                     "Error: A converter name must contain only invariant characters.\n"
 378                     "%s is not a valid converter name.\n",
 379                     data.staticData.name);
 380                 if(U_SUCCESS(err)) {
 381                     err = U_INVALID_TABLE_FORMAT;
 382                 }
 383             }
 384
 385             uprv_strcpy(cnvNameWithPkg, cnvName);
 386
 387             localError = U_ZERO_ERROR;
 388             writeConverterData(&data, cnvNameWithPkg, destdir, &localError);
 389
 390             if(U_FAILURE(localError))
 391             {
 392                 /* if an error is found, print out an error msg and keep going*/
 393                 fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
 394                     u_errorName(localError));
 395                 if(U_SUCCESS(err)) {
 396                     err = localError;
 397                 }
 398             }
 399             else if (printFilename)
 400             {
 401                 puts(outBasename);
 402             }
 403         }
 404         fflush(stdout);
 405         fflush(stderr);
 406
 407         cleanupConvData(&data);
 408     }
 409
 410     return err;
 411 }
 412
 413 static void
 414 getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) {
 415     if( (name[0]=='i' || name[0]=='I') &&
 416         (name[1]=='b' || name[1]=='B') &&
 417         (name[2]=='m' || name[2]=='M')
 418     ) {
 419         name+=3;
 420         if(*name=='-') {
 421             ++name;
 422         }
 423         *pPlatform=UCNV_IBM;
 424         *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10);
 425     } else {
 426         *pPlatform=UCNV_UNKNOWN;
 427         *pCCSID=0;
 428     }
 429 }
 430
 431 static void
 432 readHeader(ConvData *data,
 433            FileStream* convFile,
 434            const char* converterName,
 435            UErrorCode *pErrorCode) {
 436     char line[1024];
 437     char *s, *key, *value;
 438     const UConverterStaticData *prototype;
 439     UConverterStaticData *staticData;
 440
 441     if(U_FAILURE(*pErrorCode)) {
 442         return;
 443     }
 444
 445     staticData=&data->staticData;
 446     staticData->platform=UCNV_IBM;
 447     staticData->subCharLen=0;
 448
 449     while(T_FileStream_readLine(convFile, line, sizeof(line))) {
 450         /* basic parsing and handling of state-related items */
 451         if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) {
 452             continue;
 453         }
 454
 455         /* stop at the beginning of the mapping section */
 456         if(uprv_strcmp(line, "CHARMAP")==0) {
 457             break;
 458         }
 459
 460         /* collect the information from the header field, ignore unknown keys */
 461         if(uprv_strcmp(key, "code_set_name")==0) {
 462             if(*value!=0) {
 463                 uprv_strcpy((char *)staticData->name, value);
 464                 getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
 465             }
 466         } else if(uprv_strcmp(key, "subchar")==0) {
 467             uint8_t bytes[UCNV_EXT_MAX_BYTES];
 468             int8_t length;
 469
 470             s=value;
 471             length=ucm_parseBytes(bytes, line, (const char **)&s);
 472             if(1<=length && length<=4 && *s==0) {
 473                 staticData->subCharLen=length;
 474                 uprv_memcpy(staticData->subChar, bytes, length);
 475             } else {
 476                 fprintf(stderr, "error: illegal <subchar> %s\n", value);
 477                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 478                 return;
 479             }
 480         } else if(uprv_strcmp(key, "subchar1")==0) {
 481             uint8_t bytes[UCNV_EXT_MAX_BYTES];
 482
 483             s=value;
 484             if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) {
 485                 staticData->subChar1=bytes[0];
 486             } else {
 487                 fprintf(stderr, "error: illegal <subchar1> %s\n", value);
 488                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 489                 return;
 490             }
 491         }
 492     }
 493
 494     /* copy values from the UCMFile to the static data */
 495     staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength;
 496     staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength;
 497     staticData->conversionType=data->ucm->states.conversionType;
 498
 499     if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
 500         fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n");
 501         *pErrorCode=U_INVALID_TABLE_FORMAT;
 502         return;
 503     }
 504
 505     /*
 506      * Now that we know the type, copy any 'default' values from the table.
 507      * We need not check the type any further because the parser only
 508      * recognizes what we have prototypes for.
 509      *
 510      * For delta (extension-only) tables, copy values from the base file
 511      * instead, see createConverter().
 512      */
 513     if(data->ucm->baseName[0]==0) {
 514         prototype=ucnv_converterStaticData[staticData->conversionType];
 515         if(prototype!=NULL) {
 516             if(staticData->name[0]==0) {
 517                 uprv_strcpy((char *)staticData->name, prototype->name);
 518             }
 519
 520             if(staticData->codepage==0) {
 521                 staticData->codepage=prototype->codepage;
 522             }
 523
 524             if(staticData->platform==0) {
 525                 staticData->platform=prototype->platform;
 526             }
 527
 528             if(staticData->minBytesPerChar==0) {
 529                 staticData->minBytesPerChar=prototype->minBytesPerChar;
 530             }
 531
 532             if(staticData->maxBytesPerChar==0) {
 533                 staticData->maxBytesPerChar=prototype->maxBytesPerChar;
 534             }
 535
 536             if(staticData->subCharLen==0) {
 537                 staticData->subCharLen=prototype->subCharLen;
 538                 if(prototype->subCharLen>0) {
 539                     uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
 540                 }
 541             }
 542         }
 543     }
 544
 545     if(data->ucm->states.outputType<0) {
 546         data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1;
 547     }
 548
 549     if( staticData->subChar1!=0 &&
 550             (staticData->minBytesPerChar>1 ||
 551                 (staticData->conversionType!=UCNV_MBCS &&
 552                  staticData->conversionType!=UCNV_EBCDIC_STATEFUL))
 553     ) {
 554         fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n");
 555         *pErrorCode=U_INVALID_TABLE_FORMAT;
 556     }
 557 }
 558
 559 /* return TRUE if a base table was read, FALSE for an extension table */
 560 static UBool
 561 readFile(ConvData *data, const char* converterName,
 562          UErrorCode *pErrorCode) {
 563     char line[1024];
 564     char *end;
 565     FileStream *convFile;
 566
 567     UCMStates *baseStates;
 568     UBool dataIsBase;
 569
 570     if(U_FAILURE(*pErrorCode)) {
 571         return FALSE;
 572     }
 573
 574     data->ucm=ucm_open();
 575
 576     convFile=T_FileStream_open(converterName, "r");
 577     if(convFile==NULL) {
 578         *pErrorCode=U_FILE_ACCESS_ERROR;
 579         return FALSE;
 580     }
 581
 582     readHeader(data, convFile, converterName, pErrorCode);
 583     if(U_FAILURE(*pErrorCode)) {
 584         return FALSE;
 585     }
 586
 587     if(data->ucm->baseName[0]==0) {
 588         dataIsBase=TRUE;
 589         baseStates=&data->ucm->states;
 590         ucm_processStates(baseStates, IGNORE_SISO_CHECK);
 591     } else {
 592         dataIsBase=FALSE;
 593         baseStates=NULL;
 594     }
 595
 596     /* read the base table */
 597     ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode);
 598     if(U_FAILURE(*pErrorCode)) {
 599         return FALSE;
 600     }
 601
 602     /* read an extension table if there is one */
 603     while(T_FileStream_readLine(convFile, line, sizeof(line))) {
 604         end=uprv_strchr(line, 0);
 605         while(line<end &&
 606               (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) {
 607             --end;
 608         }
 609         *end=0;
 610
 611         if(line[0]=='#' || u_skipWhitespace(line)==end) {
 612             continue; /* ignore empty and comment lines */
 613         }
 614
 615         if(0==uprv_strcmp(line, "CHARMAP")) {
 616             /* read the extension table */
 617             ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode);
 618         } else {
 619             fprintf(stderr, "unexpected text after the base mapping table\n");
 620         }
 621         break;
 622     }
 623
 624     T_FileStream_close(convFile);
 625
 626     if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) {
 627         fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
 628         *pErrorCode=U_INVALID_TABLE_FORMAT;
 629     }
 630
 631     return dataIsBase;
 632 }
 633
 634 static void
 635 createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) {
 636     ConvData baseData;
 637     UBool dataIsBase;
 638
 639     UConverterStaticData *staticData;
 640     UCMStates *states, *baseStates;
 641
 642     if(U_FAILURE(*pErrorCode)) {
 643         return;
 644     }
 645
 646     initConvData(data);
 647
 648     dataIsBase=readFile(data, converterName, pErrorCode);
 649     if(U_FAILURE(*pErrorCode)) {
 650         return;
 651     }
 652
 653     staticData=&data->staticData;
 654     states=&data->ucm->states;
 655
 656     if(dataIsBase) {
 657         /*
 658          * Build a normal .cnv file with a base table
 659          * and an optional extension table.
 660          */
 661         data->cnvData=MBCSOpen(data->ucm);
 662         if(data->cnvData==NULL) {
 663             *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 664
 665         } else if(!data->cnvData->isValid(data->cnvData,
 666                             staticData->subChar, staticData->subCharLen)
 667         ) {
 668             fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
 669             *pErrorCode=U_INVALID_TABLE_FORMAT;
 670
 671         } else if(staticData->subChar1!=0 &&
 672                     !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1)
 673         ) {
 674             fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
 675             *pErrorCode=U_INVALID_TABLE_FORMAT;
 676
 677         } else if(
 678             data->ucm->ext->mappingsLength>0 &&
 679             !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
 680         ) {
 681             *pErrorCode=U_INVALID_TABLE_FORMAT;
 682         } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) {
 683             /* sort the table so that it can be turned into UTF-8-friendly data */
 684             ucm_sortTable(data->ucm->base);
 685         }
 686
 687         if(U_SUCCESS(*pErrorCode)) {
 688             if(
 689                 /* add the base table after ucm_checkBaseExt()! */
 690                 !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData)
 691             ) {
 692                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 693             } else {
 694                 /*
 695                  * addTable() may have requested moving more mappings to the extension table
 696                  * if they fit into the base toUnicode table but not into the
 697                  * base fromUnicode table.
 698                  * (Especially for UTF-8-friendly fromUnicode tables.)
 699                  * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them
 700                  * to be excluded from the extension toUnicode data.
 701                  * See MBCSOkForBaseFromUnicode() for which mappings do not fit into
 702                  * the base fromUnicode table.
 703                  */
 704                 ucm_moveMappings(data->ucm->base, data->ucm->ext);
 705                 ucm_sortTable(data->ucm->ext);
 706                 if(data->ucm->ext->mappingsLength>0) {
 707                     /* prepare the extension table, if there is one */
 708                     data->extData=CnvExtOpen(data->ucm);
 709                     if(data->extData==NULL) {
 710                         *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 711                     } else if(
 712                         !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)
 713                     ) {
 714                         *pErrorCode=U_INVALID_TABLE_FORMAT;
 715                     }
 716                 }
 717             }
 718         }
 719     } else {
 720         /* Build an extension-only .cnv file. */
 721         char baseFilename[500];
 722         char *basename;
 723
 724         initConvData(&baseData);
 725
 726         /* assemble a path/filename for data->ucm->baseName */
 727         uprv_strcpy(baseFilename, converterName);
 728         basename=(char *)findBasename(baseFilename);
 729         uprv_strcpy(basename, data->ucm->baseName);
 730         uprv_strcat(basename, ".ucm");
 731
 732         /* read the base table */
 733         dataIsBase=readFile(&baseData, baseFilename, pErrorCode);
 734         if(U_FAILURE(*pErrorCode)) {
 735             return;
 736         } else if(!dataIsBase) {
 737             fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename);
 738             *pErrorCode=U_INVALID_TABLE_FORMAT;
 739         } else {
 740             /* prepare the extension table */
 741             data->extData=CnvExtOpen(data->ucm);
 742             if(data->extData==NULL) {
 743                 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 744             } else {
 745                 /* fill in gaps in extension file header fields */
 746                 UCMapping *m, *mLimit;
 747                 uint8_t fallbackFlags;
 748
 749                 baseStates=&baseData.ucm->states;
 750                 if(states->conversionType==UCNV_DBCS) {
 751                     staticData->minBytesPerChar=(int8_t)(states->minCharLength=2);
 752                 } else if(states->minCharLength==0) {
 753                     staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength);
 754                 }
 755                 if(states->maxCharLength<states->minCharLength) {
 756                     staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength);
 757                 }
 758
 759                 if(staticData->subCharLen==0) {
 760                     uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4);
 761                     staticData->subCharLen=baseData.staticData.subCharLen;
 762                 }
 763                 /*
 764                  * do not copy subChar1 -
 765                  * only use what is explicitly specified
 766                  * because it cannot be unset in the extension file header
 767                  */
 768
 769                 /* get the fallback flags */
 770                 fallbackFlags=0;
 771                 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
 772                     m<mLimit && fallbackFlags!=3;
 773                     ++m
 774                 ) {
 775                     if(m->f==1) {
 776                         fallbackFlags|=1;
 777                     } else if(m->f==3) {
 778                         fallbackFlags|=2;
 779                     }
 780                 }
 781
 782                 if(fallbackFlags&1) {
 783                     staticData->hasFromUnicodeFallback=TRUE;
 784                 }
 785                 if(fallbackFlags&2) {
 786                     staticData->hasToUnicodeFallback=TRUE;
 787                 }
 788
 789                 if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) {
 790                     fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
 791                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 792
 793                 } else if(staticData->subChar1!=0 && 1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) {
 794                     fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
 795                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 796
 797                 } else if(
 798                     !ucm_checkValidity(data->ucm->ext, baseStates) ||
 799                     !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
 800                 ) {
 801                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 802                 } else {
 803                     if(states->maxCharLength>1) {
 804                         /*
 805                          * When building a normal .cnv file with a base table
 806                          * for an MBCS (not SBCS) table with explicit precision flags,
 807                          * the MBCSAddTable() function marks some mappings for moving
 808                          * to the extension table.
 809                          * They fit into the base toUnicode table but not into the
 810                          * base fromUnicode table.
 811                          * (Note: We do have explicit precision flags because they are
 812                          * required for extension table generation, and
 813                          * ucm_checkBaseExt() verified it.)
 814                          *
 815                          * We do not call MBCSAddTable() here (we probably could)
 816                          * so we need to do the analysis before building the extension table.
 817                          * We assume that MBCSAddTable() will build a UTF-8-friendly table.
 818                          * Redundant mappings in the extension table are ok except they cost some size.
 819                          *
 820                          * Do this after ucm_checkBaseExt().
 821                          */
 822                         const MBCSData *mbcsData=MBCSGetDummy();
 823                         int32_t needsMove=0;
 824                         for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
 825                             m<mLimit;
 826                             ++m
 827                         ) {
 828                             if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) {
 829                                 m->f|=MBCS_FROM_U_EXT_FLAG;
 830                                 m->moveFlag=UCM_MOVE_TO_EXT;
 831                                 ++needsMove;
 832                             }
 833                         }
 834
 835                         if(needsMove!=0) {
 836                             ucm_moveMappings(baseData.ucm->base, data->ucm->ext);
 837                             ucm_sortTable(data->ucm->ext);
 838                         }
 839                     }
 840                     if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) {
 841                         *pErrorCode=U_INVALID_TABLE_FORMAT;
 842                     }
 843                 }
 844             }
 845         }
 846
 847         cleanupConvData(&baseData);
 848     }
 849 }
 850
 851 /*
 852  * Hey, Emacs, please set the following:
 853  *
 854  * Local Variables:
 855  * indent-tabs-mode: nil
 856  * End:
 857  *
 858  */