icuSources/tools/makeconv/makeconv.c

   1 /*
   2  ********************************************************************************
   3  *
   4  *   Copyright (C) 1998-2008, International Business Machines
   5  *   Corporation and others.  All Rights Reserved.
   6  *
   7  ********************************************************************************
   8  *
   9  *
  10  *  makeconv.c:
  11  *  tool creating a binary (compressed) representation of the conversion mapping
  12  *  table (IBM NLTC ucmap format).
  13  *
  14  *  05/04/2000    helena     Added fallback mapping into the picture...
  15  *  06/29/2000  helena      Major rewrite of the callback APIs.
  16  */
  17
  18 #include <stdio.h>
  19 #include "unicode/putil.h"
  20 #include "unicode/ucnv_err.h"
  21 #include "ucnv_bld.h"
  22 #include "ucnv_imp.h"
  23 #include "ucnv_cnv.h"
  24 #include "cstring.h"
  25 #include "cmemory.h"
  26 #include "uinvchar.h"
  27 #include "filestrm.h"
  28 #include "toolutil.h"
  29 #include "uoptions.h"
  30 #include "unicode/udata.h"
  31 #include "unewdata.h"
  32 #include "uparse.h"
  33 #include "ucm.h"
  34 #include "makeconv.h"
  35 #include "genmbcs.h"
  36
  37 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  38
  39 #define DEBUG 0
  40
  41 typedef struct ConvData {
  42     UCMFile *ucm;
  43     NewConverter *cnvData, *extData;
  44     UConverterSharedData sharedData;
  45     UConverterStaticData staticData;
  46 } ConvData;
  47
  48 static void
  49 initConvData(ConvData *data) {
  50     uprv_memset(data, 0, sizeof(ConvData));
  51     data->sharedData.structSize=sizeof(UConverterSharedData);
  52     data->staticData.structSize=sizeof(UConverterStaticData);
  53     data->sharedData.staticData=&data->staticData;
  54 }
  55
  56 static void
  57 cleanupConvData(ConvData *data) {
  58     if(data!=NULL) {
  59         if(data->cnvData!=NULL) {
  60             data->cnvData->close(data->cnvData);
  61             data->cnvData=NULL;
  62         }
  63         if(data->extData!=NULL) {
  64             data->extData->close(data->extData);
  65             data->extData=NULL;
  66         }
  67         ucm_close(data->ucm);
  68         data->ucm=NULL;
  69     }
  70 }
  71
  72 /*
  73  * from ucnvstat.c - static prototypes of data-based converters
  74  */
  75 extern const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES];
  76
  77 /*
  78  * Global - verbosity
  79  */
  80 UBool VERBOSE = FALSE;
  81 UBool SMALL = FALSE;
  82
  83 static void
  84 createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode);
  85
  86 /*
  87  * Set up the UNewData and write the converter..
  88  */
  89 static void
  90 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status);
  91
  92 UBool haveCopyright=TRUE;
  93
  94 static UDataInfo dataInfo={
  95     sizeof(UDataInfo),
  96     0,
  97
  98     U_IS_BIG_ENDIAN,
  99     U_CHARSET_FAMILY,
 100     sizeof(UChar),
 101     0,
 102
 103     {0x63, 0x6e, 0x76, 0x74},     /* dataFormat="cnvt" */
 104     {6, 2, 0, 0},                 /* formatVersion */
 105     {0, 0, 0, 0}                  /* dataVersion (calculated at runtime) */
 106 };
 107
 108 static void
 109 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status)
 110 {
 111     UNewDataMemory *mem = NULL;
 112     uint32_t sz2;
 113     uint32_t size = 0;
 114     int32_t tableType;
 115
 116     if(U_FAILURE(*status))
 117       {
 118         return;
 119       }
 120
 121     tableType=TABLE_NONE;
 122     if(data->cnvData!=NULL) {
 123         tableType|=TABLE_BASE;
 124     }
 125     if(data->extData!=NULL) {
 126         tableType|=TABLE_EXT;
 127     }
 128
 129     mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status);
 130
 131     if(U_FAILURE(*status))
 132       {
 133         fprintf(stderr, "Couldn't create the udata %s.%s: %s\n",
 134                 cnvName,
 135                 "cnv",
 136                 u_errorName(*status));
 137         return;
 138       }
 139
 140     if(VERBOSE)
 141       {
 142         printf("- Opened udata %s.%s\n", cnvName, "cnv");
 143       }
 144
 145
 146     /* all read only, clean, platform independent data.  Mmmm. :)  */
 147     udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData));
 148     size += sizeof(UConverterStaticData); /* Is 4-aligned  - by size */
 149     /* Now, write the table */
 150     if(tableType&TABLE_BASE) {
 151         size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType);
 152     }
 153     if(tableType&TABLE_EXT) {
 154         size += data->extData->write(data->extData, &data->staticData, mem, tableType);
 155     }
 156
 157     sz2 = udata_finish(mem, status);
 158     if(size != sz2)
 159     {
 160         fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size);
 161         *status=U_INTERNAL_PROGRAM_ERROR;
 162     }
 163     if(VERBOSE)
 164     {
 165       printf("- Wrote %u bytes to the udata.\n", (int)sz2);
 166     }
 167 }
 168
 169 enum {
 170     OPT_HELP_H,
 171     OPT_HELP_QUESTION_MARK,
 172     OPT_COPYRIGHT,
 173     OPT_VERSION,
 174     OPT_DESTDIR,
 175     OPT_VERBOSE,
 176     OPT_SMALL,
 177     OPT_COUNT
 178 };
 179
 180 static UOption options[]={
 181     UOPTION_HELP_H,
 182     UOPTION_HELP_QUESTION_MARK,
 183     UOPTION_COPYRIGHT,
 184     UOPTION_VERSION,
 185     UOPTION_DESTDIR,
 186     UOPTION_VERBOSE,
 187     { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }
 188 };
 189
 190 int main(int argc, char* argv[])
 191 {
 192     ConvData data;
 193     UErrorCode err = U_ZERO_ERROR, localError;
 194     char outFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 195     const char* destdir, *arg;
 196     size_t destdirlen;
 197     char* dot = NULL, *outBasename;
 198     char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 199     char cnvNameWithPkg[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 200     UVersionInfo icuVersion;
 201     UBool printFilename;
 202
 203     err = U_ZERO_ERROR;
 204
 205     U_MAIN_INIT_ARGS(argc, argv);
 206
 207     /* Set up the ICU version number */
 208     u_getVersion(icuVersion);
 209     uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));
 210
 211     /* preset then read command line options */
 212     options[OPT_DESTDIR].value=u_getDataDirectory();
 213     argc=u_parseArgs(argc, argv, LENGTHOF(options), options);
 214
 215     /* error handling, printing usage message */
 216     if(argc<0) {
 217         fprintf(stderr,
 218             "error in command line argument \"%s\"\n",
 219             argv[-argc]);
 220     } else if(argc<2) {
 221         argc=-1;
 222     }
 223     if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) {
 224         FILE *stdfile=argc<0 ? stderr : stdout;
 225         fprintf(stdfile,
 226             "usage: %s [-options] files...\n"
 227             "\tread .ucm codepage mapping files and write .cnv files\n"
 228             "options:\n"
 229             "\t-h or -? or --help  this usage text\n"
 230             "\t-V or --version     show a version message\n"
 231             "\t-c or --copyright   include a copyright notice\n"
 232             "\t-d or --destdir     destination directory, followed by the path\n"
 233             "\t-v or --verbose     Turn on verbose output\n",
 234             argv[0]);
 235         fprintf(stdfile,
 236             "\t      --small       Generate smaller .cnv files. They will be\n"
 237             "\t                    significantly smaller but may not be compatible with\n"
 238             "\t                    older versions of ICU and will require heap memory\n"
 239             "\t                    allocation when loaded.\n");
 240         return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
 241     }
 242
 243     if(options[OPT_VERSION].doesOccur) {
 244         printf("makeconv version %hu.%hu, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
 245                dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
 246         printf("%s\n", U_COPYRIGHT_STRING);
 247         exit(0);
 248     }
 249
 250     /* get the options values */
 251     haveCopyright = options[OPT_COPYRIGHT].doesOccur;
 252     destdir = options[OPT_DESTDIR].value;
 253     VERBOSE = options[OPT_VERBOSE].doesOccur;
 254     SMALL = options[OPT_SMALL].doesOccur;
 255
 256     if (destdir != NULL && *destdir != 0) {
 257         uprv_strcpy(outFileName, destdir);
 258         destdirlen = uprv_strlen(destdir);
 259         outBasename = outFileName + destdirlen;
 260         if (*(outBasename - 1) != U_FILE_SEP_CHAR) {
 261             *outBasename++ = U_FILE_SEP_CHAR;
 262             ++destdirlen;
 263         }
 264     } else {
 265         destdirlen = 0;
 266         outBasename = outFileName;
 267     }
 268
 269 #if DEBUG
 270     {
 271       int i;
 272       printf("makeconv: processing %d files...\n", argc - 1);
 273       for(i=1; i<argc; ++i) {
 274         printf("%s ", argv[i]);
 275       }
 276       printf("\n");
 277       fflush(stdout);
 278     }
 279 #endif
 280
 281     err = U_ZERO_ERROR;
 282     printFilename = (UBool) (argc > 2 || VERBOSE);
 283     for (++argv; --argc; ++argv)
 284     {
 285         arg = getLongPathname(*argv);
 286
 287         /* Check for potential buffer overflow */
 288         if(strlen(arg) > UCNV_MAX_FULL_FILE_NAME_LENGTH)
 289         {
 290             fprintf(stderr, "%s\n", u_errorName(U_BUFFER_OVERFLOW_ERROR));
 291             return U_BUFFER_OVERFLOW_ERROR;
 292         }
 293
 294         /*produces the right destination path for display*/
 295         if (destdirlen != 0)
 296         {
 297             const char *basename;
 298
 299             /* find the last file sepator */
 300             basename = findBasename(arg);
 301             uprv_strcpy(outBasename, basename);
 302         }
 303         else
 304         {
 305             uprv_strcpy(outFileName, arg);
 306         }
 307
 308         /*removes the extension if any is found*/
 309         dot = uprv_strrchr(outBasename, '.');
 310         if (dot)
 311         {
 312             *dot = '\0';
 313         }
 314
 315         /* the basename without extension is the converter name */
 316         uprv_strcpy(cnvName, outBasename);
 317
 318         /*Adds the target extension*/
 319         uprv_strcat(outBasename, CONVERTER_FILE_EXTENSION);
 320
 321 #if DEBUG
 322         printf("makeconv: processing %s  ...\n", arg);
 323         fflush(stdout);
 324 #endif
 325         localError = U_ZERO_ERROR;
 326         initConvData(&data);
 327         createConverter(&data, arg, &localError);
 328
 329         if (U_FAILURE(localError))
 330         {
 331             /* if an error is found, print out an error msg and keep going */
 332             fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
 333                 u_errorName(localError));
 334             if(U_SUCCESS(err)) {
 335                 err = localError;
 336             }
 337         }
 338         else
 339         {
 340             /* Insure the static data name matches the  file name */
 341             /* Changed to ignore directory and only compare base name
 342              LDH 1/2/08*/
 343             char *p;
 344             p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */
 345
 346             if(p == NULL)            /* OK, try alternate */
 347             {
 348                 p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR);
 349                 if(p == NULL)
 350                 {
 351                     p=cnvName; /* If no separators, no problem */
 352                 }
 353             }
 354             else
 355             {
 356                 p++;   /* If found separtor, don't include it in compare */
 357             }
 358             if(uprv_stricmp(p,data.staticData.name))
 359             {
 360                 fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
 361                     cnvName,  CONVERTER_FILE_EXTENSION,
 362                     data.staticData.name);
 363             }
 364
 365             uprv_strcpy((char*)data.staticData.name, cnvName);
 366
 367             if(!uprv_isInvariantString((char*)data.staticData.name, -1)) {
 368                 fprintf(stderr,
 369                     "Error: A converter name must contain only invariant characters.\n"
 370                     "%s is not a valid converter name.\n",
 371                     data.staticData.name);
 372                 if(U_SUCCESS(err)) {
 373                     err = U_INVALID_TABLE_FORMAT;
 374                 }
 375             }
 376
 377             uprv_strcpy(cnvNameWithPkg, cnvName);
 378
 379             localError = U_ZERO_ERROR;
 380             writeConverterData(&data, cnvNameWithPkg, destdir, &localError);
 381
 382             if(U_FAILURE(localError))
 383             {
 384                 /* if an error is found, print out an error msg and keep going*/
 385                 fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
 386                     u_errorName(localError));
 387                 if(U_SUCCESS(err)) {
 388                     err = localError;
 389                 }
 390             }
 391             else if (printFilename)
 392             {
 393                 puts(outBasename);
 394             }
 395         }
 396         fflush(stdout);
 397         fflush(stderr);
 398
 399         cleanupConvData(&data);
 400     }
 401
 402     return err;
 403 }
 404
 405 static void
 406 getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) {
 407     if( (name[0]=='i' || name[0]=='I') &&
 408         (name[1]=='b' || name[1]=='B') &&
 409         (name[2]=='m' || name[2]=='M')
 410     ) {
 411         name+=3;
 412         if(*name=='-') {
 413             ++name;
 414         }
 415         *pPlatform=UCNV_IBM;
 416         *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10);
 417     } else {
 418         *pPlatform=UCNV_UNKNOWN;
 419         *pCCSID=0;
 420     }
 421 }
 422
 423 static void
 424 readHeader(ConvData *data,
 425            FileStream* convFile,
 426            const char* converterName,
 427            UErrorCode *pErrorCode) {
 428     char line[200];
 429     char *s, *key, *value;
 430     const UConverterStaticData *prototype;
 431     UConverterStaticData *staticData;
 432
 433     if(U_FAILURE(*pErrorCode)) {
 434         return;
 435     }
 436
 437     staticData=&data->staticData;
 438     staticData->platform=UCNV_IBM;
 439     staticData->subCharLen=0;
 440
 441     while(T_FileStream_readLine(convFile, line, sizeof(line))) {
 442         /* basic parsing and handling of state-related items */
 443         if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) {
 444             continue;
 445         }
 446
 447         /* stop at the beginning of the mapping section */
 448         if(uprv_strcmp(line, "CHARMAP")==0) {
 449             break;
 450         }
 451
 452         /* collect the information from the header field, ignore unknown keys */
 453         if(uprv_strcmp(key, "code_set_name")==0) {
 454             if(*value!=0) {
 455                 uprv_strcpy((char *)staticData->name, value);
 456                 getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
 457             }
 458         } else if(uprv_strcmp(key, "subchar")==0) {
 459             uint8_t bytes[UCNV_EXT_MAX_BYTES];
 460             int8_t length;
 461
 462             s=value;
 463             length=ucm_parseBytes(bytes, line, (const char **)&s);
 464             if(1<=length && length<=4 && *s==0) {
 465                 staticData->subCharLen=length;
 466                 uprv_memcpy(staticData->subChar, bytes, length);
 467             } else {
 468                 fprintf(stderr, "error: illegal <subchar> %s\n", value);
 469                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 470                 return;
 471             }
 472         } else if(uprv_strcmp(key, "subchar1")==0) {
 473             uint8_t bytes[UCNV_EXT_MAX_BYTES];
 474
 475             s=value;
 476             if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) {
 477                 staticData->subChar1=bytes[0];
 478             } else {
 479                 fprintf(stderr, "error: illegal <subchar1> %s\n", value);
 480                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 481                 return;
 482             }
 483         }
 484     }
 485
 486     /* copy values from the UCMFile to the static data */
 487     staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength;
 488     staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength;
 489     staticData->conversionType=data->ucm->states.conversionType;
 490
 491     if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
 492         fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n");
 493         *pErrorCode=U_INVALID_TABLE_FORMAT;
 494         return;
 495     }
 496
 497     /*
 498      * Now that we know the type, copy any 'default' values from the table.
 499      * We need not check the type any further because the parser only
 500      * recognizes what we have prototypes for.
 501      *
 502      * For delta (extension-only) tables, copy values from the base file
 503      * instead, see createConverter().
 504      */
 505     if(data->ucm->baseName[0]==0) {
 506         prototype=ucnv_converterStaticData[staticData->conversionType];
 507         if(prototype!=NULL) {
 508             if(staticData->name[0]==0) {
 509                 uprv_strcpy((char *)staticData->name, prototype->name);
 510             }
 511
 512             if(staticData->codepage==0) {
 513                 staticData->codepage=prototype->codepage;
 514             }
 515
 516             if(staticData->platform==0) {
 517                 staticData->platform=prototype->platform;
 518             }
 519
 520             if(staticData->minBytesPerChar==0) {
 521                 staticData->minBytesPerChar=prototype->minBytesPerChar;
 522             }
 523
 524             if(staticData->maxBytesPerChar==0) {
 525                 staticData->maxBytesPerChar=prototype->maxBytesPerChar;
 526             }
 527
 528             if(staticData->subCharLen==0) {
 529                 staticData->subCharLen=prototype->subCharLen;
 530                 if(prototype->subCharLen>0) {
 531                     uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
 532                 }
 533             }
 534         }
 535     }
 536
 537     if(data->ucm->states.outputType<0) {
 538         data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1;
 539     }
 540
 541     if( staticData->subChar1!=0 &&
 542             (staticData->minBytesPerChar>1 ||
 543                 (staticData->conversionType!=UCNV_MBCS &&
 544                  staticData->conversionType!=UCNV_EBCDIC_STATEFUL))
 545     ) {
 546         fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n");
 547         *pErrorCode=U_INVALID_TABLE_FORMAT;
 548     }
 549 }
 550
 551 /* return TRUE if a base table was read, FALSE for an extension table */
 552 static UBool
 553 readFile(ConvData *data, const char* converterName,
 554          UErrorCode *pErrorCode) {
 555     char line[200];
 556     char *end;
 557     FileStream *convFile;
 558
 559     UCMStates *baseStates;
 560     UBool dataIsBase;
 561
 562     if(U_FAILURE(*pErrorCode)) {
 563         return FALSE;
 564     }
 565
 566     data->ucm=ucm_open();
 567
 568     convFile=T_FileStream_open(converterName, "r");
 569     if(convFile==NULL) {
 570         *pErrorCode=U_FILE_ACCESS_ERROR;
 571         return FALSE;
 572     }
 573
 574     readHeader(data, convFile, converterName, pErrorCode);
 575     if(U_FAILURE(*pErrorCode)) {
 576         return FALSE;
 577     }
 578
 579     if(data->ucm->baseName[0]==0) {
 580         dataIsBase=TRUE;
 581         baseStates=&data->ucm->states;
 582         ucm_processStates(baseStates);
 583     } else {
 584         dataIsBase=FALSE;
 585         baseStates=NULL;
 586     }
 587
 588     /* read the base table */
 589     ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode);
 590     if(U_FAILURE(*pErrorCode)) {
 591         return FALSE;
 592     }
 593
 594     /* read an extension table if there is one */
 595     while(T_FileStream_readLine(convFile, line, sizeof(line))) {
 596         end=uprv_strchr(line, 0);
 597         while(line<end &&
 598               (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) {
 599             --end;
 600         }
 601         *end=0;
 602
 603         if(line[0]=='#' || u_skipWhitespace(line)==end) {
 604             continue; /* ignore empty and comment lines */
 605         }
 606
 607         if(0==uprv_strcmp(line, "CHARMAP")) {
 608             /* read the extension table */
 609             ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode);
 610         } else {
 611             fprintf(stderr, "unexpected text after the base mapping table\n");
 612         }
 613         break;
 614     }
 615
 616     T_FileStream_close(convFile);
 617
 618     if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) {
 619         fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
 620         *pErrorCode=U_INVALID_TABLE_FORMAT;
 621     }
 622
 623     return dataIsBase;
 624 }
 625
 626 static void
 627 createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) {
 628     ConvData baseData;
 629     UBool dataIsBase;
 630
 631     UConverterStaticData *staticData;
 632     UCMStates *states, *baseStates;
 633
 634     if(U_FAILURE(*pErrorCode)) {
 635         return;
 636     }
 637
 638     initConvData(data);
 639
 640     dataIsBase=readFile(data, converterName, pErrorCode);
 641     if(U_FAILURE(*pErrorCode)) {
 642         return;
 643     }
 644
 645     staticData=&data->staticData;
 646     states=&data->ucm->states;
 647
 648     if(dataIsBase) {
 649         /*
 650          * Build a normal .cnv file with a base table
 651          * and an optional extension table.
 652          */
 653         data->cnvData=MBCSOpen(data->ucm);
 654         if(data->cnvData==NULL) {
 655             *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 656
 657         } else if(!data->cnvData->isValid(data->cnvData,
 658                             staticData->subChar, staticData->subCharLen)
 659         ) {
 660             fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
 661             *pErrorCode=U_INVALID_TABLE_FORMAT;
 662
 663         } else if(staticData->subChar1!=0 &&
 664                     !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1)
 665         ) {
 666             fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
 667             *pErrorCode=U_INVALID_TABLE_FORMAT;
 668
 669         } else if(
 670             data->ucm->ext->mappingsLength>0 &&
 671             !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
 672         ) {
 673             *pErrorCode=U_INVALID_TABLE_FORMAT;
 674         } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) {
 675             /* sort the table so that it can be turned into UTF-8-friendly data */
 676             ucm_sortTable(data->ucm->base);
 677         }
 678
 679         if(U_SUCCESS(*pErrorCode)) {
 680             if(
 681                 /* add the base table after ucm_checkBaseExt()! */
 682                 !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData)
 683             ) {
 684                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 685             } else {
 686                 /*
 687                  * addTable() may have requested moving more mappings to the extension table
 688                  * if they fit into the base toUnicode table but not into the
 689                  * base fromUnicode table.
 690                  * (Especially for UTF-8-friendly fromUnicode tables.)
 691                  * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them
 692                  * to be excluded from the extension toUnicode data.
 693                  * See MBCSOkForBaseFromUnicode() for which mappings do not fit into
 694                  * the base fromUnicode table.
 695                  */
 696                 ucm_moveMappings(data->ucm->base, data->ucm->ext);
 697                 ucm_sortTable(data->ucm->ext);
 698                 if(data->ucm->ext->mappingsLength>0) {
 699                     /* prepare the extension table, if there is one */
 700                     data->extData=CnvExtOpen(data->ucm);
 701                     if(data->extData==NULL) {
 702                         *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 703                     } else if(
 704                         !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)
 705                     ) {
 706                         *pErrorCode=U_INVALID_TABLE_FORMAT;
 707                     }
 708                 }
 709             }
 710         }
 711     } else {
 712         /* Build an extension-only .cnv file. */
 713         char baseFilename[500];
 714         char *basename;
 715
 716         initConvData(&baseData);
 717
 718         /* assemble a path/filename for data->ucm->baseName */
 719         uprv_strcpy(baseFilename, converterName);
 720         basename=(char *)findBasename(baseFilename);
 721         uprv_strcpy(basename, data->ucm->baseName);
 722         uprv_strcat(basename, ".ucm");
 723
 724         /* read the base table */
 725         dataIsBase=readFile(&baseData, baseFilename, pErrorCode);
 726         if(U_FAILURE(*pErrorCode)) {
 727             return;
 728         } else if(!dataIsBase) {
 729             fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename);
 730             *pErrorCode=U_INVALID_TABLE_FORMAT;
 731         } else {
 732             /* prepare the extension table */
 733             data->extData=CnvExtOpen(data->ucm);
 734             if(data->extData==NULL) {
 735                 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 736             } else {
 737                 /* fill in gaps in extension file header fields */
 738                 UCMapping *m, *mLimit;
 739                 uint8_t fallbackFlags;
 740
 741                 baseStates=&baseData.ucm->states;
 742                 if(states->conversionType==UCNV_DBCS) {
 743                     staticData->minBytesPerChar=(int8_t)(states->minCharLength=2);
 744                 } else if(states->minCharLength==0) {
 745                     staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength);
 746                 }
 747                 if(states->maxCharLength<states->minCharLength) {
 748                     staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength);
 749                 }
 750
 751                 if(staticData->subCharLen==0) {
 752                     uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4);
 753                     staticData->subCharLen=baseData.staticData.subCharLen;
 754                 }
 755                 /*
 756                  * do not copy subChar1 -
 757                  * only use what is explicitly specified
 758                  * because it cannot be unset in the extension file header
 759                  */
 760
 761                 /* get the fallback flags */
 762                 fallbackFlags=0;
 763                 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
 764                     m<mLimit && fallbackFlags!=3;
 765                     ++m
 766                 ) {
 767                     if(m->f==1) {
 768                         fallbackFlags|=1;
 769                     } else if(m->f==3) {
 770                         fallbackFlags|=2;
 771                     }
 772                 }
 773
 774                 if(fallbackFlags&1) {
 775                     staticData->hasFromUnicodeFallback=TRUE;
 776                 }
 777                 if(fallbackFlags&2) {
 778                     staticData->hasToUnicodeFallback=TRUE;
 779                 }
 780
 781                 if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) {
 782                     fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
 783                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 784
 785                 } else if(1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) {
 786                     fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
 787                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 788
 789                 } else if(
 790                     !ucm_checkValidity(data->ucm->ext, baseStates) ||
 791                     !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
 792                 ) {
 793                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 794                 } else {
 795                     if(states->maxCharLength>1) {
 796                         /*
 797                          * When building a normal .cnv file with a base table
 798                          * for an MBCS (not SBCS) table with explicit precision flags,
 799                          * the MBCSAddTable() function marks some mappings for moving
 800                          * to the extension table.
 801                          * They fit into the base toUnicode table but not into the
 802                          * base fromUnicode table.
 803                          * (Note: We do have explicit precision flags because they are
 804                          * required for extension table generation, and
 805                          * ucm_checkBaseExt() verified it.)
 806                          *
 807                          * We do not call MBCSAddTable() here (we probably could)
 808                          * so we need to do the analysis before building the extension table.
 809                          * We assume that MBCSAddTable() will build a UTF-8-friendly table.
 810                          * Redundant mappings in the extension table are ok except they cost some size.
 811                          *
 812                          * Do this after ucm_checkBaseExt().
 813                          */
 814                         const MBCSData *mbcsData=MBCSGetDummy();
 815                         int32_t needsMove=0;
 816                         for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
 817                             m<mLimit;
 818                             ++m
 819                         ) {
 820                             if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) {
 821                                 m->f|=MBCS_FROM_U_EXT_FLAG;
 822                                 m->moveFlag=UCM_MOVE_TO_EXT;
 823                                 ++needsMove;
 824                             }
 825                         }
 826
 827                         if(needsMove!=0) {
 828                             ucm_moveMappings(baseData.ucm->base, data->ucm->ext);
 829                             ucm_sortTable(data->ucm->ext);
 830                         }
 831                     }
 832                     if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) {
 833                         *pErrorCode=U_INVALID_TABLE_FORMAT;
 834                     }
 835                 }
 836             }
 837         }
 838
 839         cleanupConvData(&baseData);
 840     }
 841 }
 842
 843 /*
 844  * Hey, Emacs, please set the following:
 845  *
 846  * Local Variables:
 847  * indent-tabs-mode: nil
 848  * End:
 849  *
 850  */