icuSources/tools/makeconv/makeconv.cpp

   1 /*
   2  ********************************************************************************
   3  *
   4  *   Copyright (C) 1998-2015, International Business Machines
   5  *   Corporation and others.  All Rights Reserved.
   6  *
   7  ********************************************************************************
   8  *
   9  *
  10  *  makeconv.cpp:
  11  *  tool creating a binary (compressed) representation of the conversion mapping
  12  *  table (IBM NLTC ucmap format).
  13  *
  14  *  05/04/2000    helena     Added fallback mapping into the picture...
  15  *  06/29/2000  helena      Major rewrite of the callback APIs.
  16  */
  17
  18 #include <stdio.h>
  19 #include "unicode/putil.h"
  20 #include "unicode/ucnv_err.h"
  21 #include "charstr.h"
  22 #include "ucnv_bld.h"
  23 #include "ucnv_imp.h"
  24 #include "ucnv_cnv.h"
  25 #include "cstring.h"
  26 #include "cmemory.h"
  27 #include "uinvchar.h"
  28 #include "filestrm.h"
  29 #include "toolutil.h"
  30 #include "uoptions.h"
  31 #include "unicode/udata.h"
  32 #include "unewdata.h"
  33 #include "uparse.h"
  34 #include "ucm.h"
  35 #include "makeconv.h"
  36 #include "genmbcs.h"
  37
  38 #define DEBUG 0
  39
  40 typedef struct ConvData {
  41     UCMFile *ucm;
  42     NewConverter *cnvData, *extData;
  43     UConverterSharedData sharedData;
  44     UConverterStaticData staticData;
  45 } ConvData;
  46
  47 static void
  48 initConvData(ConvData *data) {
  49     uprv_memset(data, 0, sizeof(ConvData));
  50     data->sharedData.structSize=sizeof(UConverterSharedData);
  51     data->staticData.structSize=sizeof(UConverterStaticData);
  52     data->sharedData.staticData=&data->staticData;
  53 }
  54
  55 static void
  56 cleanupConvData(ConvData *data) {
  57     if(data!=NULL) {
  58         if(data->cnvData!=NULL) {
  59             data->cnvData->close(data->cnvData);
  60             data->cnvData=NULL;
  61         }
  62         if(data->extData!=NULL) {
  63             data->extData->close(data->extData);
  64             data->extData=NULL;
  65         }
  66         ucm_close(data->ucm);
  67         data->ucm=NULL;
  68     }
  69 }
  70
  71 /*
  72  * from ucnvstat.c - static prototypes of data-based converters
  73  */
  74 U_CAPI const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES];
  75
  76 /*
  77  * Global - verbosity
  78  */
  79 UBool VERBOSE = FALSE;
  80 UBool QUIET = FALSE;
  81 UBool SMALL = FALSE;
  82 UBool IGNORE_SISO_CHECK = FALSE;
  83
  84 static void
  85 createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode);
  86
  87 /*
  88  * Set up the UNewData and write the converter..
  89  */
  90 static void
  91 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status);
  92
  93 UBool haveCopyright=TRUE;
  94
  95 static UDataInfo dataInfo={
  96     sizeof(UDataInfo),
  97     0,
  98
  99     U_IS_BIG_ENDIAN,
 100     U_CHARSET_FAMILY,
 101     sizeof(UChar),
 102     0,
 103
 104     {0x63, 0x6e, 0x76, 0x74},     /* dataFormat="cnvt" */
 105     {6, 2, 0, 0},                 /* formatVersion */
 106     {0, 0, 0, 0}                  /* dataVersion (calculated at runtime) */
 107 };
 108
 109 static void
 110 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status)
 111 {
 112     UNewDataMemory *mem = NULL;
 113     uint32_t sz2;
 114     uint32_t size = 0;
 115     int32_t tableType;
 116
 117     if(U_FAILURE(*status))
 118       {
 119         return;
 120       }
 121
 122     tableType=TABLE_NONE;
 123     if(data->cnvData!=NULL) {
 124         tableType|=TABLE_BASE;
 125     }
 126     if(data->extData!=NULL) {
 127         tableType|=TABLE_EXT;
 128     }
 129
 130     mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status);
 131
 132     if(U_FAILURE(*status))
 133       {
 134         fprintf(stderr, "Couldn't create the udata %s.%s: %s\n",
 135                 cnvName,
 136                 "cnv",
 137                 u_errorName(*status));
 138         return;
 139       }
 140
 141     if(VERBOSE)
 142       {
 143         printf("- Opened udata %s.%s\n", cnvName, "cnv");
 144       }
 145
 146
 147     /* all read only, clean, platform independent data.  Mmmm. :)  */
 148     udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData));
 149     size += sizeof(UConverterStaticData); /* Is 4-aligned  - by size */
 150     /* Now, write the table */
 151     if(tableType&TABLE_BASE) {
 152         size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType);
 153     }
 154     if(tableType&TABLE_EXT) {
 155         size += data->extData->write(data->extData, &data->staticData, mem, tableType);
 156     }
 157
 158     sz2 = udata_finish(mem, status);
 159     if(size != sz2)
 160     {
 161         fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size);
 162         *status=U_INTERNAL_PROGRAM_ERROR;
 163     }
 164     if(VERBOSE)
 165     {
 166       printf("- Wrote %u bytes to the udata.\n", (int)sz2);
 167     }
 168 }
 169
 170 enum {
 171     OPT_HELP_H,
 172     OPT_HELP_QUESTION_MARK,
 173     OPT_COPYRIGHT,
 174     OPT_VERSION,
 175     OPT_DESTDIR,
 176     OPT_VERBOSE,
 177     OPT_SMALL,
 178     OPT_IGNORE_SISO_CHECK,
 179     OPT_QUIET,
 180
 181     OPT_COUNT
 182 };
 183
 184 static UOption options[]={
 185     UOPTION_HELP_H,
 186     UOPTION_HELP_QUESTION_MARK,
 187     UOPTION_COPYRIGHT,
 188     UOPTION_VERSION,
 189     UOPTION_DESTDIR,
 190     UOPTION_VERBOSE,
 191     { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 },
 192     { "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 },
 193     UOPTION_QUIET,
 194 };
 195
 196 int main(int argc, char* argv[])
 197 {
 198     ConvData data;
 199     char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
 200
 201     U_MAIN_INIT_ARGS(argc, argv);
 202
 203     /* Set up the ICU version number */
 204     UVersionInfo icuVersion;
 205     u_getVersion(icuVersion);
 206     uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));
 207
 208     /* preset then read command line options */
 209     options[OPT_DESTDIR].value=u_getDataDirectory();
 210     argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
 211
 212     /* error handling, printing usage message */
 213     if(argc<0) {
 214         fprintf(stderr,
 215             "error in command line argument \"%s\"\n",
 216             argv[-argc]);
 217     } else if(argc<2) {
 218         argc=-1;
 219     }
 220     if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) {
 221         FILE *stdfile=argc<0 ? stderr : stdout;
 222         fprintf(stdfile,
 223             "usage: %s [-options] files...\n"
 224             "\tread .ucm codepage mapping files and write .cnv files\n"
 225             "options:\n"
 226             "\t-h or -? or --help  this usage text\n"
 227             "\t-V or --version     show a version message\n"
 228             "\t-c or --copyright   include a copyright notice\n"
 229             "\t-d or --destdir     destination directory, followed by the path\n"
 230             "\t-v or --verbose     Turn on verbose output\n"
 231             "\t-q or --quiet       do not display warnings and progress\n",
 232             argv[0]);
 233         fprintf(stdfile,
 234             "\t      --small       Generate smaller .cnv files. They will be\n"
 235             "\t                    significantly smaller but may not be compatible with\n"
 236             "\t                    older versions of ICU and will require heap memory\n"
 237             "\t                    allocation when loaded.\n"
 238             "\t      --ignore-siso-check         Use SI/SO other than 0xf/0xe.\n");
 239         return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
 240     }
 241
 242     if(options[OPT_VERSION].doesOccur) {
 243         printf("makeconv version %u.%u, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
 244                dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
 245         printf("%s\n", U_COPYRIGHT_STRING);
 246         exit(0);
 247     }
 248
 249     /* get the options values */
 250     haveCopyright = options[OPT_COPYRIGHT].doesOccur;
 251     const char *destdir = options[OPT_DESTDIR].value;
 252     VERBOSE = options[OPT_VERBOSE].doesOccur;
 253     QUIET = options[OPT_QUIET].doesOccur;
 254     SMALL = options[OPT_SMALL].doesOccur;
 255
 256     if (options[OPT_IGNORE_SISO_CHECK].doesOccur) {
 257         IGNORE_SISO_CHECK = TRUE;
 258     }
 259
 260     icu::CharString outFileName;
 261     UErrorCode err = U_ZERO_ERROR;
 262     if (destdir != NULL && *destdir != 0) {
 263         outFileName.append(destdir, err).ensureEndsWithFileSeparator(err);
 264         if (U_FAILURE(err)) {
 265             return err;
 266         }
 267     }
 268     int32_t outBasenameStart = outFileName.length();
 269
 270 #if DEBUG
 271     {
 272       int i;
 273       printf("makeconv: processing %d files...\n", argc - 1);
 274       for(i=1; i<argc; ++i) {
 275         printf("%s ", argv[i]);
 276       }
 277       printf("\n");
 278       fflush(stdout);
 279     }
 280 #endif
 281
 282     UBool printFilename = (UBool) (argc > 2 || VERBOSE);
 283     for (++argv; --argc; ++argv)
 284     {
 285         UErrorCode localError = U_ZERO_ERROR;
 286         const char *arg = getLongPathname(*argv);
 287
 288         /*produces the right destination path for display*/
 289         outFileName.truncate(outBasenameStart);
 290         if (outBasenameStart != 0)
 291         {
 292             /* find the last file sepator */
 293             const char *basename = findBasename(arg);
 294             outFileName.append(basename, localError);
 295         }
 296         else
 297         {
 298             outFileName.append(arg, localError);
 299         }
 300         if (U_FAILURE(localError)) {
 301             return localError;
 302         }
 303
 304         /*removes the extension if any is found*/
 305         int32_t lastDotIndex = outFileName.lastIndexOf('.');
 306         if (lastDotIndex >= outBasenameStart) {
 307             outFileName.truncate(lastDotIndex);
 308         }
 309
 310         /* the basename without extension is the converter name */
 311         if ((outFileName.length() - outBasenameStart) >= UPRV_LENGTHOF(cnvName)) {
 312             fprintf(stderr, "converter name %s too long\n", outFileName.data() + outBasenameStart);
 313             return U_BUFFER_OVERFLOW_ERROR;
 314         }
 315         uprv_strcpy(cnvName, outFileName.data() + outBasenameStart);
 316
 317         /*Adds the target extension*/
 318         outFileName.append(CONVERTER_FILE_EXTENSION, localError);
 319         if (U_FAILURE(localError)) {
 320             return localError;
 321         }
 322
 323 #if DEBUG
 324         printf("makeconv: processing %s  ...\n", arg);
 325         fflush(stdout);
 326 #endif
 327         initConvData(&data);
 328         createConverter(&data, arg, &localError);
 329
 330         if (U_FAILURE(localError))
 331         {
 332             /* if an error is found, print out an error msg and keep going */
 333             fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n",
 334                     outFileName.data(), arg, u_errorName(localError));
 335             if(U_SUCCESS(err)) {
 336                 err = localError;
 337             }
 338         }
 339         else
 340         {
 341             /* Insure the static data name matches the  file name */
 342             /* Changed to ignore directory and only compare base name
 343              LDH 1/2/08*/
 344             char *p;
 345             p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */
 346
 347             if(p == NULL)            /* OK, try alternate */
 348             {
 349                 p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR);
 350                 if(p == NULL)
 351                 {
 352                     p=cnvName; /* If no separators, no problem */
 353                 }
 354             }
 355             else
 356             {
 357                 p++;   /* If found separator, don't include it in compare */
 358             }
 359             if(uprv_stricmp(p,data.staticData.name) && !QUIET)
 360             {
 361                 fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
 362                     cnvName,  CONVERTER_FILE_EXTENSION,
 363                     data.staticData.name);
 364             }
 365
 366             uprv_strcpy((char*)data.staticData.name, cnvName);
 367
 368             if(!uprv_isInvariantString((char*)data.staticData.name, -1)) {
 369                 fprintf(stderr,
 370                     "Error: A converter name must contain only invariant characters.\n"
 371                     "%s is not a valid converter name.\n",
 372                     data.staticData.name);
 373                 if(U_SUCCESS(err)) {
 374                     err = U_INVALID_TABLE_FORMAT;
 375                 }
 376             }
 377
 378             localError = U_ZERO_ERROR;
 379             writeConverterData(&data, cnvName, destdir, &localError);
 380
 381             if(U_FAILURE(localError))
 382             {
 383                 /* if an error is found, print out an error msg and keep going*/
 384                 fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName.data(), arg,
 385                     u_errorName(localError));
 386                 if(U_SUCCESS(err)) {
 387                     err = localError;
 388                 }
 389             }
 390             else if (printFilename)
 391             {
 392                 puts(outFileName.data() + outBasenameStart);
 393             }
 394         }
 395         fflush(stdout);
 396         fflush(stderr);
 397
 398         cleanupConvData(&data);
 399     }
 400
 401     return err;
 402 }
 403
 404 static void
 405 getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) {
 406     if( (name[0]=='i' || name[0]=='I') &&
 407         (name[1]=='b' || name[1]=='B') &&
 408         (name[2]=='m' || name[2]=='M')
 409     ) {
 410         name+=3;
 411         if(*name=='-') {
 412             ++name;
 413         }
 414         *pPlatform=UCNV_IBM;
 415         *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10);
 416     } else {
 417         *pPlatform=UCNV_UNKNOWN;
 418         *pCCSID=0;
 419     }
 420 }
 421
 422 static void
 423 readHeader(ConvData *data,
 424            FileStream* convFile,
 425            UErrorCode *pErrorCode) {
 426     char line[1024];
 427     char *s, *key, *value;
 428     const UConverterStaticData *prototype;
 429     UConverterStaticData *staticData;
 430
 431     if(U_FAILURE(*pErrorCode)) {
 432         return;
 433     }
 434
 435     staticData=&data->staticData;
 436     staticData->platform=UCNV_IBM;
 437     staticData->subCharLen=0;
 438
 439     while(T_FileStream_readLine(convFile, line, sizeof(line))) {
 440         /* basic parsing and handling of state-related items */
 441         if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) {
 442             continue;
 443         }
 444
 445         /* stop at the beginning of the mapping section */
 446         if(uprv_strcmp(line, "CHARMAP")==0) {
 447             break;
 448         }
 449
 450         /* collect the information from the header field, ignore unknown keys */
 451         if(uprv_strcmp(key, "code_set_name")==0) {
 452             if(*value!=0) {
 453                 uprv_strcpy((char *)staticData->name, value);
 454                 getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
 455             }
 456         } else if(uprv_strcmp(key, "subchar")==0) {
 457             uint8_t bytes[UCNV_EXT_MAX_BYTES];
 458             int8_t length;
 459
 460             s=value;
 461             length=ucm_parseBytes(bytes, line, (const char **)&s);
 462             if(1<=length && length<=4 && *s==0) {
 463                 staticData->subCharLen=length;
 464                 uprv_memcpy(staticData->subChar, bytes, length);
 465             } else {
 466                 fprintf(stderr, "error: illegal <subchar> %s\n", value);
 467                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 468                 return;
 469             }
 470         } else if(uprv_strcmp(key, "subchar1")==0) {
 471             uint8_t bytes[UCNV_EXT_MAX_BYTES];
 472
 473             s=value;
 474             if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) {
 475                 staticData->subChar1=bytes[0];
 476             } else {
 477                 fprintf(stderr, "error: illegal <subchar1> %s\n", value);
 478                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 479                 return;
 480             }
 481         }
 482     }
 483
 484     /* copy values from the UCMFile to the static data */
 485     staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength;
 486     staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength;
 487     staticData->conversionType=data->ucm->states.conversionType;
 488
 489     if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
 490         fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n");
 491         *pErrorCode=U_INVALID_TABLE_FORMAT;
 492         return;
 493     }
 494
 495     /*
 496      * Now that we know the type, copy any 'default' values from the table.
 497      * We need not check the type any further because the parser only
 498      * recognizes what we have prototypes for.
 499      *
 500      * For delta (extension-only) tables, copy values from the base file
 501      * instead, see createConverter().
 502      */
 503     if(data->ucm->baseName[0]==0) {
 504         prototype=ucnv_converterStaticData[staticData->conversionType];
 505         if(prototype!=NULL) {
 506             if(staticData->name[0]==0) {
 507                 uprv_strcpy((char *)staticData->name, prototype->name);
 508             }
 509
 510             if(staticData->codepage==0) {
 511                 staticData->codepage=prototype->codepage;
 512             }
 513
 514             if(staticData->platform==0) {
 515                 staticData->platform=prototype->platform;
 516             }
 517
 518             if(staticData->minBytesPerChar==0) {
 519                 staticData->minBytesPerChar=prototype->minBytesPerChar;
 520             }
 521
 522             if(staticData->maxBytesPerChar==0) {
 523                 staticData->maxBytesPerChar=prototype->maxBytesPerChar;
 524             }
 525
 526             if(staticData->subCharLen==0) {
 527                 staticData->subCharLen=prototype->subCharLen;
 528                 if(prototype->subCharLen>0) {
 529                     uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
 530                 }
 531             }
 532         }
 533     }
 534
 535     if(data->ucm->states.outputType<0) {
 536         data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1;
 537     }
 538
 539     if( staticData->subChar1!=0 &&
 540             (staticData->minBytesPerChar>1 ||
 541                 (staticData->conversionType!=UCNV_MBCS &&
 542                  staticData->conversionType!=UCNV_EBCDIC_STATEFUL))
 543     ) {
 544         fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n");
 545         *pErrorCode=U_INVALID_TABLE_FORMAT;
 546     }
 547 }
 548
 549 /* return TRUE if a base table was read, FALSE for an extension table */
 550 static UBool
 551 readFile(ConvData *data, const char* converterName,
 552          UErrorCode *pErrorCode) {
 553     char line[1024];
 554     char *end;
 555     FileStream *convFile;
 556
 557     UCMStates *baseStates;
 558     UBool dataIsBase;
 559
 560     if(U_FAILURE(*pErrorCode)) {
 561         return FALSE;
 562     }
 563
 564     data->ucm=ucm_open();
 565
 566     convFile=T_FileStream_open(converterName, "r");
 567     if(convFile==NULL) {
 568         *pErrorCode=U_FILE_ACCESS_ERROR;
 569         return FALSE;
 570     }
 571
 572     readHeader(data, convFile, pErrorCode);
 573     if(U_FAILURE(*pErrorCode)) {
 574         return FALSE;
 575     }
 576
 577     if(data->ucm->baseName[0]==0) {
 578         dataIsBase=TRUE;
 579         baseStates=&data->ucm->states;
 580         ucm_processStates(baseStates, IGNORE_SISO_CHECK);
 581     } else {
 582         dataIsBase=FALSE;
 583         baseStates=NULL;
 584     }
 585
 586     /* read the base table */
 587     ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode);
 588     if(U_FAILURE(*pErrorCode)) {
 589         return FALSE;
 590     }
 591
 592     /* read an extension table if there is one */
 593     while(T_FileStream_readLine(convFile, line, sizeof(line))) {
 594         end=uprv_strchr(line, 0);
 595         while(line<end &&
 596               (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) {
 597             --end;
 598         }
 599         *end=0;
 600
 601         if(line[0]=='#' || u_skipWhitespace(line)==end) {
 602             continue; /* ignore empty and comment lines */
 603         }
 604
 605         if(0==uprv_strcmp(line, "CHARMAP")) {
 606             /* read the extension table */
 607             ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode);
 608         } else {
 609             fprintf(stderr, "unexpected text after the base mapping table\n");
 610         }
 611         break;
 612     }
 613
 614     T_FileStream_close(convFile);
 615
 616     if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) {
 617         fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
 618         *pErrorCode=U_INVALID_TABLE_FORMAT;
 619     }
 620
 621     return dataIsBase;
 622 }
 623
 624 static void
 625 createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) {
 626     ConvData baseData;
 627     UBool dataIsBase;
 628
 629     UConverterStaticData *staticData;
 630     UCMStates *states, *baseStates;
 631
 632     if(U_FAILURE(*pErrorCode)) {
 633         return;
 634     }
 635
 636     initConvData(data);
 637
 638     dataIsBase=readFile(data, converterName, pErrorCode);
 639     if(U_FAILURE(*pErrorCode)) {
 640         return;
 641     }
 642
 643     staticData=&data->staticData;
 644     states=&data->ucm->states;
 645
 646     if(dataIsBase) {
 647         /*
 648          * Build a normal .cnv file with a base table
 649          * and an optional extension table.
 650          */
 651         data->cnvData=MBCSOpen(data->ucm);
 652         if(data->cnvData==NULL) {
 653             *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 654
 655         } else if(!data->cnvData->isValid(data->cnvData,
 656                             staticData->subChar, staticData->subCharLen)
 657         ) {
 658             fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
 659             *pErrorCode=U_INVALID_TABLE_FORMAT;
 660
 661         } else if(staticData->subChar1!=0 &&
 662                     !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1)
 663         ) {
 664             fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
 665             *pErrorCode=U_INVALID_TABLE_FORMAT;
 666
 667         } else if(
 668             data->ucm->ext->mappingsLength>0 &&
 669             !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
 670         ) {
 671             *pErrorCode=U_INVALID_TABLE_FORMAT;
 672         } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) {
 673             /* sort the table so that it can be turned into UTF-8-friendly data */
 674             ucm_sortTable(data->ucm->base);
 675         }
 676
 677         if(U_SUCCESS(*pErrorCode)) {
 678             if(
 679                 /* add the base table after ucm_checkBaseExt()! */
 680                 !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData)
 681             ) {
 682                 *pErrorCode=U_INVALID_TABLE_FORMAT;
 683             } else {
 684                 /*
 685                  * addTable() may have requested moving more mappings to the extension table
 686                  * if they fit into the base toUnicode table but not into the
 687                  * base fromUnicode table.
 688                  * (Especially for UTF-8-friendly fromUnicode tables.)
 689                  * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them
 690                  * to be excluded from the extension toUnicode data.
 691                  * See MBCSOkForBaseFromUnicode() for which mappings do not fit into
 692                  * the base fromUnicode table.
 693                  */
 694                 ucm_moveMappings(data->ucm->base, data->ucm->ext);
 695                 ucm_sortTable(data->ucm->ext);
 696                 if(data->ucm->ext->mappingsLength>0) {
 697                     /* prepare the extension table, if there is one */
 698                     data->extData=CnvExtOpen(data->ucm);
 699                     if(data->extData==NULL) {
 700                         *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 701                     } else if(
 702                         !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)
 703                     ) {
 704                         *pErrorCode=U_INVALID_TABLE_FORMAT;
 705                     }
 706                 }
 707             }
 708         }
 709     } else {
 710         /* Build an extension-only .cnv file. */
 711         char baseFilename[500];
 712         char *basename;
 713
 714         initConvData(&baseData);
 715
 716         /* assemble a path/filename for data->ucm->baseName */
 717         uprv_strcpy(baseFilename, converterName);
 718         basename=(char *)findBasename(baseFilename);
 719         uprv_strcpy(basename, data->ucm->baseName);
 720         uprv_strcat(basename, ".ucm");
 721
 722         /* read the base table */
 723         dataIsBase=readFile(&baseData, baseFilename, pErrorCode);
 724         if(U_FAILURE(*pErrorCode)) {
 725             return;
 726         } else if(!dataIsBase) {
 727             fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename);
 728             *pErrorCode=U_INVALID_TABLE_FORMAT;
 729         } else {
 730             /* prepare the extension table */
 731             data->extData=CnvExtOpen(data->ucm);
 732             if(data->extData==NULL) {
 733                 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
 734             } else {
 735                 /* fill in gaps in extension file header fields */
 736                 UCMapping *m, *mLimit;
 737                 uint8_t fallbackFlags;
 738
 739                 baseStates=&baseData.ucm->states;
 740                 if(states->conversionType==UCNV_DBCS) {
 741                     staticData->minBytesPerChar=(int8_t)(states->minCharLength=2);
 742                 } else if(states->minCharLength==0) {
 743                     staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength);
 744                 }
 745                 if(states->maxCharLength<states->minCharLength) {
 746                     staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength);
 747                 }
 748
 749                 if(staticData->subCharLen==0) {
 750                     uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4);
 751                     staticData->subCharLen=baseData.staticData.subCharLen;
 752                 }
 753                 /*
 754                  * do not copy subChar1 -
 755                  * only use what is explicitly specified
 756                  * because it cannot be unset in the extension file header
 757                  */
 758
 759                 /* get the fallback flags */
 760                 fallbackFlags=0;
 761                 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
 762                     m<mLimit && fallbackFlags!=3;
 763                     ++m
 764                 ) {
 765                     if(m->f==1) {
 766                         fallbackFlags|=1;
 767                     } else if(m->f==3) {
 768                         fallbackFlags|=2;
 769                     }
 770                 }
 771
 772                 if(fallbackFlags&1) {
 773                     staticData->hasFromUnicodeFallback=TRUE;
 774                 }
 775                 if(fallbackFlags&2) {
 776                     staticData->hasToUnicodeFallback=TRUE;
 777                 }
 778
 779                 if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) {
 780                     fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
 781                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 782
 783                 } else if(staticData->subChar1!=0 && 1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) {
 784                     fprintf(stderr, "       the subchar1 byte is illegal in this codepage structure!\n");
 785                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 786
 787                 } else if(
 788                     !ucm_checkValidity(data->ucm->ext, baseStates) ||
 789                     !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
 790                 ) {
 791                     *pErrorCode=U_INVALID_TABLE_FORMAT;
 792                 } else {
 793                     if(states->maxCharLength>1) {
 794                         /*
 795                          * When building a normal .cnv file with a base table
 796                          * for an MBCS (not SBCS) table with explicit precision flags,
 797                          * the MBCSAddTable() function marks some mappings for moving
 798                          * to the extension table.
 799                          * They fit into the base toUnicode table but not into the
 800                          * base fromUnicode table.
 801                          * (Note: We do have explicit precision flags because they are
 802                          * required for extension table generation, and
 803                          * ucm_checkBaseExt() verified it.)
 804                          *
 805                          * We do not call MBCSAddTable() here (we probably could)
 806                          * so we need to do the analysis before building the extension table.
 807                          * We assume that MBCSAddTable() will build a UTF-8-friendly table.
 808                          * Redundant mappings in the extension table are ok except they cost some size.
 809                          *
 810                          * Do this after ucm_checkBaseExt().
 811                          */
 812                         const MBCSData *mbcsData=MBCSGetDummy();
 813                         int32_t needsMove=0;
 814                         for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
 815                             m<mLimit;
 816                             ++m
 817                         ) {
 818                             if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) {
 819                                 m->f|=MBCS_FROM_U_EXT_FLAG;
 820                                 m->moveFlag=UCM_MOVE_TO_EXT;
 821                                 ++needsMove;
 822                             }
 823                         }
 824
 825                         if(needsMove!=0) {
 826                             ucm_moveMappings(baseData.ucm->base, data->ucm->ext);
 827                             ucm_sortTable(data->ucm->ext);
 828                         }
 829                     }
 830                     if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) {
 831                         *pErrorCode=U_INVALID_TABLE_FORMAT;
 832                     }
 833                 }
 834             }
 835         }
 836
 837         cleanupConvData(&baseData);
 838     }
 839 }
 840
 841 /*
 842  * Hey, Emacs, please set the following:
 843  *
 844  * Local Variables:
 845  * indent-tabs-mode: nil
 846  * End:
 847  *
 848  */