icuSources/common/ucnvmbcs.h

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 ******************************************************************************
   5 *
   6 *   Copyright (C) 2000-2013, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 ******************************************************************************
  10 *   file name:  ucnvmbcs.h
  11 *   encoding:   UTF-8
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2000jul07
  16 *   created by: Markus W. Scherer
  17 */
  18
  19 #ifndef __UCNVMBCS_H__
  20 #define __UCNVMBCS_H__
  21
  22 #include "unicode/utypes.h"
  23
  24 #if !UCONFIG_NO_CONVERSION
  25
  26 #include "unicode/ucnv.h"
  27 #include "ucnv_cnv.h"
  28 #include "ucnv_ext.h"
  29
  30 /**
  31  * ICU conversion (.cnv) data file structure, following the usual UDataInfo
  32  * header.
  33  *
  34  * Format version: 6.2
  35  *
  36  * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
  37  *                                min/max bytes per character, etc.
  38  *                                see ucnv_bld.h
  39  *
  40  * --------------------
  41  *
  42  * The static data is followed by conversionType-specific data structures.
  43  * At the moment, there are only variations of MBCS converters. They all have
  44  * the same toUnicode structures, while the fromUnicode structures for SBCS
  45  * differ from those for other MBCS-style converters.
  46  *
  47  * _MBCSHeader.version 5 is optional and not backward-compatible
  48  * (as usual for changes in the major version field).
  49  *
  50  * Versions 5.m work like versions 4.m except:
  51  * - The _MBCSHeader has variable length (and is always longer than in version 4).
  52  *   See the struct _MBCSHeader further description below.
  53  * - There is a set of flags which indicate further incompatible changes.
  54  *   (Reader code must reject the file if it does not recognize them all.)
  55  * - In particular, one of these flags indicates that most of the fromUnicode
  56  *   data is missing and must be reconstituted from the toUnicode data
  57  *   and from the utf8Friendly mbcsIndex at load time.
  58  *   (This only works with a utf8Friendly table.)
  59  *   In this case, makeconv may increase maxFastUChar automatically to U+FFFF.
  60  *
  61  * The first of these versions is 5.3, which is like 4.3 except for the differences above.
  62  *
  63  * When possible, makeconv continues to generate version 4.m files.
  64  *
  65  * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4)
  66  * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details.
  67  *
  68  * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures
  69  * slightly and optionally adds a table for conversion to MBCS (non-SBCS)
  70  * charsets.
  71  *
  72  * The modifications are to make the data utf8Friendly. Not every 4.3 file
  73  * file contains utf8Friendly data.
  74  * It is utf8Friendly if _MBCSHeader.version[2]!=0.
  75  * In this case, the data structures are utf8Friendly up to the code point
  76  *   maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff)
  77  *
  78  * A utf8Friendly file has fromUnicode stage 3 entries for code points up to
  79  * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from
  80  * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries.
  81  *
  82  * In addition, a utf8Friendly MBCS file contains an additional
  83  *   uint16_t mbcsIndex[(maxFastUChar+1)>>6];
  84  * which replaces the stage 1 and 2 tables for indexing with bits from the
  85  * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table,
  86  * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks
  87  * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to
  88  * the extension data structure. This also allows for faster roundtrip
  89  * conversion from UTF-16.
  90  *
  91  * SBCS files do not contain an additional sbcsIndex[] array because the
  92  * proportional size increase would be noticeable, but the runtime
  93  * code builds one for the code point range for which the runtime conversion
  94  * code is optimized.
  95  *
  96  * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv
  97  * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in
  98  * the file, a larger maxFastUChar only affects stage 3 block allocation size
  99  * and is free in empty blocks. (Larger blocks with sparse contents cause larger
 100  * files.) U+1FFF includes almost all of the small scripts.
 101  * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with
 102  * 0xe0. This includes most scripts with legacy SBCS charsets.
 103  * The initial runtime implementation using 4.3 files only builds an sbcsIndex
 104  * for code points up to U+0FFF.
 105  *
 106  * For MBCS, maxFastUChar should be at least U+D7FF (=initial value).
 107  * This boundary is convenient because practically all of the commonly used
 108  * characters are below it, and because it is the boundary to surrogate
 109  * code points, above which special handling is necessary anyway.
 110  * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.)
 111  *
 112  * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be
 113  * useful especially for conversion from UTF-8 when the input can be assumed
 114  * to be valid, because the surrogate range would then not have to be
 115  * checked.
 116  * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value
 117  * overflow because with the all-unassigned block 0 and nearly full mappings
 118  * from the BMP it is theoretically possible that an index into stage 3
 119  * exceeds 16 bits.)
 120  *
 121  * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
 122  * If it is present, then an ICU version reading header versions 4.0 or 4.1
 123  * will be able to use the base table and ignore the extension.
 124  *
 125  * The unicodeMask in the static data is part of the base table data structure.
 126  * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
 127  * fromUnicode stage 1 array.
 128  * The static data unicodeMask refers only to the base table's properties if
 129  * a base table is included.
 130  * In an extension-only file, the static data unicodeMask is 0.
 131  * The extension data indexes have a separate field with the unicodeMask flags.
 132  *
 133  * MBCS-style data structure following the static data.
 134  * Offsets are counted in bytes from the beginning of the MBCS header structure.
 135  * Details about usage in comments in ucnvmbcs.c.
 136  *
 137  * struct _MBCSHeader (see the definition in this header file below)
 138  * contains 32-bit fields as follows:
 139  * 8 values:
 140  *  0   uint8_t[4]  MBCS version in UVersionInfo format (currently 4.3.x.0)
 141  *  1   uint32_t    countStates
 142  *  2   uint32_t    countToUFallbacks
 143  *  3   uint32_t    offsetToUCodeUnits
 144  *  4   uint32_t    offsetFromUTable
 145  *  5   uint32_t    offsetFromUBytes
 146  *  6   uint32_t    flags, bits:
 147  *                      31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
 148  *                                                0 for older versions and if
 149  *                                                there is not extension structure
 150  *                       7.. 0 outputType
 151  *  7   uint32_t    fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
 152  *                  counts bytes in fromUBytes[]
 153  *
 154  * New and required in version 5:
 155  *  8   uint32_t    options, bits:
 156  *                      31..16 reserved for flags that can be added without breaking
 157  *                                 backward compatibility
 158  *                      15.. 6 reserved for flags whose addition will break
 159  *                                 backward compatibility
 160  *                           6 MBCS_OPT_FROM_U -- if set,
 161  *                                 then most of the fromUnicode data is omitted;
 162  *                                 fullStage2Length is present and the missing
 163  *                                 bottom part of stage 2 must be reconstituted from
 164  *                                 the toUnicode data;
 165  *                                 stage 3 is missing completely as well;
 166  *                                 not used for SBCS tables
 167  *                       5.. 0 length of the _MBCSHeader (number of uint32_t)
 168  *
 169  * New and optional in version 5:
 170  *  9   uint32_t    fullStage2Length: used if MBCS_OPT_FROM_U is set
 171  *                                 specifies the full length of stage 2
 172  *                                 including the omitted part
 173  *
 174  * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
 175  *     -- base table name for extension-only table
 176  *     char baseTableName[variable]; -- with NUL plus padding for 4-alignment
 177  *
 178  *     -- all _MBCSHeader fields except for version and flags are 0
 179  * } else {
 180  *     -- normal base table with optional extension
 181  *
 182  *     int32_t stateTable[countStates][256];
 183  *
 184  *     struct _MBCSToUFallback { (fallbacks are sorted by offset)
 185  *         uint32_t offset;
 186  *         UChar32 codePoint;
 187  *     } toUFallbacks[countToUFallbacks];
 188  *
 189  *     uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
 190  *                  (padded to an even number of units)
 191  *
 192  *     -- stage 1 tables
 193  *     if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
 194  *         -- stage 1 table for all of Unicode
 195  *         uint16_t fromUTable[0x440]; (32-bit-aligned)
 196  *     } else {
 197  *         -- BMP-only tables have a smaller stage 1 table
 198  *         uint16_t fromUTable[0x40]; (32-bit-aligned)
 199  *     }
 200  *
 201  *     -- stage 2 tables
 202  *        length determined by top of stage 1 and bottom of stage 3 tables
 203  *     if(outputType==MBCS_OUTPUT_1) {
 204  *         -- SBCS: pure indexes
 205  *         uint16_t stage 2 indexes[?];
 206  *     } else {
 207  *         -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
 208  *         uint32_t stage 2 flags and indexes[?];
 209  *         if(options&MBCS_OPT_NO_FROM_U) {
 210  *             stage 2 really has length fullStage2Length
 211  *             and the omitted lower part must be reconstituted from
 212  *             the toUnicode data
 213  *         }
 214  *     }
 215  *
 216  *     -- stage 3 tables with byte results
 217  *     if(outputType==MBCS_OUTPUT_1) {
 218  *         -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
 219  *         uint16_t fromUBytes[fromUBytesLength/2];
 220  *     } else if(!(options&MBCS_OPT_NO_FROM_U)) {
 221  *         -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
 222  *         uint8_t fromUBytes[fromUBytesLength]; or
 223  *         uint16_t fromUBytes[fromUBytesLength/2]; or
 224  *         uint32_t fromUBytes[fromUBytesLength/4];
 225  *     } else {
 226  *         fromUBytes[] must be reconstituted from the toUnicode data
 227  *     }
 228  *
 229  *     -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher
 230  *     if(outputType!=MBCS_OUTPUT_1 &&
 231  *        _MBCSHeader.version[1]>=3 &&
 232  *        (maxFastUChar=_MBCSHeader.version[2])!=0
 233  *     ) {
 234  *         maxFastUChar=(maxFastUChar<<8)|0xff;
 235  *         uint16_t mbcsIndex[(maxFastUChar+1)>>6];
 236  *     }
 237  * }
 238  *
 239  * -- extension table, details see ucnv_ext.h
 240  * int32_t indexes[>=32]; ...
 241  */
 242
 243 /* MBCS converter data and state -------------------------------------------- */
 244
 245 enum {
 246     MBCS_MAX_STATE_COUNT=128
 247 };
 248
 249 /**
 250  * MBCS action codes for conversions to Unicode.
 251  * These values are in bits 23..20 of the state table entries.
 252  */
 253 enum {
 254     MBCS_STATE_VALID_DIRECT_16,
 255     MBCS_STATE_VALID_DIRECT_20,
 256
 257     MBCS_STATE_FALLBACK_DIRECT_16,
 258     MBCS_STATE_FALLBACK_DIRECT_20,
 259
 260     MBCS_STATE_VALID_16,
 261     MBCS_STATE_VALID_16_PAIR,
 262
 263     MBCS_STATE_UNASSIGNED,
 264     MBCS_STATE_ILLEGAL,
 265
 266     MBCS_STATE_CHANGE_ONLY
 267 };
 268
 269 /* Macros for state table entries */
 270 #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset))
 271 #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset))
 272 #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))
 273
 274 #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value))
 275 #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000)
 276 #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L))
 277 #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value))
 278 #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value))
 279
 280 #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L))
 281
 282 #define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
 283
 284 #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
 285 #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)
 286
 287 #define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24)
 288 #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)
 289
 290 #define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
 291 #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
 292 #define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf)
 293 #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
 294 #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)
 295
 296 #define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0)
 297
 298 /* single-byte fromUnicode: get the 16-bit result word */
 299 #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]
 300
 301 /* single-byte fromUnicode using the sbcsIndex */
 302 #define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
 303
 304 /* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */
 305 #define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ]
 306
 307 /* multi-byte fromUnicode: get the 32-bit stage 2 entry */
 308 #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
 309 #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)
 310
 311 #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
 312 #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
 313
 314 #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3)
 315
 316 /* double-byte fromUnicode using the mbcsIndex */
 317 #define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
 318
 319 /* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */
 320 #define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ]
 321
 322
 323 /**
 324  * MBCS output types for conversions from Unicode.
 325  * These per-converter types determine the storage method in stage 3 of the lookup table,
 326  * mostly how many bytes are stored per entry.
 327  */
 328 enum {
 329     MBCS_OUTPUT_1,          /* 0 */
 330     MBCS_OUTPUT_2,          /* 1 */
 331     MBCS_OUTPUT_3,          /* 2 */
 332     MBCS_OUTPUT_4,          /* 3 */
 333
 334     MBCS_OUTPUT_3_EUC=8,    /* 8 */
 335     MBCS_OUTPUT_4_EUC,      /* 9 */
 336
 337     MBCS_OUTPUT_2_SISO=12,  /* c */
 338     MBCS_OUTPUT_2_HZ,       /* d */
 339
 340     MBCS_OUTPUT_EXT_ONLY,   /* e */
 341
 342     MBCS_OUTPUT_COUNT,
 343
 344     MBCS_OUTPUT_DBCS_ONLY=0xdb  /* runtime-only type for DBCS-only handling of SISO tables */
 345 };
 346
 347 /**
 348  * Fallbacks to Unicode are stored outside the normal state table and code point structures
 349  * in a vector of items of this type. They are sorted by offset.
 350  */
 351 typedef struct {
 352     uint32_t offset;
 353     UChar32 codePoint;
 354 } _MBCSToUFallback;
 355
 356 /** Constants for fast and UTF-8-friendly conversion. */
 357 enum {
 358     SBCS_FAST_MAX=0x0fff,               /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */
 359     SBCS_FAST_LIMIT=SBCS_FAST_MAX+1,    /* =0x1000 */
 360     MBCS_FAST_MAX=0xd7ff,               /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */
 361     MBCS_FAST_LIMIT=MBCS_FAST_MAX+1     /* =0xd800 */
 362 };
 363
 364 /**
 365  * This is the MBCS part of the UConverterTable union (a runtime data structure).
 366  * It keeps all the per-converter data and points into the loaded mapping tables.
 367  *
 368  * utf8Friendly data structures added with _MBCSHeader.version 4.3
 369  */
 370 typedef struct UConverterMBCSTable {
 371     /* toUnicode */
 372     uint8_t countStates, dbcsOnlyState, stateTableOwned;
 373     uint32_t countToUFallbacks;
 374
 375     const int32_t (*stateTable)/*[countStates]*/[256];
 376     int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */
 377     const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/;
 378     const _MBCSToUFallback *toUFallbacks;
 379
 380     /* fromUnicode */
 381     const uint16_t *fromUnicodeTable;
 382     const uint16_t *mbcsIndex;              /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */
 383     uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */
 384     const uint8_t *fromUnicodeBytes;
 385     uint8_t *swapLFNLFromUnicodeBytes;      /* for swaplfnl */
 386     uint32_t fromUBytesLength;
 387     uint8_t outputType, unicodeMask;
 388     UBool utf8Friendly;                     /* for utf8Friendly data */
 389     UChar maxFastUChar;                     /* for utf8Friendly data */
 390
 391     /* roundtrips */
 392     uint32_t asciiRoundtrips;
 393
 394     /* reconstituted data that was omitted from the .cnv file */
 395     uint8_t *reconstitutedData;
 396
 397     /* converter name for swaplfnl */
 398     char *swapLFNLName;
 399
 400     /* extension data */
 401     struct UConverterSharedData *baseSharedData;
 402     const int32_t *extIndexes;
 403 } UConverterMBCSTable;
 404
 405 #define UCNV_MBCS_TABLE_INITIALIZER { \
 406     /* toUnicode */ \
 407     0, 0, 0, \
 408     0, \
 409      \
 410     NULL, \
 411     NULL, \
 412     NULL, \
 413     NULL, \
 414      \
 415     /* fromUnicode */ \
 416     NULL, \
 417     NULL, \
 418     { 0 }, \
 419     NULL, \
 420     NULL, \
 421     0, \
 422     0, 0, \
 423     FALSE, \
 424     0, \
 425      \
 426     /* roundtrips */ \
 427     0, \
 428      \
 429     /* reconstituted data that was omitted from the .cnv file */ \
 430     NULL, \
 431      \
 432     /* converter name for swaplfnl */ \
 433     NULL, \
 434      \
 435     /* extension data */ \
 436     NULL, \
 437     NULL \
 438 }
 439
 440 enum {
 441     MBCS_OPT_LENGTH_MASK=0x3f,
 442     MBCS_OPT_NO_FROM_U=0x40,
 443     /*
 444      * If any of the following options bits are set,
 445      * then the file must be rejected.
 446      */
 447     MBCS_OPT_INCOMPATIBLE_MASK=0xffc0,
 448     /*
 449      * Remove bits from this mask as more options are recognized
 450      * by all implementations that use this constant.
 451      */
 452     MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80
 453 };
 454
 455 enum {
 456     MBCS_HEADER_V4_LENGTH=8,
 457     MBCS_HEADER_V5_MIN_LENGTH=9
 458 };
 459
 460 /**
 461  * MBCS data header. See data format description above.
 462  */
 463 typedef struct {
 464     UVersionInfo version;
 465     uint32_t countStates,
 466              countToUFallbacks,
 467              offsetToUCodeUnits,
 468              offsetFromUTable,
 469              offsetFromUBytes,
 470              flags,
 471              fromUBytesLength;
 472
 473     /* new and required in version 5 */
 474     uint32_t options;
 475
 476     /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */
 477     uint32_t fullStage2Length;  /* number of 32-bit units */
 478 } _MBCSHeader;
 479
 480 #define UCNV_MBCS_HEADER_INITIALIZER { { 0 },  0, 0, 0, 0, 0, 0, 0,  0,  0 }
 481
 482 /*
 483  * This is a simple version of _MBCSGetNextUChar() that is used
 484  * by other converter implementations.
 485  * It only returns an "assigned" result if it consumes the entire input.
 486  * It does not use state from the converter, nor error codes.
 487  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 488  * It handles conversion extensions but not GB 18030.
 489  *
 490  * Return value:
 491  * U+fffe   unassigned
 492  * U+ffff   illegal
 493  * otherwise the Unicode code point
 494  */
 495 U_CFUNC UChar32
 496 ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
 497                         const char *source, int32_t length,
 498                         UBool useFallback);
 499
 500 /**
 501  * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
 502  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 503  * It does not handle conversion extensions (_extToU()).
 504  */
 505 U_CFUNC UChar32
 506 ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
 507                               uint8_t b, UBool useFallback);
 508
 509 /**
 510  * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
 511  * It works for single-byte, single-state codepages that only map
 512  * to and from BMP code points, and it always
 513  * returns fallback values.
 514  */
 515 #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
 516     (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])
 517
 518 /**
 519  * This is an internal function that allows other converter implementations
 520  * to check whether a byte is a lead byte.
 521  */
 522 U_CFUNC UBool
 523 ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);
 524
 525 /** This is a macro version of _MBCSIsLeadByte(). */
 526 #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
 527     (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])
 528
 529 /*
 530  * This is another simple conversion function for internal use by other
 531  * conversion implementations.
 532  * It does not use the converter state nor call callbacks.
 533  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 534  * It handles conversion extensions but not GB 18030.
 535  *
 536  * It converts one single Unicode code point into codepage bytes, encoded
 537  * as one 32-bit value. The function returns the number of bytes in *pValue:
 538  * 1..4 the number of bytes in *pValue
 539  * 0    unassigned (*pValue undefined)
 540  * -1   illegal (currently not used, *pValue undefined)
 541  *
 542  * *pValue will contain the resulting bytes with the last byte in bits 7..0,
 543  * the second to last byte in bits 15..8, etc.
 544  * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
 545  */
 546 U_CFUNC int32_t
 547 ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
 548                  UChar32 c, uint32_t *pValue,
 549                  UBool useFallback);
 550
 551 /**
 552  * This version of _MBCSFromUChar32() is optimized for single-byte codepages.
 553  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 554  *
 555  * It returns the codepage byte for the code point, or -1 if it is unassigned.
 556  */
 557 U_CFUNC int32_t
 558 ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
 559                        UChar32 c,
 560                        UBool useFallback);
 561
 562 /**
 563  * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
 564  * we cheat a little about the type, returning the old types if appropriate.
 565  */
 566 U_CFUNC UConverterType
 567 ucnv_MBCSGetType(const UConverter* converter);
 568
 569 U_CFUNC void
 570 ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
 571                             UErrorCode *pErrorCode);
 572 U_CFUNC void
 573 ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
 574                           UErrorCode *pErrorCode);
 575
 576 /*
 577  * Internal function returning a UnicodeSet for toUnicode() conversion.
 578  * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
 579  * In the future, if we add support for fallback sets, this function
 580  * needs to be updated.
 581  * Handles extensions.
 582  * Does not empty the set first.
 583  */
 584 U_CFUNC void
 585 ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
 586                                  const USetAdder *sa,
 587                                  UConverterUnicodeSet which,
 588                                  UErrorCode *pErrorCode);
 589
 590 /*
 591  * Same as ucnv_MBCSGetUnicodeSetForUnicode() but
 592  * the set can be filtered by encoding scheme.
 593  * Used by stateful converters which share regular conversion tables
 594  * but only use a subset of their mappings.
 595  */
 596 U_CFUNC void
 597 ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData,
 598                                          const USetAdder *sa,
 599                                          UConverterUnicodeSet which,
 600                                          UConverterSetFilter filter,
 601                                          UErrorCode *pErrorCode);
 602
 603 #endif
 604
 605 #endif