X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/374ca955a76ecab1204ca8bfa63ff9238d998416..a01113dcd0f39d5da295ef82785beff9ed86fe38:/icuSources/common/ucnvmbcs.h diff --git a/icuSources/common/ucnvmbcs.h b/icuSources/common/ucnvmbcs.h index 4ac4358a..209cdc53 100644 --- a/icuSources/common/ucnvmbcs.h +++ b/icuSources/common/ucnvmbcs.h @@ -1,12 +1,14 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html /* ****************************************************************************** * -* Copyright (C) 2000-2004, International Business Machines +* Copyright (C) 2000-2013, International Business Machines * Corporation and others. All Rights Reserved. * ****************************************************************************** * file name: ucnvmbcs.h -* encoding: US-ASCII +* encoding: UTF-8 * tab size: 8 (not used) * indentation:4 * @@ -23,6 +25,7 @@ #include "unicode/ucnv.h" #include "ucnv_cnv.h" +#include "ucnv_ext.h" /** * ICU conversion (.cnv) data file structure, following the usual UDataInfo @@ -41,6 +44,80 @@ * the same toUnicode structures, while the fromUnicode structures for SBCS * differ from those for other MBCS-style converters. * + * _MBCSHeader.version 5 is optional and not backward-compatible + * (as usual for changes in the major version field). + * + * Versions 5.m work like versions 4.m except: + * - The _MBCSHeader has variable length (and is always longer than in version 4). + * See the struct _MBCSHeader further description below. + * - There is a set of flags which indicate further incompatible changes. + * (Reader code must reject the file if it does not recognize them all.) + * - In particular, one of these flags indicates that most of the fromUnicode + * data is missing and must be reconstituted from the toUnicode data + * and from the utf8Friendly mbcsIndex at load time. + * (This only works with a utf8Friendly table.) + * In this case, makeconv may increase maxFastUChar automatically to U+FFFF. + * + * The first of these versions is 5.3, which is like 4.3 except for the differences above. + * + * When possible, makeconv continues to generate version 4.m files. + * + * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4) + * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details. + * + * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures + * slightly and optionally adds a table for conversion to MBCS (non-SBCS) + * charsets. + * + * The modifications are to make the data utf8Friendly. Not every 4.3 file + * file contains utf8Friendly data. + * It is utf8Friendly if _MBCSHeader.version[2]!=0. + * In this case, the data structures are utf8Friendly up to the code point + * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff) + * + * A utf8Friendly file has fromUnicode stage 3 entries for code points up to + * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from + * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries. + * + * In addition, a utf8Friendly MBCS file contains an additional + * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; + * which replaces the stage 1 and 2 tables for indexing with bits from the + * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table, + * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks + * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to + * the extension data structure. This also allows for faster roundtrip + * conversion from UTF-16. + * + * SBCS files do not contain an additional sbcsIndex[] array because the + * proportional size increase would be noticeable, but the runtime + * code builds one for the code point range for which the runtime conversion + * code is optimized. + * + * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv + * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in + * the file, a larger maxFastUChar only affects stage 3 block allocation size + * and is free in empty blocks. (Larger blocks with sparse contents cause larger + * files.) U+1FFF includes almost all of the small scripts. + * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with + * 0xe0. This includes most scripts with legacy SBCS charsets. + * The initial runtime implementation using 4.3 files only builds an sbcsIndex + * for code points up to U+0FFF. + * + * For MBCS, maxFastUChar should be at least U+D7FF (=initial value). + * This boundary is convenient because practically all of the commonly used + * characters are below it, and because it is the boundary to surrogate + * code points, above which special handling is necessary anyway. + * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.) + * + * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be + * useful especially for conversion from UTF-8 when the input can be assumed + * to be valid, because the surrogate range would then not have to be + * checked. + * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value + * overflow because with the all-unassigned block 0 and nearly full mappings + * from the BMP it is theoretically possible that an index into stage 3 + * exceeds 16 bits.) + * * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. * If it is present, then an ICU version reading header versions 4.0 or 4.1 * will be able to use the base table and ignore the extension. @@ -60,7 +137,7 @@ * struct _MBCSHeader (see the definition in this header file below) * contains 32-bit fields as follows: * 8 values: - * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.2.0.0) + * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0) * 1 uint32_t countStates * 2 uint32_t countToUFallbacks * 3 uint32_t offsetToUCodeUnits @@ -74,6 +151,26 @@ * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher * counts bytes in fromUBytes[] * + * New and required in version 5: + * 8 uint32_t options, bits: + * 31..16 reserved for flags that can be added without breaking + * backward compatibility + * 15.. 6 reserved for flags whose addition will break + * backward compatibility + * 6 MBCS_OPT_FROM_U -- if set, + * then most of the fromUnicode data is omitted; + * fullStage2Length is present and the missing + * bottom part of stage 2 must be reconstituted from + * the toUnicode data; + * stage 3 is missing completely as well; + * not used for SBCS tables + * 5.. 0 length of the _MBCSHeader (number of uint32_t) + * + * New and optional in version 5: + * 9 uint32_t fullStage2Length: used if MBCS_OPT_FROM_U is set + * specifies the full length of stage 2 + * including the omitted part + * * if(outputType==MBCS_OUTPUT_EXT_ONLY) { * -- base table name for extension-only table * char baseTableName[variable]; -- with NUL plus padding for 4-alignment @@ -100,7 +197,7 @@ * -- BMP-only tables have a smaller stage 1 table * uint16_t fromUTable[0x40]; (32-bit-aligned) * } - * + * * -- stage 2 tables * length determined by top of stage 1 and bottom of stage 3 tables * if(outputType==MBCS_OUTPUT_1) { @@ -109,17 +206,33 @@ * } else { * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes * uint32_t stage 2 flags and indexes[?]; + * if(options&MBCS_OPT_NO_FROM_U) { + * stage 2 really has length fullStage2Length + * and the omitted lower part must be reconstituted from + * the toUnicode data + * } * } - * + * * -- stage 3 tables with byte results * if(outputType==MBCS_OUTPUT_1) { * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c * uint16_t fromUBytes[fromUBytesLength/2]; - * } else { + * } else if(!(options&MBCS_OPT_NO_FROM_U)) { * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c * uint8_t fromUBytes[fromUBytesLength]; or * uint16_t fromUBytes[fromUBytesLength/2]; or * uint32_t fromUBytes[fromUBytesLength/4]; + * } else { + * fromUBytes[] must be reconstituted from the toUnicode data + * } + * + * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher + * if(outputType!=MBCS_OUTPUT_1 && + * _MBCSHeader.version[1]>=3 && + * (maxFastUChar=_MBCSHeader.version[2])!=0 + * ) { + * maxFastUChar=(maxFastUChar<<8)|0xff; + * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; * } * } * @@ -166,23 +279,31 @@ enum { #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L)) -#define MBCS_ENTRY_STATE(entry) (((entry)>>24)&0x7f) +#define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0) #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0) -#define MBCS_ENTRY_TRANSITION_STATE(entry) ((entry)>>24) +#define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24) #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff) -#define MBCS_ENTRY_FINAL_STATE(entry) (((entry)>>24)&0x7f) +#define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000) -#define MBCS_ENTRY_FINAL_ACTION(entry) (((entry)>>20)&0xf) +#define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf) #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff) #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry) +#define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0) + /* single-byte fromUnicode: get the 16-bit result word */ #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ] +/* single-byte fromUnicode using the sbcsIndex */ +#define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] + +/* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */ +#define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ] + /* multi-byte fromUnicode: get the 32-bit stage 2 entry */ #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0) @@ -192,6 +313,12 @@ enum { #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3) +/* double-byte fromUnicode using the mbcsIndex */ +#define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] + +/* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */ +#define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ] + /** * MBCS output types for conversions from Unicode. @@ -226,9 +353,19 @@ typedef struct { UChar32 codePoint; } _MBCSToUFallback; +/** Constants for fast and UTF-8-friendly conversion. */ +enum { + SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */ + SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */ + MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */ + MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */ +}; + /** * This is the MBCS part of the UConverterTable union (a runtime data structure). * It keeps all the per-converter data and points into the loaded mapping tables. + * + * utf8Friendly data structures added with _MBCSHeader.version 4.3 */ typedef struct UConverterMBCSTable { /* toUnicode */ @@ -242,10 +379,20 @@ typedef struct UConverterMBCSTable { /* fromUnicode */ const uint16_t *fromUnicodeTable; + const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */ + uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */ const uint8_t *fromUnicodeBytes; - uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ + uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ uint32_t fromUBytesLength; uint8_t outputType, unicodeMask; + UBool utf8Friendly; /* for utf8Friendly data */ + UChar maxFastUChar; /* for utf8Friendly data */ + + /* roundtrips */ + uint32_t asciiRoundtrips; + + /* reconstituted data that was omitted from the .cnv file */ + uint8_t *reconstitutedData; /* converter name for swaplfnl */ char *swapLFNLName; @@ -255,6 +402,61 @@ typedef struct UConverterMBCSTable { const int32_t *extIndexes; } UConverterMBCSTable; +#define UCNV_MBCS_TABLE_INITIALIZER { \ + /* toUnicode */ \ + 0, 0, 0, \ + 0, \ + \ + NULL, \ + NULL, \ + NULL, \ + NULL, \ + \ + /* fromUnicode */ \ + NULL, \ + NULL, \ + { 0 }, \ + NULL, \ + NULL, \ + 0, \ + 0, 0, \ + FALSE, \ + 0, \ + \ + /* roundtrips */ \ + 0, \ + \ + /* reconstituted data that was omitted from the .cnv file */ \ + NULL, \ + \ + /* converter name for swaplfnl */ \ + NULL, \ + \ + /* extension data */ \ + NULL, \ + NULL \ +} + +enum { + MBCS_OPT_LENGTH_MASK=0x3f, + MBCS_OPT_NO_FROM_U=0x40, + /* + * If any of the following options bits are set, + * then the file must be rejected. + */ + MBCS_OPT_INCOMPATIBLE_MASK=0xffc0, + /* + * Remove bits from this mask as more options are recognized + * by all implementations that use this constant. + */ + MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80 +}; + +enum { + MBCS_HEADER_V4_LENGTH=8, + MBCS_HEADER_V5_MIN_LENGTH=9 +}; + /** * MBCS data header. See data format description above. */ @@ -267,8 +469,16 @@ typedef struct { offsetFromUBytes, flags, fromUBytesLength; + + /* new and required in version 5 */ + uint32_t options; + + /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */ + uint32_t fullStage2Length; /* number of 32-bit units */ } _MBCSHeader; +#define UCNV_MBCS_HEADER_INITIALIZER { { 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 } + /* * This is a simple version of _MBCSGetNextUChar() that is used * by other converter implementations. @@ -363,21 +573,6 @@ U_CFUNC void ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, UErrorCode *pErrorCode); -/* - * Internal function returning a UnicodeSet for toUnicode() conversion. - * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. - * In the future, if we add support for reverse-fallback sets, this function - * needs to be updated, and called for each initial state. - * Does not currently handle extensions. - * Does not empty the set first. - */ -U_CFUNC void -ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData, - USetAdder *sa, - UConverterUnicodeSet which, - uint8_t state, int32_t lowByte, int32_t highByte, - UErrorCode *pErrorCode); - /* * Internal function returning a UnicodeSet for toUnicode() conversion. * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. @@ -388,9 +583,22 @@ ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData, */ U_CFUNC void ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, - USetAdder *sa, - UConverterUnicodeSet which, - UErrorCode *pErrorCode); + const USetAdder *sa, + UConverterUnicodeSet which, + UErrorCode *pErrorCode); + +/* + * Same as ucnv_MBCSGetUnicodeSetForUnicode() but + * the set can be filtered by encoding scheme. + * Used by stateful converters which share regular conversion tables + * but only use a subset of their mappings. + */ +U_CFUNC void +ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, + const USetAdder *sa, + UConverterUnicodeSet which, + UConverterSetFilter filter, + UErrorCode *pErrorCode); #endif