[apple/icu.git] / icuSources / common / ucnvmbcs.h

/*
******************************************************************************
*
*   Copyright (C) 2000-2004, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
******************************************************************************
*   file name:  ucnvmbcs.h
*   encoding:   US-ASCII
*   tab size:   8 (not used)
*   indentation:4
*
*   created on: 2000jul07
*   created by: Markus W. Scherer
*/

#ifndef __UCNVMBCS_H__
#define __UCNVMBCS_H__

#include "unicode/utypes.h"

#if !UCONFIG_NO_CONVERSION

#include "unicode/ucnv.h"
#include "ucnv_cnv.h"

/**
 * ICU conversion (.cnv) data file structure, following the usual UDataInfo
 * header.
 *
 * Format version: 6.2
 *
 * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
 *                                min/max bytes per character, etc.
 *                                see ucnv_bld.h
 *
 * --------------------
 *
 * The static data is followed by conversionType-specific data structures.
 * At the moment, there are only variations of MBCS converters. They all have
 * the same toUnicode structures, while the fromUnicode structures for SBCS
 * differ from those for other MBCS-style converters.
 *
 * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
 * If it is present, then an ICU version reading header versions 4.0 or 4.1
 * will be able to use the base table and ignore the extension.
 *
 * The unicodeMask in the static data is part of the base table data structure.
 * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
 * fromUnicode stage 1 array.
 * The static data unicodeMask refers only to the base table's properties if
 * a base table is included.
 * In an extension-only file, the static data unicodeMask is 0.
 * The extension data indexes have a separate field with the unicodeMask flags.
 *
 * MBCS-style data structure following the static data.
 * Offsets are counted in bytes from the beginning of the MBCS header structure.
 * Details about usage in comments in ucnvmbcs.c.
 *
 * struct _MBCSHeader (see the definition in this header file below)
 * contains 32-bit fields as follows:
 * 8 values:
 *  0   uint8_t[4]  MBCS version in UVersionInfo format (currently 4.2.0.0)
 *  1   uint32_t    countStates
 *  2   uint32_t    countToUFallbacks
 *  3   uint32_t    offsetToUCodeUnits
 *  4   uint32_t    offsetFromUTable
 *  5   uint32_t    offsetFromUBytes
 *  6   uint32_t    flags, bits:
 *                      31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
 *                                                0 for older versions and if
 *                                                there is not extension structure
 *                       7.. 0 outputType
 *  7   uint32_t    fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
 *                  counts bytes in fromUBytes[]
 *
 * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
 *     -- base table name for extension-only table
 *     char baseTableName[variable]; -- with NUL plus padding for 4-alignment
 *
 *     -- all _MBCSHeader fields except for version and flags are 0
 * } else {
 *     -- normal base table with optional extension
 *
 *     int32_t stateTable[countStates][256];
 *    
 *     struct _MBCSToUFallback { (fallbacks are sorted by offset)
 *         uint32_t offset;
 *         UChar32 codePoint;
 *     } toUFallbacks[countToUFallbacks];
 *    
 *     uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
 *                  (padded to an even number of units)
 *    
 *     -- stage 1 tables
 *     if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
 *         -- stage 1 table for all of Unicode
 *         uint16_t fromUTable[0x440]; (32-bit-aligned)
 *     } else {
 *         -- BMP-only tables have a smaller stage 1 table
 *         uint16_t fromUTable[0x40]; (32-bit-aligned)
 *     }
 *    
 *     -- stage 2 tables
 *        length determined by top of stage 1 and bottom of stage 3 tables
 *     if(outputType==MBCS_OUTPUT_1) {
 *         -- SBCS: pure indexes
 *         uint16_t stage 2 indexes[?];
 *     } else {
 *         -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
 *         uint32_t stage 2 flags and indexes[?];
 *     }
 *    
 *     -- stage 3 tables with byte results
 *     if(outputType==MBCS_OUTPUT_1) {
 *         -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
 *         uint16_t fromUBytes[fromUBytesLength/2];
 *     } else {
 *         -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
 *         uint8_t fromUBytes[fromUBytesLength]; or
 *         uint16_t fromUBytes[fromUBytesLength/2]; or
 *         uint32_t fromUBytes[fromUBytesLength/4];
 *     }
 * }
 *
 * -- extension table, details see ucnv_ext.h
 * int32_t indexes[>=32]; ...
 */

/* MBCS converter data and state -------------------------------------------- */

enum {
    MBCS_MAX_STATE_COUNT=128
};

/**
 * MBCS action codes for conversions to Unicode.
 * These values are in bits 23..20 of the state table entries.
 */
enum {
    MBCS_STATE_VALID_DIRECT_16,
    MBCS_STATE_VALID_DIRECT_20,

    MBCS_STATE_FALLBACK_DIRECT_16,
    MBCS_STATE_FALLBACK_DIRECT_20,

    MBCS_STATE_VALID_16,
    MBCS_STATE_VALID_16_PAIR,

    MBCS_STATE_UNASSIGNED,
    MBCS_STATE_ILLEGAL,

    MBCS_STATE_CHANGE_ONLY
};

/* Macros for state table entries */
#define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset))
#define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset))
#define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))

#define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value))
#define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000)
#define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L))
#define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value))
#define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value))

#define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L))

#define MBCS_ENTRY_STATE(entry) (((entry)>>24)&0x7f)

#define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
#define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)

#define MBCS_ENTRY_TRANSITION_STATE(entry) ((entry)>>24)
#define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)

#define MBCS_ENTRY_FINAL_STATE(entry) (((entry)>>24)&0x7f)
#define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
#define MBCS_ENTRY_FINAL_ACTION(entry) (((entry)>>20)&0xf)
#define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
#define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)

/* single-byte fromUnicode: get the 16-bit result word */
#define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]

/* multi-byte fromUnicode: get the 32-bit stage 2 entry */
#define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
#define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)

#define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
#define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]

#define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3)


/**
 * MBCS output types for conversions from Unicode.
 * These per-converter types determine the storage method in stage 3 of the lookup table,
 * mostly how many bytes are stored per entry.
 */
enum {
    MBCS_OUTPUT_1,          /* 0 */
    MBCS_OUTPUT_2,          /* 1 */
    MBCS_OUTPUT_3,          /* 2 */
    MBCS_OUTPUT_4,          /* 3 */

    MBCS_OUTPUT_3_EUC=8,    /* 8 */
    MBCS_OUTPUT_4_EUC,      /* 9 */

    MBCS_OUTPUT_2_SISO=12,  /* c */
    MBCS_OUTPUT_2_HZ,       /* d */

    MBCS_OUTPUT_EXT_ONLY,   /* e */

    MBCS_OUTPUT_COUNT,

    MBCS_OUTPUT_DBCS_ONLY=0xdb  /* runtime-only type for DBCS-only handling of SISO tables */
};

/**
 * Fallbacks to Unicode are stored outside the normal state table and code point structures
 * in a vector of items of this type. They are sorted by offset.
 */
typedef struct {
    uint32_t offset;
    UChar32 codePoint;
} _MBCSToUFallback;

/**
 * This is the MBCS part of the UConverterTable union (a runtime data structure).
 * It keeps all the per-converter data and points into the loaded mapping tables.
 */
typedef struct UConverterMBCSTable {
    /* toUnicode */
    uint8_t countStates, dbcsOnlyState, stateTableOwned;
    uint32_t countToUFallbacks;

    const int32_t (*stateTable)/*[countStates]*/[256];
    int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */
    const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/;
    const _MBCSToUFallback *toUFallbacks;

    /* fromUnicode */
    const uint16_t *fromUnicodeTable;
    const uint8_t *fromUnicodeBytes;
    uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */
    uint32_t fromUBytesLength;
    uint8_t outputType, unicodeMask;

    /* converter name for swaplfnl */
    char *swapLFNLName;

    /* extension data */
    struct UConverterSharedData *baseSharedData;
    const int32_t *extIndexes;
} UConverterMBCSTable;

/**
 * MBCS data header. See data format description above.
 */
typedef struct {
    UVersionInfo version;
    uint32_t countStates,
             countToUFallbacks,
             offsetToUCodeUnits,
             offsetFromUTable,
             offsetFromUBytes,
             flags,
             fromUBytesLength;
} _MBCSHeader;

/*
 * This is a simple version of _MBCSGetNextUChar() that is used
 * by other converter implementations.
 * It only returns an "assigned" result if it consumes the entire input.
 * It does not use state from the converter, nor error codes.
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 * It handles conversion extensions but not GB 18030.
 *
 * Return value:
 * U+fffe   unassigned
 * U+ffff   illegal
 * otherwise the Unicode code point
 */
U_CFUNC UChar32
ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
                        const char *source, int32_t length,
                        UBool useFallback);

/**
 * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 * It does not handle conversion extensions (_extToU()).
 */
U_CFUNC UChar32
ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
                              uint8_t b, UBool useFallback);

/**
 * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
 * It works for single-byte, single-state codepages that only map
 * to and from BMP code points, and it always
 * returns fallback values.
 */
#define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
    (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])

/**
 * This is an internal function that allows other converter implementations
 * to check whether a byte is a lead byte.
 */
U_CFUNC UBool
ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);

/** This is a macro version of _MBCSIsLeadByte(). */
#define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
    (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])

/*
 * This is another simple conversion function for internal use by other
 * conversion implementations.
 * It does not use the converter state nor call callbacks.
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 * It handles conversion extensions but not GB 18030.
 *
 * It converts one single Unicode code point into codepage bytes, encoded
 * as one 32-bit value. The function returns the number of bytes in *pValue:
 * 1..4 the number of bytes in *pValue
 * 0    unassigned (*pValue undefined)
 * -1   illegal (currently not used, *pValue undefined)
 *
 * *pValue will contain the resulting bytes with the last byte in bits 7..0,
 * the second to last byte in bits 15..8, etc.
 * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
 */
U_CFUNC int32_t
ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
                 UChar32 c, uint32_t *pValue,
                 UBool useFallback);

/**
 * This version of _MBCSFromUChar32() is optimized for single-byte codepages.
 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
 *
 * It returns the codepage byte for the code point, or -1 if it is unassigned.
 */
U_CFUNC int32_t
ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
                       UChar32 c,
                       UBool useFallback);

/**
 * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
 * we cheat a little about the type, returning the old types if appropriate.
 */
U_CFUNC UConverterType
ucnv_MBCSGetType(const UConverter* converter);

U_CFUNC void 
ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
                            UErrorCode *pErrorCode);
U_CFUNC void 
ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
                          UErrorCode *pErrorCode);

/*
 * Internal function returning a UnicodeSet for toUnicode() conversion.
 * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
 * In the future, if we add support for reverse-fallback sets, this function
 * needs to be updated, and called for each initial state.
 * Does not currently handle extensions.
 * Does not empty the set first.
 */
U_CFUNC void
ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData,
                           const USetAdder *sa,
                           UConverterUnicodeSet which,
                           uint8_t state, int32_t lowByte, int32_t highByte,
                           UErrorCode *pErrorCode);

/*
 * Internal function returning a UnicodeSet for toUnicode() conversion.
 * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
 * In the future, if we add support for fallback sets, this function
 * needs to be updated.
 * Handles extensions.
 * Does not empty the set first.
 */
U_CFUNC void
ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
                             const USetAdder *sa,
                             UConverterUnicodeSet which,
                             UErrorCode *pErrorCode);

#endif

#endif
Commit	Line	Data
b75a7d8f A	1	/*
	2	******************************************************************************
	3	*
374ca955	4	* Copyright (C) 2000-2004, International Business Machines
b75a7d8f A	5	* Corporation and others. All Rights Reserved.
	6	*
	7	******************************************************************************
	8	* file name: ucnvmbcs.h
	9	* encoding: US-ASCII
	10	* tab size: 8 (not used)
	11	* indentation:4
	12	*
	13	* created on: 2000jul07
	14	* created by: Markus W. Scherer
	15	*/
	16
	17	#ifndef __UCNVMBCS_H__
	18	#define __UCNVMBCS_H__
	19
	20	#include "unicode/utypes.h"
374ca955 A	21
	22	#if !UCONFIG_NO_CONVERSION
	23
b75a7d8f	24	#include "unicode/ucnv.h"
374ca955 A	25	#include "ucnv_cnv.h"
	26
	27	/**
	28	* ICU conversion (.cnv) data file structure, following the usual UDataInfo
	29	* header.
	30	*
	31	* Format version: 6.2
	32	*
	33	* struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
	34	* min/max bytes per character, etc.
	35	* see ucnv_bld.h
	36	*
	37	* --------------------
	38	*
	39	* The static data is followed by conversionType-specific data structures.
	40	* At the moment, there are only variations of MBCS converters. They all have
	41	* the same toUnicode structures, while the fromUnicode structures for SBCS
	42	* differ from those for other MBCS-style converters.
	43	*
	44	* _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
	45	* If it is present, then an ICU version reading header versions 4.0 or 4.1
	46	* will be able to use the base table and ignore the extension.
	47	*
	48	* The unicodeMask in the static data is part of the base table data structure.
	49	* Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
	50	* fromUnicode stage 1 array.
	51	* The static data unicodeMask refers only to the base table's properties if
	52	* a base table is included.
	53	* In an extension-only file, the static data unicodeMask is 0.
	54	* The extension data indexes have a separate field with the unicodeMask flags.
	55	*
	56	* MBCS-style data structure following the static data.
	57	* Offsets are counted in bytes from the beginning of the MBCS header structure.
	58	* Details about usage in comments in ucnvmbcs.c.
	59	*
	60	* struct _MBCSHeader (see the definition in this header file below)
	61	* contains 32-bit fields as follows:
	62	* 8 values:
	63	* 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.2.0.0)
	64	* 1 uint32_t countStates
	65	* 2 uint32_t countToUFallbacks
	66	* 3 uint32_t offsetToUCodeUnits
	67	* 4 uint32_t offsetFromUTable
	68	* 5 uint32_t offsetFromUBytes
	69	* 6 uint32_t flags, bits:
	70	* 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
	71	* 0 for older versions and if
	72	* there is not extension structure
	73	* 7.. 0 outputType
	74	* 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
	75	* counts bytes in fromUBytes[]
	76	*
	77	* if(outputType==MBCS_OUTPUT_EXT_ONLY) {
	78	* -- base table name for extension-only table
	79	* char baseTableName[variable]; -- with NUL plus padding for 4-alignment
	80	*
	81	* -- all _MBCSHeader fields except for version and flags are 0
	82	* } else {
	83	* -- normal base table with optional extension
	84	*
	85	* int32_t stateTable[countStates][256];
	86	*
	87	* struct _MBCSToUFallback { (fallbacks are sorted by offset)
	88	* uint32_t offset;
89	* UChar32 codePoint;
90	* } toUFallbacks[countToUFallbacks];
91	*
92	* uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
93	* (padded to an even number of units)
94	*
95	* -- stage 1 tables
96	* if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
97	* -- stage 1 table for all of Unicode
98	* uint16_t fromUTable[0x440]; (32-bit-aligned)
99	* } else {
100	* -- BMP-only tables have a smaller stage 1 table
101	* uint16_t fromUTable[0x40]; (32-bit-aligned)
102	* }
103	*
104	* -- stage 2 tables
105	* length determined by top of stage 1 and bottom of stage 3 tables
106	* if(outputType==MBCS_OUTPUT_1) {
107	* -- SBCS: pure indexes
108	* uint16_t stage 2 indexes[?];
109	* } else {
110	* -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
111	* uint32_t stage 2 flags and indexes[?];
112	* }
113	*
114	* -- stage 3 tables with byte results
115	* if(outputType==MBCS_OUTPUT_1) {
116	* -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
117	* uint16_t fromUBytes[fromUBytesLength/2];
118	* } else {
119	* -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
120	* uint8_t fromUBytes[fromUBytesLength]; or
121	* uint16_t fromUBytes[fromUBytesLength/2]; or
122	* uint32_t fromUBytes[fromUBytesLength/4];
123	* }
124	* }
125	*
126	* -- extension table, details see ucnv_ext.h
127	* int32_t indexes[>=32]; ...
128	*/
b75a7d8f A	129
	130	/* MBCS converter data and state -------------------------------------------- */
	131
374ca955 A	132	enum {
	133	MBCS_MAX_STATE_COUNT=128
	134	};
	135
b75a7d8f A	136	/**
	137	* MBCS action codes for conversions to Unicode.
	138	* These values are in bits 23..20 of the state table entries.
	139	*/
	140	enum {
	141	MBCS_STATE_VALID_DIRECT_16,
	142	MBCS_STATE_VALID_DIRECT_20,
	143
	144	MBCS_STATE_FALLBACK_DIRECT_16,
	145	MBCS_STATE_FALLBACK_DIRECT_20,
	146
	147	MBCS_STATE_VALID_16,
	148	MBCS_STATE_VALID_16_PAIR,
	149
	150	MBCS_STATE_UNASSIGNED,
	151	MBCS_STATE_ILLEGAL,
	152
	153	MBCS_STATE_CHANGE_ONLY
	154	};
	155
	156	/* Macros for state table entries */
	157	#define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)\|(offset))
	158	#define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)\|(offset))
	159	#define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))
	160
	161	#define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000\|((int32_t)(state)<<24L)\|((action)<<20L)\|(value))
	162	#define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)\|0x80000000)
	163	#define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)\|((int32_t)(action)<<20L))
	164	#define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)\|(value))
	165	#define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)\|((int32_t)(action)<<20L)\|(value))
	166
	167	#define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)\|((int32_t)(state)<<24L))
	168
	169	#define MBCS_ENTRY_STATE(entry) (((entry)>>24)&0x7f)
	170
	171	#define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
	172	#define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)
	173
	174	#define MBCS_ENTRY_TRANSITION_STATE(entry) ((entry)>>24)
	175	#define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)
	176
	177	#define MBCS_ENTRY_FINAL_STATE(entry) (((entry)>>24)&0x7f)
	178	#define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
	179	#define MBCS_ENTRY_FINAL_ACTION(entry) (((entry)>>20)&0xf)
	180	#define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
	181	#define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)
	182
	183	/* single-byte fromUnicode: get the 16-bit result word */
	184	#define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]
	185
	186	/* multi-byte fromUnicode: get the 32-bit stage 2 entry */
	187	#define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
	188	#define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)
	189
	190	#define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t )(bytes))[16(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
	191	#define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t )(bytes))[16(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
	192
	193	#define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))3)
	194
	195
	196	/**
	197	* MBCS output types for conversions from Unicode.
	198	* These per-converter types determine the storage method in stage 3 of the lookup table,
	199	* mostly how many bytes are stored per entry.
200	*/
201	enum {
202	MBCS_OUTPUT_1, /* 0 */
203	MBCS_OUTPUT_2, /* 1 */
204	MBCS_OUTPUT_3, /* 2 */
205	MBCS_OUTPUT_4, /* 3 */
206
207	MBCS_OUTPUT_3_EUC=8, /* 8 */
208	MBCS_OUTPUT_4_EUC, /* 9 */
209
210	MBCS_OUTPUT_2_SISO=12, /* c */
374ca955 A	211	MBCS_OUTPUT_2_HZ, /* d */
	212
	213	MBCS_OUTPUT_EXT_ONLY, /* e */
	214
	215	MBCS_OUTPUT_COUNT,
	216
	217	MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */
b75a7d8f A	218	};
	219
	220	/**
	221	* Fallbacks to Unicode are stored outside the normal state table and code point structures
	222	* in a vector of items of this type. They are sorted by offset.
	223	*/
	224	typedef struct {
	225	uint32_t offset;
	226	UChar32 codePoint;
	227	} _MBCSToUFallback;
	228
	229	/**
	230	* This is the MBCS part of the UConverterTable union (a runtime data structure).
	231	* It keeps all the per-converter data and points into the loaded mapping tables.
	232	*/
	233	typedef struct UConverterMBCSTable {
	234	/* toUnicode */
374ca955	235	uint8_t countStates, dbcsOnlyState, stateTableOwned;
b75a7d8f A	236	uint32_t countToUFallbacks;
	237
	238	const int32_t (stateTable)/[countStates]*/[256];
	239	int32_t (swapLFNLStateTable)/[countStates]/[256]; / for swaplfnl */
	240	const uint16_t unicodeCodeUnits/[countUnicodeResults]*/;
	241	const _MBCSToUFallback *toUFallbacks;
	242
	243	/* fromUnicode */
	244	const uint16_t *fromUnicodeTable;
	245	const uint8_t *fromUnicodeBytes;
	246	uint8_t swapLFNLFromUnicodeBytes; / for swaplfnl */
	247	uint32_t fromUBytesLength;
	248	uint8_t outputType, unicodeMask;
	249
	250	/* converter name for swaplfnl */
	251	char *swapLFNLName;
374ca955 A	252
	253	/* extension data */
	254	struct UConverterSharedData *baseSharedData;
	255	const int32_t *extIndexes;
b75a7d8f A	256	} UConverterMBCSTable;
	257
	258	/**
374ca955	259	* MBCS data header. See data format description above.
b75a7d8f A	260	*/
	261	typedef struct {
	262	UVersionInfo version;
	263	uint32_t countStates,
	264	countToUFallbacks,
	265	offsetToUCodeUnits,
	266	offsetFromUTable,
	267	offsetFromUBytes,
	268	flags,
	269	fromUBytesLength;
	270	} _MBCSHeader;
	271
374ca955	272	/*
b75a7d8f A	273	* This is a simple version of _MBCSGetNextUChar() that is used
b75a7d8f A	274	* by other converter implementations.
374ca955	275	* It only returns an "assigned" result if it consumes the entire input.
b75a7d8f A	276	* It does not use state from the converter, nor error codes.
b75a7d8f A	277	* It does not handle the EBCDIC swaplfnl option (set in UConverter).
374ca955	278	* It handles conversion extensions but not GB 18030.
b75a7d8f A	279	*
	280	* Return value:
	281	* U+fffe unassigned
	282	* U+ffff illegal
	283	* otherwise the Unicode code point
	284	*/
	285	U_CFUNC UChar32
374ca955 A	286	ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
374ca955 A	287	const char *source, int32_t length,
b75a7d8f A	288	UBool useFallback);
	289
	290	/**
	291	* This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
	292	* It does not handle the EBCDIC swaplfnl option (set in UConverter).
374ca955	293	* It does not handle conversion extensions (_extToU()).
b75a7d8f A	294	*/
b75a7d8f A	295	U_CFUNC UChar32
374ca955	296	ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
b75a7d8f A	297	uint8_t b, UBool useFallback);
	298
	299	/**
	300	* This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
	301	* It works for single-byte, single-state codepages that only map
	302	* to and from BMP code points, and it always
	303	* returns fallback values.
	304	*/
	305	#define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
374ca955	306	(UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])
b75a7d8f A	307
	308	/**
	309	* This is an internal function that allows other converter implementations
	310	* to check whether a byte is a lead byte.
	311	*/
	312	U_CFUNC UBool
374ca955	313	ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);
b75a7d8f A	314
	315	/** This is a macro version of _MBCSIsLeadByte(). */
	316	#define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
374ca955	317	(UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])
b75a7d8f	318
374ca955	319	/*
b75a7d8f A	320	* This is another simple conversion function for internal use by other
	321	* conversion implementations.
	322	* It does not use the converter state nor call callbacks.
	323	* It does not handle the EBCDIC swaplfnl option (set in UConverter).
374ca955	324	* It handles conversion extensions but not GB 18030.
b75a7d8f A	325	*
	326	* It converts one single Unicode code point into codepage bytes, encoded
	327	* as one 32-bit value. The function returns the number of bytes in *pValue:
	328	* 1..4 the number of bytes in *pValue
	329	* 0 unassigned (*pValue undefined)
	330	* -1 illegal (currently not used, *pValue undefined)
	331	*
	332	* *pValue will contain the resulting bytes with the last byte in bits 7..0,
	333	* the second to last byte in bits 15..8, etc.
	334	* Currently, the function assumes but does not check that 0<=c<=0x10ffff.
	335	*/
	336	U_CFUNC int32_t
374ca955	337	ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
b75a7d8f A	338	UChar32 c, uint32_t *pValue,
	339	UBool useFallback);
	340
	341	/**
	342	* This version of _MBCSFromUChar32() is optimized for single-byte codepages.
	343	* It does not handle the EBCDIC swaplfnl option (set in UConverter).
	344	*
	345	* It returns the codepage byte for the code point, or -1 if it is unassigned.
	346	*/
	347	U_CFUNC int32_t
374ca955	348	ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
b75a7d8f A	349	UChar32 c,
	350	UBool useFallback);
	351
	352	/**
	353	* SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
	354	* we cheat a little about the type, returning the old types if appropriate.
	355	*/
	356	U_CFUNC UConverterType
374ca955	357	ucnv_MBCSGetType(const UConverter* converter);
b75a7d8f A	358
b75a7d8f A	359	U_CFUNC void
374ca955	360	ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
b75a7d8f A	361	UErrorCode *pErrorCode);
b75a7d8f A	362	U_CFUNC void
374ca955	363	ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
b75a7d8f A	364	UErrorCode *pErrorCode);
b75a7d8f A	365
374ca955 A	366	/*
	367	* Internal function returning a UnicodeSet for toUnicode() conversion.
	368	* Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
	369	* In the future, if we add support for reverse-fallback sets, this function
	370	* needs to be updated, and called for each initial state.
	371	* Does not currently handle extensions.
	372	* Does not empty the set first.
	373	*/
	374	U_CFUNC void
	375	ucnv_MBCSGetUnicodeSetForBytes(const UConverterSharedData *sharedData,
73c04bcf	376	const USetAdder *sa,
374ca955 A	377	UConverterUnicodeSet which,
	378	uint8_t state, int32_t lowByte, int32_t highByte,
	379	UErrorCode *pErrorCode);
	380
	381	/*
	382	* Internal function returning a UnicodeSet for toUnicode() conversion.
	383	* Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
	384	* In the future, if we add support for fallback sets, this function
	385	* needs to be updated.
	386	* Handles extensions.
	387	* Does not empty the set first.
	388	*/
	389	U_CFUNC void
	390	ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
73c04bcf	391	const USetAdder *sa,
374ca955 A	392	UConverterUnicodeSet which,
	393	UErrorCode *pErrorCode);
	394
	395	#endif
b75a7d8f A	396
b75a7d8f A	397	#endif