]>
Commit | Line | Data |
---|---|---|
374ca955 A |
1 | /* |
2 | ******************************************************************************* | |
3 | * | |
73c04bcf | 4 | * Copyright (C) 2003-2005, International Business Machines |
374ca955 A |
5 | * Corporation and others. All Rights Reserved. |
6 | * | |
7 | ******************************************************************************* | |
8 | * file name: ucm.h | |
9 | * encoding: US-ASCII | |
10 | * tab size: 8 (not used) | |
11 | * indentation:4 | |
12 | * | |
13 | * created on: 2003jun20 | |
14 | * created by: Markus W. Scherer | |
15 | * | |
16 | * Definitions for the .ucm file parser and handler module ucm.c. | |
17 | */ | |
18 | ||
19 | #ifndef __UCM_H__ | |
20 | #define __UCM_H__ | |
21 | ||
22 | #include "unicode/utypes.h" | |
23 | #include "ucnvmbcs.h" | |
24 | #include "ucnv_ext.h" | |
25 | #include "filestrm.h" | |
26 | #include <stdio.h> | |
27 | ||
73c04bcf A |
28 | #if !UCONFIG_NO_CONVERSION |
29 | ||
374ca955 A |
30 | U_CDECL_BEGIN |
31 | ||
32 | /* | |
33 | * Per-mapping data structure | |
34 | * | |
35 | * u if uLen==1: Unicode code point | |
36 | * else index to uLen code points | |
37 | * b if bLen<=4: up to 4 bytes | |
38 | * else index to bLen bytes | |
39 | * uLen number of code points | |
40 | * bLen number of words containing left-justified bytes | |
41 | * bIsMultipleChars indicates that the bytes contain more than one sequence | |
42 | * according to the state table | |
43 | * f flag for roundtrip (0), fallback (1), sub mapping (2), reverse fallback (3) | |
44 | * same values as in the source file after | | |
45 | */ | |
46 | typedef struct UCMapping { | |
47 | UChar32 u; | |
48 | union { | |
49 | uint32_t index; | |
50 | uint8_t bytes[4]; | |
51 | } b; | |
52 | int8_t uLen, bLen, f, moveFlag; | |
53 | } UCMapping; | |
54 | ||
55 | enum { | |
56 | UCM_FLAGS_INITIAL, /* no mappings parsed yet */ | |
57 | UCM_FLAGS_EXPLICIT, /* .ucm file has mappings with | fallback indicators */ | |
58 | UCM_FLAGS_IMPLICIT, /* .ucm file has mappings without | fallback indicators, later wins */ | |
59 | UCM_FLAGS_MIXED /* both implicit and explicit */ | |
60 | }; | |
61 | ||
62 | typedef struct UCMTable { | |
63 | UCMapping *mappings; | |
64 | int32_t mappingsCapacity, mappingsLength; | |
65 | ||
66 | UChar32 *codePoints; | |
67 | int32_t codePointsCapacity, codePointsLength; | |
68 | ||
69 | uint8_t *bytes; | |
70 | int32_t bytesCapacity, bytesLength; | |
71 | ||
72 | /* index map for mapping by bytes first */ | |
73 | int32_t *reverseMap; | |
74 | ||
75 | uint8_t unicodeMask; | |
76 | int8_t flagsType; /* UCM_FLAGS_INITIAL etc. */ | |
77 | UBool isSorted; | |
78 | } UCMTable; | |
79 | ||
80 | enum { | |
81 | MBCS_STATE_FLAG_DIRECT=1, | |
82 | MBCS_STATE_FLAG_SURROGATES, | |
83 | ||
84 | MBCS_STATE_FLAG_READY=16 | |
85 | }; | |
86 | ||
87 | typedef struct UCMStates { | |
88 | int32_t stateTable[MBCS_MAX_STATE_COUNT][256]; | |
89 | uint32_t stateFlags[MBCS_MAX_STATE_COUNT], | |
90 | stateOffsetSum[MBCS_MAX_STATE_COUNT]; | |
91 | ||
92 | int32_t countStates, minCharLength, maxCharLength, countToUCodeUnits; | |
93 | int8_t conversionType, outputType; | |
94 | } UCMStates; | |
95 | ||
96 | typedef struct UCMFile { | |
97 | UCMTable *base, *ext; | |
98 | UCMStates states; | |
99 | ||
100 | char baseName[UCNV_MAX_CONVERTER_NAME_LENGTH]; | |
101 | } UCMFile; | |
102 | ||
103 | /* simple accesses ---------------------------------------------------------- */ | |
104 | ||
105 | #define UCM_GET_CODE_POINTS(t, m) \ | |
106 | (((m)->uLen==1) ? &(m)->u : (t)->codePoints+(m)->u) | |
107 | ||
108 | #define UCM_GET_BYTES(t, m) \ | |
109 | (((m)->bLen<=4) ? (m)->b.bytes : (t)->bytes+(m)->b.index) | |
110 | ||
111 | /* APIs --------------------------------------------------------------------- */ | |
112 | ||
113 | U_CAPI UCMFile * U_EXPORT2 | |
114 | ucm_open(void); | |
115 | ||
116 | U_CAPI void U_EXPORT2 | |
117 | ucm_close(UCMFile *ucm); | |
118 | ||
119 | U_CAPI UBool U_EXPORT2 | |
120 | ucm_parseHeaderLine(UCMFile *ucm, | |
121 | char *line, char **pKey, char **pValue); | |
122 | ||
123 | /* @return -1 illegal bytes 0 suitable for base table 1 needs to go into extension table */ | |
124 | U_CAPI int32_t U_EXPORT2 | |
125 | ucm_mappingType(UCMStates *baseStates, | |
126 | UCMapping *m, | |
127 | UChar32 codePoints[UCNV_EXT_MAX_UCHARS], | |
128 | uint8_t bytes[UCNV_EXT_MAX_BYTES]); | |
129 | ||
130 | /* add a mapping to the base or extension table as appropriate */ | |
131 | U_CAPI UBool U_EXPORT2 | |
132 | ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates, | |
133 | UCMapping *m, | |
134 | UChar32 codePoints[UCNV_EXT_MAX_UCHARS], | |
135 | uint8_t bytes[UCNV_EXT_MAX_BYTES]); | |
136 | ||
137 | U_CAPI UBool U_EXPORT2 | |
138 | ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates); | |
139 | ||
140 | ||
141 | U_CAPI UCMTable * U_EXPORT2 | |
142 | ucm_openTable(void); | |
143 | ||
144 | U_CAPI void U_EXPORT2 | |
145 | ucm_closeTable(UCMTable *table); | |
146 | ||
147 | U_CAPI void U_EXPORT2 | |
148 | ucm_resetTable(UCMTable *table); | |
149 | ||
150 | U_CAPI void U_EXPORT2 | |
151 | ucm_sortTable(UCMTable *t); | |
152 | ||
153 | /** | |
154 | * Read a table from a .ucm file, from after the CHARMAP line to | |
155 | * including the END CHARMAP line. | |
156 | */ | |
157 | U_CAPI void U_EXPORT2 | |
158 | ucm_readTable(UCMFile *ucm, FileStream* convFile, | |
159 | UBool forBase, UCMStates *baseStates, | |
160 | UErrorCode *pErrorCode); | |
161 | ||
162 | /** | |
163 | * Check the validity of mappings against a base table's states; | |
164 | * necessary for extension-only tables that were read before their base tables. | |
165 | */ | |
166 | U_CAPI UBool U_EXPORT2 | |
167 | ucm_checkValidity(UCMTable *ext, UCMStates *baseStates); | |
168 | ||
169 | /** | |
170 | * Check a base table against an extension table. | |
171 | * Set the moveTarget!=NULL if it is possible to move mappings from the base. | |
172 | * This is the case where base and extension tables are parsed from a single file | |
173 | * (moveTarget==ext) | |
174 | * or when delta file mappings are subtracted from a base table. | |
175 | * | |
176 | * When a base table cannot be modified because a delta file is parsed in makeconv, | |
177 | * then set moveTarget=NULL. | |
178 | * | |
179 | * if(intersectBase) then mappings that exist in the base table but not in | |
180 | * the extension table are moved to moveTarget instead of showing an error. | |
181 | * | |
182 | * Special mode: | |
183 | * If intersectBase==2 for a DBCS extension table, then SBCS mappings are | |
184 | * not moved out of the base unless their Unicode input requires it. | |
185 | * This helps ucmkbase generate base tables for DBCS-only extension .cnv files. | |
186 | * | |
187 | * For both tables in the same file, the extension table is automatically | |
188 | * built. | |
189 | * For separate files, the extension file can use a complete mapping table, | |
190 | * so that common mappings need not be stripped out manually. | |
191 | * | |
192 | * | |
193 | * Sort both tables, and then for each mapping direction: | |
194 | * | |
195 | * If intersectBase is TRUE and the base table contains a mapping | |
196 | * that does not exist in the extension table, then this mapping is moved | |
197 | * to moveTarget. | |
198 | * | |
199 | * - otherwise - | |
200 | * | |
201 | * If the base table contains a mapping for which the input sequence is | |
202 | * the same as the extension input, then | |
203 | * - if the output is the same: remove the extension mapping | |
204 | * - else: error | |
205 | * | |
206 | * If the base table contains a mapping for which the input sequence is | |
207 | * a prefix of the extension input, then | |
208 | * - if moveTarget!=NULL: move the base mapping to the moveTarget table | |
209 | * - else: error | |
210 | * | |
211 | * @return FALSE in case of an irreparable error | |
212 | */ | |
213 | U_CAPI UBool U_EXPORT2 | |
214 | ucm_checkBaseExt(UCMStates *baseStates, UCMTable *base, UCMTable *ext, | |
215 | UCMTable *moveTarget, UBool intersectBase); | |
216 | ||
217 | U_CAPI void U_EXPORT2 | |
218 | ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode); | |
219 | ||
220 | U_CAPI void U_EXPORT2 | |
221 | ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f); | |
222 | ||
223 | ||
224 | U_CAPI void U_EXPORT2 | |
225 | ucm_addState(UCMStates *states, const char *s); | |
226 | ||
227 | U_CAPI void U_EXPORT2 | |
228 | ucm_processStates(UCMStates *states); | |
229 | ||
230 | U_CAPI int32_t U_EXPORT2 | |
231 | ucm_countChars(UCMStates *states, | |
232 | const uint8_t *bytes, int32_t length); | |
233 | ||
234 | ||
235 | U_CAPI int8_t U_EXPORT2 | |
236 | ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps); | |
237 | ||
238 | U_CAPI UBool U_EXPORT2 | |
239 | ucm_parseMappingLine(UCMapping *m, | |
240 | UChar32 codePoints[UCNV_EXT_MAX_UCHARS], | |
241 | uint8_t bytes[UCNV_EXT_MAX_BYTES], | |
242 | const char *line); | |
243 | ||
244 | U_CAPI void U_EXPORT2 | |
245 | ucm_addMapping(UCMTable *table, | |
246 | UCMapping *m, | |
247 | UChar32 codePoints[UCNV_EXT_MAX_UCHARS], | |
248 | uint8_t bytes[UCNV_EXT_MAX_BYTES]); | |
249 | ||
250 | /* very makeconv-specific functions ----------------------------------------- */ | |
251 | ||
252 | /* finalize and optimize states after the toUnicode mappings are processed */ | |
253 | U_CAPI void U_EXPORT2 | |
254 | ucm_optimizeStates(UCMStates *states, | |
255 | uint16_t **pUnicodeCodeUnits, | |
256 | _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, | |
257 | UBool verbose); | |
258 | ||
259 | /* moved here because it is used inside ucmstate.c */ | |
260 | U_CAPI int32_t U_EXPORT2 | |
261 | ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, | |
262 | uint32_t offset); | |
263 | ||
264 | /* very rptp2ucm-specific functions ----------------------------------------- */ | |
265 | ||
266 | /* | |
267 | * Input: Separate tables with mappings from/to Unicode, | |
268 | * subchar and subchar1 (0 if none). | |
269 | * All mappings must have flag 0. | |
270 | * | |
271 | * Output: fromUTable will contain the union of mappings with the correct | |
272 | * precision flags, and be sorted. | |
273 | */ | |
274 | U_CAPI void U_EXPORT2 | |
275 | ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable, | |
276 | const uint8_t *subchar, int32_t subcharLength, | |
277 | uint8_t subchar1); | |
278 | ||
279 | U_CAPI UBool U_EXPORT2 | |
280 | ucm_separateMappings(UCMFile *ucm, UBool isSISO); | |
281 | ||
282 | U_CDECL_END | |
283 | ||
284 | #endif | |
73c04bcf A |
285 | |
286 | #endif | |
287 |