]> git.saurik.com Git - apple/icu.git/blame - icuSources/tools/toolutil/ucm.h
ICU-8.11.1.tar.gz
[apple/icu.git] / icuSources / tools / toolutil / ucm.h
CommitLineData
374ca955
A
1/*
2*******************************************************************************
3*
73c04bcf 4* Copyright (C) 2003-2005, International Business Machines
374ca955
A
5* Corporation and others. All Rights Reserved.
6*
7*******************************************************************************
8* file name: ucm.h
9* encoding: US-ASCII
10* tab size: 8 (not used)
11* indentation:4
12*
13* created on: 2003jun20
14* created by: Markus W. Scherer
15*
16* Definitions for the .ucm file parser and handler module ucm.c.
17*/
18
19#ifndef __UCM_H__
20#define __UCM_H__
21
22#include "unicode/utypes.h"
23#include "ucnvmbcs.h"
24#include "ucnv_ext.h"
25#include "filestrm.h"
26#include <stdio.h>
27
73c04bcf
A
28#if !UCONFIG_NO_CONVERSION
29
374ca955
A
30U_CDECL_BEGIN
31
32/*
33 * Per-mapping data structure
34 *
35 * u if uLen==1: Unicode code point
36 * else index to uLen code points
37 * b if bLen<=4: up to 4 bytes
38 * else index to bLen bytes
39 * uLen number of code points
40 * bLen number of words containing left-justified bytes
41 * bIsMultipleChars indicates that the bytes contain more than one sequence
42 * according to the state table
43 * f flag for roundtrip (0), fallback (1), sub mapping (2), reverse fallback (3)
44 * same values as in the source file after |
45 */
46typedef struct UCMapping {
47 UChar32 u;
48 union {
49 uint32_t index;
50 uint8_t bytes[4];
51 } b;
52 int8_t uLen, bLen, f, moveFlag;
53} UCMapping;
54
55enum {
56 UCM_FLAGS_INITIAL, /* no mappings parsed yet */
57 UCM_FLAGS_EXPLICIT, /* .ucm file has mappings with | fallback indicators */
58 UCM_FLAGS_IMPLICIT, /* .ucm file has mappings without | fallback indicators, later wins */
59 UCM_FLAGS_MIXED /* both implicit and explicit */
60};
61
62typedef struct UCMTable {
63 UCMapping *mappings;
64 int32_t mappingsCapacity, mappingsLength;
65
66 UChar32 *codePoints;
67 int32_t codePointsCapacity, codePointsLength;
68
69 uint8_t *bytes;
70 int32_t bytesCapacity, bytesLength;
71
72 /* index map for mapping by bytes first */
73 int32_t *reverseMap;
74
75 uint8_t unicodeMask;
76 int8_t flagsType; /* UCM_FLAGS_INITIAL etc. */
77 UBool isSorted;
78} UCMTable;
79
80enum {
81 MBCS_STATE_FLAG_DIRECT=1,
82 MBCS_STATE_FLAG_SURROGATES,
83
84 MBCS_STATE_FLAG_READY=16
85};
86
87typedef struct UCMStates {
88 int32_t stateTable[MBCS_MAX_STATE_COUNT][256];
89 uint32_t stateFlags[MBCS_MAX_STATE_COUNT],
90 stateOffsetSum[MBCS_MAX_STATE_COUNT];
91
92 int32_t countStates, minCharLength, maxCharLength, countToUCodeUnits;
93 int8_t conversionType, outputType;
94} UCMStates;
95
96typedef struct UCMFile {
97 UCMTable *base, *ext;
98 UCMStates states;
99
100 char baseName[UCNV_MAX_CONVERTER_NAME_LENGTH];
101} UCMFile;
102
103/* simple accesses ---------------------------------------------------------- */
104
105#define UCM_GET_CODE_POINTS(t, m) \
106 (((m)->uLen==1) ? &(m)->u : (t)->codePoints+(m)->u)
107
108#define UCM_GET_BYTES(t, m) \
109 (((m)->bLen<=4) ? (m)->b.bytes : (t)->bytes+(m)->b.index)
110
111/* APIs --------------------------------------------------------------------- */
112
113U_CAPI UCMFile * U_EXPORT2
114ucm_open(void);
115
116U_CAPI void U_EXPORT2
117ucm_close(UCMFile *ucm);
118
119U_CAPI UBool U_EXPORT2
120ucm_parseHeaderLine(UCMFile *ucm,
121 char *line, char **pKey, char **pValue);
122
123/* @return -1 illegal bytes 0 suitable for base table 1 needs to go into extension table */
124U_CAPI int32_t U_EXPORT2
125ucm_mappingType(UCMStates *baseStates,
126 UCMapping *m,
127 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
128 uint8_t bytes[UCNV_EXT_MAX_BYTES]);
129
130/* add a mapping to the base or extension table as appropriate */
131U_CAPI UBool U_EXPORT2
132ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates,
133 UCMapping *m,
134 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
135 uint8_t bytes[UCNV_EXT_MAX_BYTES]);
136
137U_CAPI UBool U_EXPORT2
138ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates);
139
140
141U_CAPI UCMTable * U_EXPORT2
142ucm_openTable(void);
143
144U_CAPI void U_EXPORT2
145ucm_closeTable(UCMTable *table);
146
147U_CAPI void U_EXPORT2
148ucm_resetTable(UCMTable *table);
149
150U_CAPI void U_EXPORT2
151ucm_sortTable(UCMTable *t);
152
153/**
154 * Read a table from a .ucm file, from after the CHARMAP line to
155 * including the END CHARMAP line.
156 */
157U_CAPI void U_EXPORT2
158ucm_readTable(UCMFile *ucm, FileStream* convFile,
159 UBool forBase, UCMStates *baseStates,
160 UErrorCode *pErrorCode);
161
162/**
163 * Check the validity of mappings against a base table's states;
164 * necessary for extension-only tables that were read before their base tables.
165 */
166U_CAPI UBool U_EXPORT2
167ucm_checkValidity(UCMTable *ext, UCMStates *baseStates);
168
169/**
170 * Check a base table against an extension table.
171 * Set the moveTarget!=NULL if it is possible to move mappings from the base.
172 * This is the case where base and extension tables are parsed from a single file
173 * (moveTarget==ext)
174 * or when delta file mappings are subtracted from a base table.
175 *
176 * When a base table cannot be modified because a delta file is parsed in makeconv,
177 * then set moveTarget=NULL.
178 *
179 * if(intersectBase) then mappings that exist in the base table but not in
180 * the extension table are moved to moveTarget instead of showing an error.
181 *
182 * Special mode:
183 * If intersectBase==2 for a DBCS extension table, then SBCS mappings are
184 * not moved out of the base unless their Unicode input requires it.
185 * This helps ucmkbase generate base tables for DBCS-only extension .cnv files.
186 *
187 * For both tables in the same file, the extension table is automatically
188 * built.
189 * For separate files, the extension file can use a complete mapping table,
190 * so that common mappings need not be stripped out manually.
191 *
192 *
193 * Sort both tables, and then for each mapping direction:
194 *
195 * If intersectBase is TRUE and the base table contains a mapping
196 * that does not exist in the extension table, then this mapping is moved
197 * to moveTarget.
198 *
199 * - otherwise -
200 *
201 * If the base table contains a mapping for which the input sequence is
202 * the same as the extension input, then
203 * - if the output is the same: remove the extension mapping
204 * - else: error
205 *
206 * If the base table contains a mapping for which the input sequence is
207 * a prefix of the extension input, then
208 * - if moveTarget!=NULL: move the base mapping to the moveTarget table
209 * - else: error
210 *
211 * @return FALSE in case of an irreparable error
212 */
213U_CAPI UBool U_EXPORT2
214ucm_checkBaseExt(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
215 UCMTable *moveTarget, UBool intersectBase);
216
217U_CAPI void U_EXPORT2
218ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode);
219
220U_CAPI void U_EXPORT2
221ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f);
222
223
224U_CAPI void U_EXPORT2
225ucm_addState(UCMStates *states, const char *s);
226
227U_CAPI void U_EXPORT2
228ucm_processStates(UCMStates *states);
229
230U_CAPI int32_t U_EXPORT2
231ucm_countChars(UCMStates *states,
232 const uint8_t *bytes, int32_t length);
233
234
235U_CAPI int8_t U_EXPORT2
236ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps);
237
238U_CAPI UBool U_EXPORT2
239ucm_parseMappingLine(UCMapping *m,
240 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
241 uint8_t bytes[UCNV_EXT_MAX_BYTES],
242 const char *line);
243
244U_CAPI void U_EXPORT2
245ucm_addMapping(UCMTable *table,
246 UCMapping *m,
247 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
248 uint8_t bytes[UCNV_EXT_MAX_BYTES]);
249
250/* very makeconv-specific functions ----------------------------------------- */
251
252/* finalize and optimize states after the toUnicode mappings are processed */
253U_CAPI void U_EXPORT2
254ucm_optimizeStates(UCMStates *states,
255 uint16_t **pUnicodeCodeUnits,
256 _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
257 UBool verbose);
258
259/* moved here because it is used inside ucmstate.c */
260U_CAPI int32_t U_EXPORT2
261ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
262 uint32_t offset);
263
264/* very rptp2ucm-specific functions ----------------------------------------- */
265
266/*
267 * Input: Separate tables with mappings from/to Unicode,
268 * subchar and subchar1 (0 if none).
269 * All mappings must have flag 0.
270 *
271 * Output: fromUTable will contain the union of mappings with the correct
272 * precision flags, and be sorted.
273 */
274U_CAPI void U_EXPORT2
275ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable,
276 const uint8_t *subchar, int32_t subcharLength,
277 uint8_t subchar1);
278
279U_CAPI UBool U_EXPORT2
280ucm_separateMappings(UCMFile *ucm, UBool isSISO);
281
282U_CDECL_END
283
284#endif
73c04bcf
A
285
286#endif
287