]> git.saurik.com Git - apple/icu.git/blob - icuSources/common/ucnv2022.c
projects / apple / icu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ICU-461.13.tar.gz
[apple/icu.git] / icuSources / common / ucnv2022.c
1 /*
2 **********************************************************************
3 * Copyright (C) 2000-2010, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 * file name: ucnv2022.c
7 * encoding: US-ASCII
8 * tab size: 8 (not used)
9 * indentation:4
10 *
11 * created on: 2000feb03
12 * created by: Markus W. Scherer
13 *
14 * Change history:
15 *
16 * 06/29/2000 helena Major rewrite of the callback APIs.
17 * 08/08/2000 Ram Included support for ISO-2022-JP-2
18 * Changed implementation of toUnicode
19 * function
20 * 08/21/2000 Ram Added support for ISO-2022-KR
21 * 08/29/2000 Ram Seperated implementation of EBCDIC to
22 * ucnvebdc.c
23 * 09/20/2000 Ram Added support for ISO-2022-CN
24 * Added implementations for getNextUChar()
25 * for specific 2022 country variants.
26 * 10/31/2000 Ram Implemented offsets logic functions
27 */
28
29 #include "unicode/utypes.h"
30
31 #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
32
33 #include "unicode/ucnv.h"
34 #include "unicode/uset.h"
35 #include "unicode/ucnv_err.h"
36 #include "unicode/ucnv_cb.h"
37 #include "ucnv_imp.h"
38 #include "ucnv_bld.h"
39 #include "ucnv_cnv.h"
40 #include "ucnvmbcs.h"
41 #include "cstring.h"
42 #include "cmemory.h"
43
44 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
45
46 #ifdef U_ENABLE_GENERIC_ISO_2022
47 /*
48 * I am disabling the generic ISO-2022 converter after proposing to do so on
49 * the icu mailing list two days ago.
50 *
51 * Reasons:
52 * 1. It does not fully support the ISO-2022/ECMA-35 specification with all of
53 * its designation sequences, single shifts with return to the previous state,
54 * switch-with-no-return to UTF-16BE or similar, etc.
55 * This is unlike the language-specific variants like ISO-2022-JP which
56 * require a much smaller repertoire of ISO-2022 features.
57 * These variants continue to be supported.
58 * 2. I believe that no one is really using the generic ISO-2022 converter
59 * but rather always one of the language-specific variants.
60 * Note that ICU's generic ISO-2022 converter has always output one escape
61 * sequence followed by UTF-8 for the whole stream.
62 * 3. Switching between subcharsets is extremely slow, because each time
63 * the previous converter is closed and a new one opened,
64 * without any kind of caching, least-recently-used list, etc.
65 * 4. The code is currently buggy, and given the above it does not seem
66 * reasonable to spend the time on maintenance.
67 * 5. ISO-2022 subcharsets should normally be used with 7-bit byte encodings.
68 * This means, for example, that when ISO-8859-7 is designated, the following
69 * ISO-2022 bytes 00..7f should be interpreted as ISO-8859-7 bytes 80..ff.
70 * The ICU ISO-2022 converter does not handle this - and has no information
71 * about which subconverter would have to be shifted vs. which is designed
72 * for 7-bit ISO-2022.
73 *
74 * Markus Scherer 2003-dec-03
75 */
76 #endif
77
78 static const char SHIFT_IN_STR[] = "\x0F";
79 static const char SHIFT_OUT_STR[] = "\x0E";
80
81 #define CR 0x0D
82 #define LF 0x0A
83 #define H_TAB 0x09
84 #define V_TAB 0x0B
85 #define SPACE 0x20
86
87 enum {
88 HWKANA_START=0xff61,
89 HWKANA_END=0xff9f
90 };
91
92 /*
93 * 94-character sets with native byte values A1..FE are encoded in ISO 2022
94 * as bytes 21..7E. (Subtract 0x80.)
95 * 96-character sets with native byte values A0..FF are encoded in ISO 2022
96 * as bytes 20..7F. (Subtract 0x80.)
97 * Do not encode C1 control codes with native bytes 80..9F
98 * as bytes 00..1F (C0 control codes).
99 */
100 enum {
101 GR94_START=0xa1,
102 GR94_END=0xfe,
103 GR96_START=0xa0,
104 GR96_END=0xff
105 };
106
107 /*
108 * ISO 2022 control codes must not be converted from Unicode
109 * because they would mess up the byte stream.
110 * The bit mask 0x0800c000 has bits set at bit positions 0xe, 0xf, 0x1b
111 * corresponding to SO, SI, and ESC.
112 */
113 #define IS_2022_CONTROL(c) (((c)<0x20) && (((uint32_t)1<<(c))&0x0800c000)!=0)
114
115 /* for ISO-2022-JP and -CN implementations */
116 typedef enum {
117 /* shared values */
118 INVALID_STATE=-1,
119 ASCII = 0,
120
121 SS2_STATE=0x10,
122 SS3_STATE,
123
124 /* JP */
125 ISO8859_1 = 1 ,
126 ISO8859_7 = 2 ,
127 JISX201 = 3,
128 JISX208 = 4,
129 JISX212 = 5,
130 GB2312 =6,
131 KSC5601 =7,
132 HWKANA_7BIT=8, /* Halfwidth Katakana 7 bit */
133
134 /* CN */
135 /* the first few enum constants must keep their values because they correspond to myConverterArray[] */
136 GB2312_1=1,
137 ISO_IR_165=2,
138 CNS_11643=3,
139
140 /*
141 * these are used in StateEnum and ISO2022State variables,
142 * but CNS_11643 must be used to index into myConverterArray[]
143 */
144 CNS_11643_0=0x20,
145 CNS_11643_1,
146 CNS_11643_2,
147 CNS_11643_3,
148 CNS_11643_4,
149 CNS_11643_5,
150 CNS_11643_6,
151 CNS_11643_7
152 } StateEnum;
153
154 /* is the StateEnum charset value for a DBCS charset? */
155 #define IS_JP_DBCS(cs) (JISX208<=(cs) && (cs)<=KSC5601)
156
157 #define CSM(cs) ((uint16_t)1<<(cs))
158
159 /*
160 * Each of these charset masks (with index x) contains a bit for a charset in exact correspondence
161 * to whether that charset is used in the corresponding version x of ISO_2022,locale=ja,version=x
162 *
163 * Note: The converter uses some leniency:
164 * - The escape sequence ESC ( I for half-width 7-bit Katakana is recognized in
165 * all versions, not just JIS7 and JIS8.
166 * - ICU does not distinguish between different versions of JIS X 0208.
167 */
168 enum { MAX_JA_VERSION=4 };
169 static const uint16_t jpCharsetMasks[MAX_JA_VERSION+1]={
170 CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT),
171 CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212),
172 CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212)|CSM(GB2312)|CSM(KSC5601)|CSM(ISO8859_1)|CSM(ISO8859_7),
173 CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212)|CSM(GB2312)|CSM(KSC5601)|CSM(ISO8859_1)|CSM(ISO8859_7),
174 CSM(ASCII)|CSM(JISX201)|CSM(JISX208)|CSM(HWKANA_7BIT)|CSM(JISX212)|CSM(GB2312)|CSM(KSC5601)|CSM(ISO8859_1)|CSM(ISO8859_7)
175 };
176
177 typedef enum {
178 ASCII1=0,
179 LATIN1,
180 SBCS,
181 DBCS,
182 MBCS,
183 HWKANA
184 }Cnv2022Type;
185
186 typedef struct ISO2022State {
187 int8_t cs[4]; /* charset number for SI (G0)/SO (G1)/SS2 (G2)/SS3 (G3) */
188 int8_t g; /* 0..3 for G0..G3 (SI/SO/SS2/SS3) */
189 int8_t prevG; /* g before single shift (SS2 or SS3) */
190 } ISO2022State;
191
192 #define UCNV_OPTIONS_VERSION_MASK 0xf
193 #define UCNV_2022_MAX_CONVERTERS 10
194
195 typedef struct{
196 UConverterSharedData *myConverterArray[UCNV_2022_MAX_CONVERTERS];
197 UConverter *currentConverter;
198 Cnv2022Type currentType;
199 ISO2022State toU2022State, fromU2022State;
200 uint32_t key;
201 uint32_t version;
202 #ifdef U_ENABLE_GENERIC_ISO_2022
203 UBool isFirstBuffer;
204 #endif
205 UBool isEmptySegment;
206 char name[30];
207 char locale[3];
208 }UConverterDataISO2022;
209
210 /* Protos */
211 /* ISO-2022 ----------------------------------------------------------------- */
212
213 /*Forward declaration */
214 U_CFUNC void
215 ucnv_fromUnicode_UTF8(UConverterFromUnicodeArgs * args,
216 UErrorCode * err);
217 U_CFUNC void
218 ucnv_fromUnicode_UTF8_OFFSETS_LOGIC(UConverterFromUnicodeArgs * args,
219 UErrorCode * err);
220
221 #define ESC_2022 0x1B /*ESC*/
222
223 typedef enum
224 {
225 INVALID_2022 = -1, /*Doesn't correspond to a valid iso 2022 escape sequence*/
226 VALID_NON_TERMINAL_2022 = 0, /*so far corresponds to a valid iso 2022 escape sequence*/
227 VALID_TERMINAL_2022 = 1, /*corresponds to a valid iso 2022 escape sequence*/
228 VALID_MAYBE_TERMINAL_2022 = 2 /*so far matches one iso 2022 escape sequence, but by adding more characters might match another escape sequence*/
229 } UCNV_TableStates_2022;
230
231 /*
232 * The way these state transition arrays work is:
233 * ex : ESC$B is the sequence for JISX208
234 * a) First Iteration: char is ESC
235 * i) Get the value of ESC from normalize_esq_chars_2022[] with int value of ESC as index
236 * int x = normalize_esq_chars_2022[27] which is equal to 1
237 * ii) Search for this value in escSeqStateTable_Key_2022[]
238 * value of x is stored at escSeqStateTable_Key_2022[0]
239 * iii) Save this index as offset
240 * iv) Get state of this sequence from escSeqStateTable_Value_2022[]
241 * escSeqStateTable_Value_2022[offset], which is VALID_NON_TERMINAL_2022
242 * b) Switch on this state and continue to next char
243 * i) Get the value of $ from normalize_esq_chars_2022[] with int value of $ as index
244 * which is normalize_esq_chars_2022[36] == 4
245 * ii) x is currently 1(from above)
246 * x<<=5 -- x is now 32
247 * x+=normalize_esq_chars_2022[36]
248 * now x is 36
249 * iii) Search for this value in escSeqStateTable_Key_2022[]
250 * value of x is stored at escSeqStateTable_Key_2022[2], so offset is 2
251 * iv) Get state of this sequence from escSeqStateTable_Value_2022[]
252 * escSeqStateTable_Value_2022[offset], which is VALID_NON_TERMINAL_2022
253 * c) Switch on this state and continue to next char
254 * i) Get the value of B from normalize_esq_chars_2022[] with int value of B as index
255 * ii) x is currently 36 (from above)
256 * x<<=5 -- x is now 1152
257 * x+=normalize_esq_chars_2022[66]
258 * now x is 1161
259 * iii) Search for this value in escSeqStateTable_Key_2022[]
260 * value of x is stored at escSeqStateTable_Key_2022[21], so offset is 21
261 * iv) Get state of this sequence from escSeqStateTable_Value_2022[21]
262 * escSeqStateTable_Value_2022[offset], which is VALID_TERMINAL_2022
263 * v) Get the converter name form escSeqStateTable_Result_2022[21] which is JISX208
264 */
265
266
267 /*Below are the 3 arrays depicting a state transition table*/
268 static const int8_t normalize_esq_chars_2022[256] = {
269 /* 0 1 2 3 4 5 6 7 8 9 */
270
271 0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
272 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
273 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,1 ,0 ,0
274 ,0 ,0 ,0 ,0 ,0 ,0 ,4 ,7 ,29 ,0
275 ,2 ,24 ,26 ,27 ,0 ,3 ,23 ,6 ,0 ,0
276 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
277 ,0 ,0 ,0 ,0 ,5 ,8 ,9 ,10 ,11 ,12
278 ,13 ,14 ,15 ,16 ,17 ,18 ,19 ,20 ,25 ,28
279 ,0 ,0 ,21 ,0 ,0 ,0 ,0 ,0 ,0 ,0
280 ,22 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
281 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
282 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
283 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
284 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
285 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
286 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
287 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
288 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
289 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
290 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
291 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
292 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
293 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
294 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
295 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0
296 ,0 ,0 ,0 ,0 ,0 ,0
297 };
298
299 #ifdef U_ENABLE_GENERIC_ISO_2022
300 /*
301 * When the generic ISO-2022 converter is completely removed, not just disabled
302 * per #ifdef, then the following state table and the associated tables that are
303 * dimensioned with MAX_STATES_2022 should be trimmed.
304 *
305 * Especially, VALID_MAYBE_TERMINAL_2022 will not be used any more, and all of
306 * the associated escape sequences starting with ESC ( B should be removed.
307 * This includes the ones with key values 1097 and all of the ones above 1000000.
308 *
309 * For the latter, the tables can simply be truncated.
310 * For the former, since the tables must be kept parallel, it is probably best
311 * to simply duplicate an adjacent table cell, parallel in all tables.
312 *
313 * It may make sense to restructure the tables, especially by using small search
314 * tables for the variants instead of indexing them parallel to the table here.
315 */
316 #endif
317
318 #define MAX_STATES_2022 74
319 static const int32_t escSeqStateTable_Key_2022[MAX_STATES_2022] = {
320 /* 0 1 2 3 4 5 6 7 8 9 */
321
322 1 ,34 ,36 ,39 ,55 ,57 ,60 ,61 ,1093 ,1096
323 ,1097 ,1098 ,1099 ,1100 ,1101 ,1102 ,1103 ,1104 ,1105 ,1106
324 ,1109 ,1154 ,1157 ,1160 ,1161 ,1176 ,1178 ,1179 ,1254 ,1257
325 ,1768 ,1773 ,1957 ,35105 ,36933 ,36936 ,36937 ,36938 ,36939 ,36940
326 ,36942 ,36943 ,36944 ,36945 ,36946 ,36947 ,36948 ,37640 ,37642 ,37644
327 ,37646 ,37711 ,37744 ,37745 ,37746 ,37747 ,37748 ,40133 ,40136 ,40138
328 ,40139 ,40140 ,40141 ,1123363 ,35947624 ,35947625 ,35947626 ,35947627 ,35947629 ,35947630
329 ,35947631 ,35947635 ,35947636 ,35947638
330 };
331
332 #ifdef U_ENABLE_GENERIC_ISO_2022
333
334 static const char* const escSeqStateTable_Result_2022[MAX_STATES_2022] = {
335 /* 0 1 2 3 4 5 6 7 8 9 */
336
337 NULL ,NULL ,NULL ,NULL ,NULL ,NULL ,NULL ,NULL ,"latin1" ,"latin1"
338 ,"latin1" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"ibm-865" ,"JISX0201" ,"JISX0201" ,"latin1"
339 ,"latin1" ,NULL ,"JISX-208" ,"ibm-5478" ,"JISX-208" ,NULL ,NULL ,NULL ,NULL ,"UTF8"
340 ,"ISO-8859-1" ,"ISO-8859-7" ,"JIS-X-208" ,NULL ,"ibm-955" ,"ibm-367" ,"ibm-952" ,"ibm-949" ,"JISX-212" ,"ibm-1383"
341 ,"ibm-952" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-964" ,"ibm-5478" ,"ibm-949" ,"ISO-IR-165"
342 ,"CNS-11643-1992,1" ,"CNS-11643-1992,2" ,"CNS-11643-1992,3" ,"CNS-11643-1992,4" ,"CNS-11643-1992,5" ,"CNS-11643-1992,6" ,"CNS-11643-1992,7" ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian"
343 ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" ,"UTF16_PlatformEndian" ,NULL ,"latin1" ,"ibm-912" ,"ibm-913" ,"ibm-914" ,"ibm-813" ,"ibm-1089"
344 ,"ibm-920" ,"ibm-915" ,"ibm-915" ,"latin1"
345 };
346
347 #endif
348
349 static const int8_t escSeqStateTable_Value_2022[MAX_STATES_2022] = {
350 /* 0 1 2 3 4 5 6 7 8 9 */
351 VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
352 ,VALID_MAYBE_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
353 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022
354 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
355 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
356 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
357 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_NON_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
358 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022 ,VALID_TERMINAL_2022
359 };
360
361
362 /* Type def for refactoring changeState_2022 code*/
363 typedef enum{
364 #ifdef U_ENABLE_GENERIC_ISO_2022
365 ISO_2022=0,
366 #endif
367 ISO_2022_JP=1,
368 ISO_2022_KR=2,
369 ISO_2022_CN=3
370 } Variant2022;
371
372 /*********** ISO 2022 Converter Protos ***********/
373 static void
374 _ISO2022Open(UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *errorCode);
375
376 static void
377 _ISO2022Close(UConverter *converter);
378
379 static void
380 _ISO2022Reset(UConverter *converter, UConverterResetChoice choice);
381
382 static const char*
383 _ISO2022getName(const UConverter* cnv);
384
385 static void
386 _ISO_2022_WriteSub(UConverterFromUnicodeArgs *args, int32_t offsetIndex, UErrorCode *err);
387
388 static UConverter *
389 _ISO_2022_SafeClone(const UConverter *cnv, void *stackBuffer, int32_t *pBufferSize, UErrorCode *status);
390
391 #ifdef U_ENABLE_GENERIC_ISO_2022
392 static void
393 T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC(UConverterToUnicodeArgs* args, UErrorCode* err);
394 #endif
395
396 /*const UConverterSharedData _ISO2022Data;*/
397 static const UConverterSharedData _ISO2022JPData;
398 static const UConverterSharedData _ISO2022KRData;
399 static const UConverterSharedData _ISO2022CNData;
400
401 /*************** Converter implementations ******************/
402
403 /* The purpose of this function is to get around gcc compiler warnings. */
404 static U_INLINE void
405 fromUWriteUInt8(UConverter *cnv,
406 const char *bytes, int32_t length,
407 uint8_t **target, const char *targetLimit,
408 int32_t **offsets,
409 int32_t sourceIndex,
410 UErrorCode *pErrorCode)
411 {
412 char *targetChars = (char *)*target;
413 ucnv_fromUWriteBytes(cnv, bytes, length, &targetChars, targetLimit,
414 offsets, sourceIndex, pErrorCode);
415 *target = (uint8_t*)targetChars;
416
417 }
418
419 static U_INLINE void
420 setInitialStateToUnicodeKR(UConverter* converter, UConverterDataISO2022 *myConverterData){
421 if(myConverterData->version == 1) {
422 UConverter *cnv = myConverterData->currentConverter;
423
424 cnv->toUnicodeStatus=0; /* offset */
425 cnv->mode=0; /* state */
426 cnv->toULength=0; /* byteIndex */
427 }
428 }
429
430 static U_INLINE void
431 setInitialStateFromUnicodeKR(UConverter* converter,UConverterDataISO2022 *myConverterData){
432 /* in ISO-2022-KR the designator sequence appears only once
433 * in a file so we append it only once
434 */
435 if( converter->charErrorBufferLength==0){
436
437 converter->charErrorBufferLength = 4;
438 converter->charErrorBuffer[0] = 0x1b;
439 converter->charErrorBuffer[1] = 0x24;
440 converter->charErrorBuffer[2] = 0x29;
441 converter->charErrorBuffer[3] = 0x43;
442 }
443 if(myConverterData->version == 1) {
444 UConverter *cnv = myConverterData->currentConverter;
445
446 cnv->fromUChar32=0;
447 cnv->fromUnicodeStatus=1; /* prevLength */
448 }
449 }
450
451 static void
452 _ISO2022Open(UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *errorCode){
453
454 char myLocale[6]={' ',' ',' ',' ',' ',' '};
455
456 cnv->extraInfo = uprv_malloc (sizeof (UConverterDataISO2022));
457 if(cnv->extraInfo != NULL) {
458 UConverterNamePieces stackPieces;
459 UConverterLoadArgs stackArgs={ (int32_t)sizeof(UConverterLoadArgs) };
460 UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) cnv->extraInfo;
461 uint32_t version;
462
463 stackArgs.onlyTestIsLoadable = pArgs->onlyTestIsLoadable;
464
465 uprv_memset(myConverterData, 0, sizeof(UConverterDataISO2022));
466 myConverterData->currentType = ASCII1;
467 cnv->fromUnicodeStatus =FALSE;
468 if(pArgs->locale){
469 uprv_strncpy(myLocale, pArgs->locale, sizeof(myLocale));
470 }
471 version = pArgs->options & UCNV_OPTIONS_VERSION_MASK;
472 myConverterData->version = version;
473 if(myLocale[0]=='j' && (myLocale[1]=='a'|| myLocale[1]=='p') &&
474 (myLocale[2]=='_' || myLocale[2]=='\0'))
475 {
476 size_t len=0;
477 /* open the required converters and cache them */
478 if(version>MAX_JA_VERSION) {
479 /* prevent indexing beyond jpCharsetMasks[] */
480 myConverterData->version = version = 0;
481 }
482 if(jpCharsetMasks[version]&CSM(ISO8859_7)) {
483 myConverterData->myConverterArray[ISO8859_7] =
484 ucnv_loadSharedData("ISO8859_7", &stackPieces, &stackArgs, errorCode);
485 }
486 myConverterData->myConverterArray[JISX208] =
487 ucnv_loadSharedData("Shift-JIS", &stackPieces, &stackArgs, errorCode);
488 if(jpCharsetMasks[version]&CSM(JISX212)) {
489 myConverterData->myConverterArray[JISX212] =
490 ucnv_loadSharedData("jisx-212", &stackPieces, &stackArgs, errorCode);
491 }
492 if(jpCharsetMasks[version]&CSM(GB2312)) {
493 myConverterData->myConverterArray[GB2312] =
494 ucnv_loadSharedData("ibm-5478", &stackPieces, &stackArgs, errorCode); /* gb_2312_80-1 */
495 }
496 if(jpCharsetMasks[version]&CSM(KSC5601)) {
497 myConverterData->myConverterArray[KSC5601] =
498 ucnv_loadSharedData("ksc_5601", &stackPieces, &stackArgs, errorCode);
499 }
500
501 /* set the function pointers to appropriate funtions */
502 cnv->sharedData=(UConverterSharedData*)(&_ISO2022JPData);
503 uprv_strcpy(myConverterData->locale,"ja");
504
505 (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ja,version=");
506 len = uprv_strlen(myConverterData->name);
507 myConverterData->name[len]=(char)(myConverterData->version+(int)'0');
508 myConverterData->name[len+1]='\0';
509 }
510 else if(myLocale[0]=='k' && (myLocale[1]=='o'|| myLocale[1]=='r') &&
511 (myLocale[2]=='_' || myLocale[2]=='\0'))
512 {
513 const char *cnvName;
514 if(version==1) {
515 cnvName="icu-internal-25546";
516 } else {
517 cnvName="ibm-949";
518 myConverterData->version=version=0;
519 }
520 if(pArgs->onlyTestIsLoadable) {
521 ucnv_canCreateConverter(cnvName, errorCode); /* errorCode carries result */
522 uprv_free(cnv->extraInfo);
523 cnv->extraInfo=NULL;
524 return;
525 } else {
526 myConverterData->currentConverter=ucnv_open(cnvName, errorCode);
527 if (U_FAILURE(*errorCode)) {
528 _ISO2022Close(cnv);
529 return;
530 }
531
532 if(version==1) {
533 (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ko,version=1");
534 uprv_memcpy(cnv->subChars, myConverterData->currentConverter->subChars, 4);
535 cnv->subCharLen = myConverterData->currentConverter->subCharLen;
536 }else{
537 (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=ko,version=0");
538 }
539
540 /* initialize the state variables */
541 setInitialStateToUnicodeKR(cnv, myConverterData);
542 setInitialStateFromUnicodeKR(cnv, myConverterData);
543
544 /* set the function pointers to appropriate funtions */
545 cnv->sharedData=(UConverterSharedData*)&_ISO2022KRData;
546 uprv_strcpy(myConverterData->locale,"ko");
547 }
548 }
549 else if(((myLocale[0]=='z' && myLocale[1]=='h') || (myLocale[0]=='c'&& myLocale[1]=='n'))&&
550 (myLocale[2]=='_' || myLocale[2]=='\0'))
551 {
552
553 /* open the required converters and cache them */
554 myConverterData->myConverterArray[GB2312_1] =
555 ucnv_loadSharedData("ibm-5478", &stackPieces, &stackArgs, errorCode);
556 if(version==1) {
557 myConverterData->myConverterArray[ISO_IR_165] =
558 ucnv_loadSharedData("iso-ir-165", &stackPieces, &stackArgs, errorCode);
559 }
560 myConverterData->myConverterArray[CNS_11643] =
561 ucnv_loadSharedData("cns-11643-1992", &stackPieces, &stackArgs, errorCode);
562
563
564 /* set the function pointers to appropriate funtions */
565 cnv->sharedData=(UConverterSharedData*)&_ISO2022CNData;
566 uprv_strcpy(myConverterData->locale,"cn");
567
568 if (version==0){
569 myConverterData->version = 0;
570 (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=zh,version=0");
571 }else if (version==1){
572 myConverterData->version = 1;
573 (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=zh,version=1");
574 }else {
575 myConverterData->version = 2;
576 (void)uprv_strcpy(myConverterData->name,"ISO_2022,locale=zh,version=2");
577 }
578 }
579 else{
580 #ifdef U_ENABLE_GENERIC_ISO_2022
581 myConverterData->isFirstBuffer = TRUE;
582
583 /* append the UTF-8 escape sequence */
584 cnv->charErrorBufferLength = 3;
585 cnv->charErrorBuffer[0] = 0x1b;
586 cnv->charErrorBuffer[1] = 0x25;
587 cnv->charErrorBuffer[2] = 0x42;
588
589 cnv->sharedData=(UConverterSharedData*)&_ISO2022Data;
590 /* initialize the state variables */
591 uprv_strcpy(myConverterData->name,"ISO_2022");
592 #else
593 *errorCode = U_UNSUPPORTED_ERROR;
594 return;
595 #endif
596 }
597
598 cnv->maxBytesPerUChar=cnv->sharedData->staticData->maxBytesPerChar;
599
600 if(U_FAILURE(*errorCode) || pArgs->onlyTestIsLoadable) {
601 _ISO2022Close(cnv);
602 }
603 } else {
604 *errorCode = U_MEMORY_ALLOCATION_ERROR;
605 }
606 }
607
608
609 static void
610 _ISO2022Close(UConverter *converter) {
611 UConverterDataISO2022* myData =(UConverterDataISO2022 *) (converter->extraInfo);
612 UConverterSharedData **array = myData->myConverterArray;
613 int32_t i;
614
615 if (converter->extraInfo != NULL) {
616 /*close the array of converter pointers and free the memory*/
617 for (i=0; i<UCNV_2022_MAX_CONVERTERS; i++) {
618 if(array[i]!=NULL) {
619 ucnv_unloadSharedDataIfReady(array[i]);
620 }
621 }
622
623 ucnv_close(myData->currentConverter);
624
625 if(!converter->isExtraLocal){
626 uprv_free (converter->extraInfo);
627 converter->extraInfo = NULL;
628 }
629 }
630 }
631
632 static void
633 _ISO2022Reset(UConverter *converter, UConverterResetChoice choice) {
634 UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) (converter->extraInfo);
635 if(choice<=UCNV_RESET_TO_UNICODE) {
636 uprv_memset(&myConverterData->toU2022State, 0, sizeof(ISO2022State));
637 myConverterData->key = 0;
638 myConverterData->isEmptySegment = FALSE;
639 }
640 if(choice!=UCNV_RESET_TO_UNICODE) {
641 uprv_memset(&myConverterData->fromU2022State, 0, sizeof(ISO2022State));
642 }
643 #ifdef U_ENABLE_GENERIC_ISO_2022
644 if(myConverterData->locale[0] == 0){
645 if(choice<=UCNV_RESET_TO_UNICODE) {
646 myConverterData->isFirstBuffer = TRUE;
647 myConverterData->key = 0;
648 if (converter->mode == UCNV_SO){
649 ucnv_close (myConverterData->currentConverter);
650 myConverterData->currentConverter=NULL;
651 }
652 converter->mode = UCNV_SI;
653 }
654 if(choice!=UCNV_RESET_TO_UNICODE) {
655 /* re-append UTF-8 escape sequence */
656 converter->charErrorBufferLength = 3;
657 converter->charErrorBuffer[0] = 0x1b;
658 converter->charErrorBuffer[1] = 0x28;
659 converter->charErrorBuffer[2] = 0x42;
660 }
661 }
662 else
663 #endif
664 {
665 /* reset the state variables */
666 if(myConverterData->locale[0] == 'k'){
667 if(choice<=UCNV_RESET_TO_UNICODE) {
668 setInitialStateToUnicodeKR(converter, myConverterData);
669 }
670 if(choice!=UCNV_RESET_TO_UNICODE) {
671 setInitialStateFromUnicodeKR(converter, myConverterData);
672 }
673 }
674 }
675 }
676
677 static const char*
678 _ISO2022getName(const UConverter* cnv){
679 if(cnv->extraInfo){
680 UConverterDataISO2022* myData= (UConverterDataISO2022*)cnv->extraInfo;
681 return myData->name;
682 }
683 return NULL;
684 }
685
686
687 /*************** to unicode *******************/
688 /****************************************************************************
689 * Recognized escape sequences are
690 * <ESC>(B ASCII
691 * <ESC>.A ISO-8859-1
692 * <ESC>.F ISO-8859-7
693 * <ESC>(J JISX-201
694 * <ESC>(I JISX-201
695 * <ESC>$B JISX-208
696 * <ESC>$@ JISX-208
697 * <ESC>$(D JISX-212
698 * <ESC>$A GB2312
699 * <ESC>$(C KSC5601
700 */
701 static const int8_t nextStateToUnicodeJP[MAX_STATES_2022]= {
702 /* 0 1 2 3 4 5 6 7 8 9 */
703 INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,SS2_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
704 ,ASCII ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,JISX201 ,HWKANA_7BIT ,JISX201 ,INVALID_STATE
705 ,INVALID_STATE ,INVALID_STATE ,JISX208 ,GB2312 ,JISX208 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
706 ,ISO8859_1 ,ISO8859_7 ,JISX208 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,KSC5601 ,JISX212 ,INVALID_STATE
707 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
708 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
709 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
710 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
711 };
712
713 /*************** to unicode *******************/
714 static const int8_t nextStateToUnicodeCN[MAX_STATES_2022]= {
715 /* 0 1 2 3 4 5 6 7 8 9 */
716 INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,SS2_STATE ,SS3_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
717 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
718 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
719 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
720 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,GB2312_1 ,INVALID_STATE ,ISO_IR_165
721 ,CNS_11643_1 ,CNS_11643_2 ,CNS_11643_3 ,CNS_11643_4 ,CNS_11643_5 ,CNS_11643_6 ,CNS_11643_7 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
722 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
723 ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE ,INVALID_STATE
724 };
725
726
727 static UCNV_TableStates_2022
728 getKey_2022(char c,int32_t* key,int32_t* offset){
729 int32_t togo;
730 int32_t low = 0;
731 int32_t hi = MAX_STATES_2022;
732 int32_t oldmid=0;
733
734 togo = normalize_esq_chars_2022[(uint8_t)c];
735 if(togo == 0) {
736 /* not a valid character anywhere in an escape sequence */
737 *key = 0;
738 *offset = 0;
739 return INVALID_2022;
740 }
741 togo = (*key << 5) + togo;
742
743 while (hi != low) /*binary search*/{
744
745 register int32_t mid = (hi+low) >> 1; /*Finds median*/
746
747 if (mid == oldmid)
748 break;
749
750 if (escSeqStateTable_Key_2022[mid] > togo){
751 hi = mid;
752 }
753 else if (escSeqStateTable_Key_2022[mid] < togo){
754 low = mid;
755 }
756 else /*we found it*/{
757 *key = togo;
758 *offset = mid;
759 return (UCNV_TableStates_2022)escSeqStateTable_Value_2022[mid];
760 }
761 oldmid = mid;
762
763 }
764
765 *key = 0;
766 *offset = 0;
767 return INVALID_2022;
768 }
769
770 /*runs through a state machine to determine the escape sequence - codepage correspondance
771 */
772 static void
773 changeState_2022(UConverter* _this,
774 const char** source,
775 const char* sourceLimit,
776 Variant2022 var,
777 UErrorCode* err){
778 UCNV_TableStates_2022 value;
779 UConverterDataISO2022* myData2022 = ((UConverterDataISO2022*)_this->extraInfo);
780 uint32_t key = myData2022->key;
781 int32_t offset = 0;
782 int8_t initialToULength = _this->toULength;
783 char c;
784
785 value = VALID_NON_TERMINAL_2022;
786 while (*source < sourceLimit) {
787 c = *(*source)++;
788 _this->toUBytes[_this->toULength++]=(uint8_t)c;
789 value = getKey_2022(c,(int32_t *) &key, &offset);
790
791 switch (value){
792
793 case VALID_NON_TERMINAL_2022 :
794 /* continue with the loop */
795 break;
796
797 case VALID_TERMINAL_2022:
798 key = 0;
799 goto DONE;
800
801 case INVALID_2022:
802 goto DONE;
803
804 case VALID_MAYBE_TERMINAL_2022:
805 #ifdef U_ENABLE_GENERIC_ISO_2022
806 /* ESC ( B is ambiguous only for ISO_2022 itself */
807 if(var == ISO_2022) {
808 /* discard toUBytes[] for ESC ( B because this sequence is correct and complete */
809 _this->toULength = 0;
810
811 /* TODO need to indicate that ESC ( B was seen; if failure, then need to replay from source or from MBCS-style replay */
812
813 /* continue with the loop */
814 value = VALID_NON_TERMINAL_2022;
815 break;
816 } else
817 #endif
818 {
819 /* not ISO_2022 itself, finish here */
820 value = VALID_TERMINAL_2022;
821 key = 0;
822 goto DONE;
823 }
824 }
825 }
826
827 DONE:
828 myData2022->key = key;
829
830 if (value == VALID_NON_TERMINAL_2022) {
831 /* indicate that the escape sequence is incomplete: key!=0 */
832 return;
833 } else if (value == INVALID_2022 ) {
834 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
835 } else /* value == VALID_TERMINAL_2022 */ {
836 switch(var){
837 #ifdef U_ENABLE_GENERIC_ISO_2022
838 case ISO_2022:
839 {
840 const char *chosenConverterName = escSeqStateTable_Result_2022[offset];
841 if(chosenConverterName == NULL) {
842 /* SS2 or SS3 */
843 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
844 _this->toUCallbackReason = UCNV_UNASSIGNED;
845 return;
846 }
847
848 _this->mode = UCNV_SI;
849 ucnv_close(myData2022->currentConverter);
850 myData2022->currentConverter = myUConverter = ucnv_open(chosenConverterName, err);
851 if(U_SUCCESS(*err)) {
852 myUConverter->fromCharErrorBehaviour = UCNV_TO_U_CALLBACK_STOP;
853 _this->mode = UCNV_SO;
854 }
855 break;
856 }
857 #endif
858 case ISO_2022_JP:
859 {
860 StateEnum tempState=(StateEnum)nextStateToUnicodeJP[offset];
861 switch(tempState) {
862 case INVALID_STATE:
863 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
864 break;
865 case SS2_STATE:
866 if(myData2022->toU2022State.cs[2]!=0) {
867 if(myData2022->toU2022State.g<2) {
868 myData2022->toU2022State.prevG=myData2022->toU2022State.g;
869 }
870 myData2022->toU2022State.g=2;
871 } else {
872 /* illegal to have SS2 before a matching designator */
873 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
874 }
875 break;
876 /* case SS3_STATE: not used in ISO-2022-JP-x */
877 case ISO8859_1:
878 case ISO8859_7:
879 if((jpCharsetMasks[myData2022->version] & CSM(tempState)) == 0) {
880 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
881 } else {
882 /* G2 charset for SS2 */
883 myData2022->toU2022State.cs[2]=(int8_t)tempState;
884 }
885 break;
886 default:
887 if((jpCharsetMasks[myData2022->version] & CSM(tempState)) == 0) {
888 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
889 } else {
890 /* G0 charset */
891 myData2022->toU2022State.cs[0]=(int8_t)tempState;
892 }
893 break;
894 }
895 }
896 break;
897 case ISO_2022_CN:
898 {
899 StateEnum tempState=(StateEnum)nextStateToUnicodeCN[offset];
900 switch(tempState) {
901 case INVALID_STATE:
902 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
903 break;
904 case SS2_STATE:
905 if(myData2022->toU2022State.cs[2]!=0) {
906 if(myData2022->toU2022State.g<2) {
907 myData2022->toU2022State.prevG=myData2022->toU2022State.g;
908 }
909 myData2022->toU2022State.g=2;
910 } else {
911 /* illegal to have SS2 before a matching designator */
912 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
913 }
914 break;
915 case SS3_STATE:
916 if(myData2022->toU2022State.cs[3]!=0) {
917 if(myData2022->toU2022State.g<2) {
918 myData2022->toU2022State.prevG=myData2022->toU2022State.g;
919 }
920 myData2022->toU2022State.g=3;
921 } else {
922 /* illegal to have SS3 before a matching designator */
923 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
924 }
925 break;
926 case ISO_IR_165:
927 if(myData2022->version==0) {
928 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
929 break;
930 }
931 /*fall through*/
932 case GB2312_1:
933 /*fall through*/
934 case CNS_11643_1:
935 myData2022->toU2022State.cs[1]=(int8_t)tempState;
936 break;
937 case CNS_11643_2:
938 myData2022->toU2022State.cs[2]=(int8_t)tempState;
939 break;
940 default:
941 /* other CNS 11643 planes */
942 if(myData2022->version==0) {
943 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
944 } else {
945 myData2022->toU2022State.cs[3]=(int8_t)tempState;
946 }
947 break;
948 }
949 }
950 break;
951 case ISO_2022_KR:
952 if(offset==0x30){
953 /* nothing to be done, just accept this one escape sequence */
954 } else {
955 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
956 }
957 break;
958
959 default:
960 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
961 break;
962 }
963 }
964 if(U_SUCCESS(*err)) {
965 _this->toULength = 0;
966 } else if(*err==U_ILLEGAL_ESCAPE_SEQUENCE) {
967 if(_this->toULength>1) {
968 /*
969 * Ticket 5691: consistent illegal sequences:
970 * - We include at least the first byte (ESC) in the illegal sequence.
971 * - If any of the non-initial bytes could be the start of a character,
972 * we stop the illegal sequence before the first one of those.
973 * In escape sequences, all following bytes are "printable", that is,
974 * unless they are completely illegal (>7f in SBCS, outside 21..7e in DBCS),
975 * they are valid single/lead bytes.
976 * For simplicity, we always only report the initial ESC byte as the
977 * illegal sequence and back out all other bytes we looked at.
978 */
979 /* Back out some bytes. */
980 int8_t backOutDistance=_this->toULength-1;
981 int8_t bytesFromThisBuffer=_this->toULength-initialToULength;
982 if(backOutDistance<=bytesFromThisBuffer) {
983 /* same as initialToULength<=1 */
984 *source-=backOutDistance;
985 } else {
986 /* Back out bytes from the previous buffer: Need to replay them. */
987 _this->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance);
988 /* same as -(initialToULength-1) */
989 /* preToULength is negative! */
990 uprv_memcpy(_this->preToU, _this->toUBytes+1, -_this->preToULength);
991 *source-=bytesFromThisBuffer;
992 }
993 _this->toULength=1;
994 }
995 } else if(*err==U_UNSUPPORTED_ESCAPE_SEQUENCE) {
996 _this->toUCallbackReason = UCNV_UNASSIGNED;
997 }
998 }
999
1000 /*Checks the characters of the buffer against valid 2022 escape sequences
1001 *if the match we return a pointer to the initial start of the sequence otherwise
1002 *we return sourceLimit
1003 */
1004 /*for 2022 looks ahead in the stream
1005 *to determine the longest possible convertible
1006 *data stream
1007 */
1008 static U_INLINE const char*
1009 getEndOfBuffer_2022(const char** source,
1010 const char* sourceLimit,
1011 UBool flush){
1012
1013 const char* mySource = *source;
1014
1015 #ifdef U_ENABLE_GENERIC_ISO_2022
1016 if (*source >= sourceLimit)
1017 return sourceLimit;
1018
1019 do{
1020
1021 if (*mySource == ESC_2022){
1022 int8_t i;
1023 int32_t key = 0;
1024 int32_t offset;
1025 UCNV_TableStates_2022 value = VALID_NON_TERMINAL_2022;
1026
1027 /* Kludge: I could not
1028 * figure out the reason for validating an escape sequence
1029 * twice - once here and once in changeState_2022().
1030 * is it possible to have an ESC character in a ISO2022
1031 * byte stream which is valid in a code page? Is it legal?
1032 */
1033 for (i=0;
1034 (mySource+i < sourceLimit)&&(value == VALID_NON_TERMINAL_2022);
1035 i++) {
1036 value = getKey_2022(*(mySource+i), &key, &offset);
1037 }
1038 if (value > 0 || *mySource==ESC_2022)
1039 return mySource;
1040
1041 if ((value == VALID_NON_TERMINAL_2022)&&(!flush) )
1042 return sourceLimit;
1043 }
1044 }while (++mySource < sourceLimit);
1045
1046 return sourceLimit;
1047 #else
1048 while(mySource < sourceLimit && *mySource != ESC_2022) {
1049 ++mySource;
1050 }
1051 return mySource;
1052 #endif
1053 }
1054
1055
1056 /* This inline function replicates code in _MBCSFromUChar32() function in ucnvmbcs.c
1057 * any future change in _MBCSFromUChar32() function should be reflected here.
1058 * @return number of bytes in *value; negative number if fallback; 0 if no mapping
1059 */
1060 static U_INLINE int32_t
1061 MBCS_FROM_UCHAR32_ISO2022(UConverterSharedData* sharedData,
1062 UChar32 c,
1063 uint32_t* value,
1064 UBool useFallback,
1065 int outputType)
1066 {
1067 const int32_t *cx;
1068 const uint16_t *table;
1069 uint32_t stage2Entry;
1070 uint32_t myValue;
1071 int32_t length;
1072 const uint8_t *p;
1073 /*
1074 * TODO(markus): Use and require new, faster MBCS conversion table structures.
1075 * Use internal version of ucnv_open() that verifies that the new structures are available,
1076 * else U_INTERNAL_PROGRAM_ERROR.
1077 */
1078 /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
1079 if(c<0x10000 || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
1080 table=sharedData->mbcs.fromUnicodeTable;
1081 stage2Entry=MBCS_STAGE_2_FROM_U(table, c);
1082 /* get the bytes and the length for the output */
1083 if(outputType==MBCS_OUTPUT_2){
1084 myValue=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
1085 if(myValue<=0xff) {
1086 length=1;
1087 } else {
1088 length=2;
1089 }
1090 } else /* outputType==MBCS_OUTPUT_3 */ {
1091 p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c);
1092 myValue=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2];
1093 if(myValue<=0xff) {
1094 length=1;
1095 } else if(myValue<=0xffff) {
1096 length=2;
1097 } else {
1098 length=3;
1099 }
1100 }
1101 /* is this code point assigned, or do we use fallbacks? */
1102 if((stage2Entry&(1<<(16+(c&0xf))))!=0) {
1103 /* assigned */
1104 *value=myValue;
1105 return length;
1106 } else if(FROM_U_USE_FALLBACK(useFallback, c) && myValue!=0) {
1107 /*
1108 * We allow a 0 byte output if the "assigned" bit is set for this entry.
1109 * There is no way with this data structure for fallback output
1110 * to be a zero byte.
1111 */
1112 *value=myValue;
1113 return -length;
1114 }
1115 }
1116
1117 cx=sharedData->mbcs.extIndexes;
1118 if(cx!=NULL) {
1119 return ucnv_extSimpleMatchFromU(cx, c, value, useFallback);
1120 }
1121
1122 /* unassigned */
1123 return 0;
1124 }
1125
1126 /* This inline function replicates code in _MBCSSingleFromUChar32() function in ucnvmbcs.c
1127 * any future change in _MBCSSingleFromUChar32() function should be reflected here.
1128 * @param retval pointer to output byte
1129 * @return 1 roundtrip byte 0 no mapping -1 fallback byte
1130 */
1131 static U_INLINE int32_t
1132 MBCS_SINGLE_FROM_UCHAR32(UConverterSharedData* sharedData,
1133 UChar32 c,
1134 uint32_t* retval,
1135 UBool useFallback)
1136 {
1137 const uint16_t *table;
1138 int32_t value;
1139 /* BMP-only codepages are stored without stage 1 entries for supplementary code points */
1140 if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) {
1141 return 0;
1142 }
1143 /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */
1144 table=sharedData->mbcs.fromUnicodeTable;
1145 /* get the byte for the output */
1146 value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c);
1147 /* is this code point assigned, or do we use fallbacks? */
1148 *retval=(uint32_t)(value&0xff);
1149 if(value>=0xf00) {
1150 return 1; /* roundtrip */
1151 } else if(useFallback ? value>=0x800 : value>=0xc00) {
1152 return -1; /* fallback taken */
1153 } else {
1154 return 0; /* no mapping */
1155 }
1156 }
1157
1158 /*
1159 * Check that the result is a 2-byte value with each byte in the range A1..FE
1160 * (strict EUC DBCS) before accepting it and subtracting 0x80 from each byte
1161 * to move it to the ISO 2022 range 21..7E.
1162 * Return 0 if out of range.
1163 */
1164 static U_INLINE uint32_t
1165 _2022FromGR94DBCS(uint32_t value) {
1166 if( (uint16_t)(value - 0xa1a1) <= (0xfefe - 0xa1a1) &&
1167 (uint8_t)(value - 0xa1) <= (0xfe - 0xa1)
1168 ) {
1169 return value - 0x8080; /* shift down to 21..7e byte range */
1170 } else {
1171 return 0; /* not valid for ISO 2022 */
1172 }
1173 }
1174
1175 #if 0 /* 5691: Call sites now check for validity. They can just += 0x8080 after that. */
1176 /*
1177 * This method does the reverse of _2022FromGR94DBCS(). Given the 2022 code point, it returns the
1178 * 2 byte value that is in the range A1..FE for each byte. Otherwise it returns the 2022 code point
1179 * unchanged.
1180 */
1181 static U_INLINE uint32_t
1182 _2022ToGR94DBCS(uint32_t value) {
1183 uint32_t returnValue = value + 0x8080;
1184 if( (uint16_t)(returnValue - 0xa1a1) <= (0xfefe - 0xa1a1) &&
1185 (uint8_t)(returnValue - 0xa1) <= (0xfe - 0xa1)) {
1186 return returnValue;
1187 } else {
1188 return value;
1189 }
1190 }
1191 #endif
1192
1193 #ifdef U_ENABLE_GENERIC_ISO_2022
1194
1195 /**********************************************************************************
1196 * ISO-2022 Converter
1197 *
1198 *
1199 */
1200
1201 static void
1202 T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC(UConverterToUnicodeArgs* args,
1203 UErrorCode* err){
1204 const char* mySourceLimit, *realSourceLimit;
1205 const char* sourceStart;
1206 const UChar* myTargetStart;
1207 UConverter* saveThis;
1208 UConverterDataISO2022* myData;
1209 int8_t length;
1210
1211 saveThis = args->converter;
1212 myData=((UConverterDataISO2022*)(saveThis->extraInfo));
1213
1214 realSourceLimit = args->sourceLimit;
1215 while (args->source < realSourceLimit) {
1216 if(myData->key == 0) { /* are we in the middle of an escape sequence? */
1217 /*Find the end of the buffer e.g : Next Escape Seq | end of Buffer*/
1218 mySourceLimit = getEndOfBuffer_2022(&(args->source), realSourceLimit, args->flush);
1219
1220 if(args->source < mySourceLimit) {
1221 if(myData->currentConverter==NULL) {
1222 myData->currentConverter = ucnv_open("ASCII",err);
1223 if(U_FAILURE(*err)){
1224 return;
1225 }
1226
1227 myData->currentConverter->fromCharErrorBehaviour = UCNV_TO_U_CALLBACK_STOP;
1228 saveThis->mode = UCNV_SO;
1229 }
1230
1231 /* convert to before the ESC or until the end of the buffer */
1232 myData->isFirstBuffer=FALSE;
1233 sourceStart = args->source;
1234 myTargetStart = args->target;
1235 args->converter = myData->currentConverter;
1236 ucnv_toUnicode(args->converter,
1237 &args->target,
1238 args->targetLimit,
1239 &args->source,
1240 mySourceLimit,
1241 args->offsets,
1242 (UBool)(args->flush && mySourceLimit == realSourceLimit),
1243 err);
1244 args->converter = saveThis;
1245
1246 if (*err == U_BUFFER_OVERFLOW_ERROR) {
1247 /* move the overflow buffer */
1248 length = saveThis->UCharErrorBufferLength = myData->currentConverter->UCharErrorBufferLength;
1249 myData->currentConverter->UCharErrorBufferLength = 0;
1250 if(length > 0) {
1251 uprv_memcpy(saveThis->UCharErrorBuffer,
1252 myData->currentConverter->UCharErrorBuffer,
1253 length*U_SIZEOF_UCHAR);
1254 }
1255 return;
1256 }
1257
1258 /*
1259 * At least one of:
1260 * -Error while converting
1261 * -Done with entire buffer
1262 * -Need to write offsets or update the current offset
1263 * (leave that up to the code in ucnv.c)
1264 *
1265 * or else we just stopped at an ESC byte and continue with changeState_2022()
1266 */
1267 if (U_FAILURE(*err) ||
1268 (args->source == realSourceLimit) ||
1269 (args->offsets != NULL && (args->target != myTargetStart || args->source != sourceStart) ||
1270 (mySourceLimit < realSourceLimit && myData->currentConverter->toULength > 0))
1271 ) {
1272 /* copy partial or error input for truncated detection and error handling */
1273 if(U_FAILURE(*err)) {
1274 length = saveThis->invalidCharLength = myData->currentConverter->invalidCharLength;
1275 if(length > 0) {
1276 uprv_memcpy(saveThis->invalidCharBuffer, myData->currentConverter->invalidCharBuffer, length);
1277 }
1278 } else {
1279 length = saveThis->toULength = myData->currentConverter->toULength;
1280 if(length > 0) {
1281 uprv_memcpy(saveThis->toUBytes, myData->currentConverter->toUBytes, length);
1282 if(args->source < mySourceLimit) {
1283 *err = U_TRUNCATED_CHAR_FOUND; /* truncated input before ESC */
1284 }
1285 }
1286 }
1287 return;
1288 }
1289 }
1290 }
1291
1292 sourceStart = args->source;
1293 changeState_2022(args->converter,
1294 &(args->source),
1295 realSourceLimit,
1296 ISO_2022,
1297 err);
1298 if (U_FAILURE(*err) || (args->source != sourceStart && args->offsets != NULL)) {
1299 /* let the ucnv.c code update its current offset */
1300 return;
1301 }
1302 }
1303 }
1304
1305 #endif
1306
1307 /*
1308 * To Unicode Callback helper function
1309 */
1310 static void
1311 toUnicodeCallback(UConverter *cnv,
1312 const uint32_t sourceChar, const uint32_t targetUniChar,
1313 UErrorCode* err){
1314 if(sourceChar>0xff){
1315 cnv->toUBytes[0] = (uint8_t)(sourceChar>>8);
1316 cnv->toUBytes[1] = (uint8_t)sourceChar;
1317 cnv->toULength = 2;
1318 }
1319 else{
1320 cnv->toUBytes[0] =(char) sourceChar;
1321 cnv->toULength = 1;
1322 }
1323
1324 if(targetUniChar == (missingCharMarker-1/*0xfffe*/)){
1325 *err = U_INVALID_CHAR_FOUND;
1326 }
1327 else{
1328 *err = U_ILLEGAL_CHAR_FOUND;
1329 }
1330 }
1331
1332 /**************************************ISO-2022-JP*************************************************/
1333
1334 /************************************** IMPORTANT **************************************************
1335 * The UConverter_fromUnicode_ISO2022_JP converter does not use ucnv_fromUnicode() functions for SBCS,DBCS and
1336 * MBCS; instead, the values are obtained directly by calling _MBCSFromUChar32().
1337 * The converter iterates over each Unicode codepoint
1338 * to obtain the equivalent codepoints from the codepages supported. Since the source buffer is
1339 * processed one char at a time it would make sense to reduce the extra processing a canned converter
1340 * would do as far as possible.
1341 *
1342 * If the implementation of these macros or structure of sharedData struct change in the future, make
1343 * sure that ISO-2022 is also changed.
1344 ***************************************************************************************************
1345 */
1346
1347 /***************************************************************************************************
1348 * Rules for ISO-2022-jp encoding
1349 * (i) Escape sequences must be fully contained within a line they should not
1350 * span new lines or CRs
1351 * (ii) If the last character on a line is represented by two bytes then an ASCII or
1352 * JIS-Roman character escape sequence should follow before the line terminates
1353 * (iii) If the first character on the line is represented by two bytes then a two
1354 * byte character escape sequence should precede it
1355 * (iv) If no escape sequence is encountered then the characters are ASCII
1356 * (v) Latin(ISO-8859-1) and Greek(ISO-8859-7) characters must be designated to G2,
1357 * and invoked with SS2 (ESC N).
1358 * (vi) If there is any G0 designation in text, there must be a switch to
1359 * ASCII or to JIS X 0201-Roman before a space character (but not
1360 * necessarily before "ESC 4/14 2/0" or "ESC N ' '") or control
1361 * characters such as tab or CRLF.
1362 * (vi) Supported encodings:
1363 * ASCII, JISX201, JISX208, JISX212, GB2312, KSC5601, ISO-8859-1,ISO-8859-7
1364 *
1365 * source : RFC-1554
1366 *
1367 * JISX201, JISX208,JISX212 : new .cnv data files created
1368 * KSC5601 : alias to ibm-949 mapping table
1369 * GB2312 : alias to ibm-1386 mapping table
1370 * ISO-8859-1 : Algorithmic implemented as LATIN1 case
1371 * ISO-8859-7 : alisas to ibm-9409 mapping table
1372 */
1373
1374 /* preference order of JP charsets */
1375 static const StateEnum jpCharsetPref[]={
1376 ASCII,
1377 JISX201,
1378 ISO8859_1,
1379 ISO8859_7,
1380 JISX208,
1381 JISX212,
1382 GB2312,
1383 KSC5601,
1384 HWKANA_7BIT
1385 };
1386
1387 /*
1388 * The escape sequences must be in order of the enum constants like JISX201 = 3,
1389 * not in order of jpCharsetPref[]!
1390 */
1391 static const char escSeqChars[][6] ={
1392 "\x1B\x28\x42", /* <ESC>(B ASCII */
1393 "\x1B\x2E\x41", /* <ESC>.A ISO-8859-1 */
1394 "\x1B\x2E\x46", /* <ESC>.F ISO-8859-7 */
1395 "\x1B\x28\x4A", /* <ESC>(J JISX-201 */
1396 "\x1B\x24\x42", /* <ESC>$B JISX-208 */
1397 "\x1B\x24\x28\x44", /* <ESC>$(D JISX-212 */
1398 "\x1B\x24\x41", /* <ESC>$A GB2312 */
1399 "\x1B\x24\x28\x43", /* <ESC>$(C KSC5601 */
1400 "\x1B\x28\x49" /* <ESC>(I HWKANA_7BIT */
1401
1402 };
1403 static const int8_t escSeqCharsLen[] ={
1404 3, /* length of <ESC>(B ASCII */
1405 3, /* length of <ESC>.A ISO-8859-1 */
1406 3, /* length of <ESC>.F ISO-8859-7 */
1407 3, /* length of <ESC>(J JISX-201 */
1408 3, /* length of <ESC>$B JISX-208 */
1409 4, /* length of <ESC>$(D JISX-212 */
1410 3, /* length of <ESC>$A GB2312 */
1411 4, /* length of <ESC>$(C KSC5601 */
1412 3 /* length of <ESC>(I HWKANA_7BIT */
1413 };
1414
1415 /*
1416 * The iteration over various code pages works this way:
1417 * i) Get the currentState from myConverterData->currentState
1418 * ii) Check if the character is mapped to a valid character in the currentState
1419 * Yes -> a) set the initIterState to currentState
1420 * b) remain in this state until an invalid character is found
1421 * No -> a) go to the next code page and find the character
1422 * iii) Before changing the state increment the current state check if the current state
1423 * is equal to the intitIteration state
1424 * Yes -> A character that cannot be represented in any of the supported encodings
1425 * break and return a U_INVALID_CHARACTER error
1426 * No -> Continue and find the character in next code page
1427 *
1428 *
1429 * TODO: Implement a priority technique where the users are allowed to set the priority of code pages
1430 */
1431
1432 /* Map 00..7F to Unicode according to JIS X 0201. */
1433 static U_INLINE uint32_t
1434 jisx201ToU(uint32_t value) {
1435 if(value < 0x5c) {
1436 return value;
1437 } else if(value == 0x5c) {
1438 return 0xa5;
1439 } else if(value == 0x7e) {
1440 return 0x203e;
1441 } else /* value <= 0x7f */ {
1442 return value;
1443 }
1444 }
1445
1446 /* Map Unicode to 00..7F according to JIS X 0201. Return U+FFFE if unmappable. */
1447 static U_INLINE uint32_t
1448 jisx201FromU(uint32_t value) {
1449 if(value<=0x7f) {
1450 if(value!=0x5c && value!=0x7e) {
1451 return value;
1452 }
1453 } else if(value==0xa5) {
1454 return 0x5c;
1455 } else if(value==0x203e) {
1456 return 0x7e;
1457 }
1458 return 0xfffe;
1459 }
1460
1461 /*
1462 * Take a valid Shift-JIS byte pair, check that it is in the range corresponding
1463 * to JIS X 0208, and convert it to a pair of 21..7E bytes.
1464 * Return 0 if the byte pair is out of range.
1465 */
1466 static U_INLINE uint32_t
1467 _2022FromSJIS(uint32_t value) {
1468 uint8_t trail;
1469
1470 if(value > 0xEFFC) {
1471 return 0; /* beyond JIS X 0208 */
1472 }
1473
1474 trail = (uint8_t)value;
1475
1476 value &= 0xff00; /* lead byte */
1477 if(value <= 0x9f00) {
1478 value -= 0x7000;
1479 } else /* 0xe000 <= value <= 0xef00 */ {
1480 value -= 0xb000;
1481 }
1482 value <<= 1;
1483
1484 if(trail <= 0x9e) {
1485 value -= 0x100;
1486 if(trail <= 0x7e) {
1487 value |= trail - 0x1f;
1488 } else {
1489 value |= trail - 0x20;
1490 }
1491 } else /* trail <= 0xfc */ {
1492 value |= trail - 0x7e;
1493 }
1494 return value;
1495 }
1496
1497 /*
1498 * Convert a pair of JIS X 0208 21..7E bytes to Shift-JIS.
1499 * If either byte is outside 21..7E make sure that the result is not valid
1500 * for Shift-JIS so that the converter catches it.
1501 * Some invalid byte values already turn into equally invalid Shift-JIS
1502 * byte values and need not be tested explicitly.
1503 */
1504 static U_INLINE void
1505 _2022ToSJIS(uint8_t c1, uint8_t c2, char bytes[2]) {
1506 if(c1&1) {
1507 ++c1;
1508 if(c2 <= 0x5f) {
1509 c2 += 0x1f;
1510 } else if(c2 <= 0x7e) {
1511 c2 += 0x20;
1512 } else {
1513 c2 = 0; /* invalid */
1514 }
1515 } else {
1516 if((uint8_t)(c2-0x21) <= ((0x7e)-0x21)) {
1517 c2 += 0x7e;
1518 } else {
1519 c2 = 0; /* invalid */
1520 }
1521 }
1522 c1 >>= 1;
1523 if(c1 <= 0x2f) {
1524 c1 += 0x70;
1525 } else if(c1 <= 0x3f) {
1526 c1 += 0xb0;
1527 } else {
1528 c1 = 0; /* invalid */
1529 }
1530 bytes[0] = (char)c1;
1531 bytes[1] = (char)c2;
1532 }
1533
1534 /*
1535 * JIS X 0208 has fallbacks from Unicode half-width Katakana to full-width (DBCS)
1536 * Katakana.
1537 * Now that we use a Shift-JIS table for JIS X 0208 we need to hardcode these fallbacks
1538 * because Shift-JIS roundtrips half-width Katakana to single bytes.
1539 * These were the only fallbacks in ICU's jisx-208.ucm file.
1540 */
1541 static const uint16_t hwkana_fb[HWKANA_END - HWKANA_START + 1] = {
1542 0x2123, /* U+FF61 */
1543 0x2156,
1544 0x2157,
1545 0x2122,
1546 0x2126,
1547 0x2572,
1548 0x2521,
1549 0x2523,
1550 0x2525,
1551 0x2527,
1552 0x2529,
1553 0x2563,
1554 0x2565,
1555 0x2567,
1556 0x2543,
1557 0x213C, /* U+FF70 */
1558 0x2522,
1559 0x2524,
1560 0x2526,
1561 0x2528,
1562 0x252A,
1563 0x252B,
1564 0x252D,
1565 0x252F,
1566 0x2531,
1567 0x2533,
1568 0x2535,
1569 0x2537,
1570 0x2539,
1571 0x253B,
1572 0x253D,
1573 0x253F, /* U+FF80 */
1574 0x2541,
1575 0x2544,
1576 0x2546,
1577 0x2548,
1578 0x254A,
1579 0x254B,
1580 0x254C,
1581 0x254D,
1582 0x254E,
1583 0x254F,
1584 0x2552,
1585 0x2555,
1586 0x2558,
1587 0x255B,
1588 0x255E,
1589 0x255F, /* U+FF90 */
1590 0x2560,
1591 0x2561,
1592 0x2562,
1593 0x2564,
1594 0x2566,
1595 0x2568,
1596 0x2569,
1597 0x256A,
1598 0x256B,
1599 0x256C,
1600 0x256D,
1601 0x256F,
1602 0x2573,
1603 0x212B,
1604 0x212C /* U+FF9F */
1605 };
1606
1607 static void
1608 UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC(UConverterFromUnicodeArgs* args, UErrorCode* err) {
1609 UConverter *cnv = args->converter;
1610 UConverterDataISO2022 *converterData;
1611 ISO2022State *pFromU2022State;
1612 uint8_t *target = (uint8_t *) args->target;
1613 const uint8_t *targetLimit = (const uint8_t *) args->targetLimit;
1614 const UChar* source = args->source;
1615 const UChar* sourceLimit = args->sourceLimit;
1616 int32_t* offsets = args->offsets;
1617 UChar32 sourceChar;
1618 char buffer[8];
1619 int32_t len, outLen;
1620 int8_t choices[10];
1621 int32_t choiceCount;
1622 uint32_t targetValue = 0;
1623 UBool useFallback;
1624
1625 int32_t i;
1626 int8_t cs, g;
1627
1628 /* set up the state */
1629 converterData = (UConverterDataISO2022*)cnv->extraInfo;
1630 pFromU2022State = &converterData->fromU2022State;
1631
1632 choiceCount = 0;
1633
1634 /* check if the last codepoint of previous buffer was a lead surrogate*/
1635 if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) {
1636 goto getTrail;
1637 }
1638
1639 while(source < sourceLimit) {
1640 if(target < targetLimit) {
1641
1642 sourceChar = *(source++);
1643 /*check if the char is a First surrogate*/
1644 if(UTF_IS_SURROGATE(sourceChar)) {
1645 if(UTF_IS_SURROGATE_FIRST(sourceChar)) {
1646 getTrail:
1647 /*look ahead to find the trail surrogate*/
1648 if(source < sourceLimit) {
1649 /* test the following code unit */
1650 UChar trail=(UChar) *source;
1651 if(UTF_IS_SECOND_SURROGATE(trail)) {
1652 source++;
1653 sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail);
1654 cnv->fromUChar32=0x00;
1655 /* convert this supplementary code point */
1656 /* exit this condition tree */
1657 } else {
1658 /* this is an unmatched lead code unit (1st surrogate) */
1659 /* callback(illegal) */
1660 *err=U_ILLEGAL_CHAR_FOUND;
1661 cnv->fromUChar32=sourceChar;
1662 break;
1663 }
1664 } else {
1665 /* no more input */
1666 cnv->fromUChar32=sourceChar;
1667 break;
1668 }
1669 } else {
1670 /* this is an unmatched trail code unit (2nd surrogate) */
1671 /* callback(illegal) */
1672 *err=U_ILLEGAL_CHAR_FOUND;
1673 cnv->fromUChar32=sourceChar;
1674 break;
1675 }
1676 }
1677
1678 /* do not convert SO/SI/ESC */
1679 if(IS_2022_CONTROL(sourceChar)) {
1680 /* callback(illegal) */
1681 *err=U_ILLEGAL_CHAR_FOUND;
1682 cnv->fromUChar32=sourceChar;
1683 break;
1684 }
1685
1686 /* do the conversion */
1687
1688 if(choiceCount == 0) {
1689 uint16_t csm;
1690
1691 /*
1692 * The csm variable keeps track of which charsets are allowed
1693 * and not used yet while building the choices[].
1694 */
1695 csm = jpCharsetMasks[converterData->version];
1696 choiceCount = 0;
1697
1698 /* JIS7/8: try single-byte half-width Katakana before JISX208 */
1699 if(converterData->version == 3 || converterData->version == 4) {
1700 choices[choiceCount++] = (int8_t)HWKANA_7BIT;
1701 }
1702 /* Do not try single-byte half-width Katakana for other versions. */
1703 csm &= ~CSM(HWKANA_7BIT);
1704
1705 /* try the current G0 charset */
1706 choices[choiceCount++] = cs = pFromU2022State->cs[0];
1707 csm &= ~CSM(cs);
1708
1709 /* try the current G2 charset */
1710 if((cs = pFromU2022State->cs[2]) != 0) {
1711 choices[choiceCount++] = cs;
1712 csm &= ~CSM(cs);
1713 }
1714
1715 /* try all the other possible charsets */
1716 for(i = 0; i < LENGTHOF(jpCharsetPref); ++i) {
1717 cs = (int8_t)jpCharsetPref[i];
1718 if(CSM(cs) & csm) {
1719 choices[choiceCount++] = cs;
1720 csm &= ~CSM(cs);
1721 }
1722 }
1723 }
1724
1725 cs = g = 0;
1726 /*
1727 * len==0: no mapping found yet
1728 * len<0: found a fallback result: continue looking for a roundtrip but no further fallbacks
1729 * len>0: found a roundtrip result, done
1730 */
1731 len = 0;
1732 /*
1733 * We will turn off useFallback after finding a fallback,
1734 * but we still get fallbacks from PUA code points as usual.
1735 * Therefore, we will also need to check that we don't overwrite
1736 * an early fallback with a later one.
1737 */
1738 useFallback = cnv->useFallback;
1739
1740 for(i = 0; i < choiceCount && len <= 0; ++i) {
1741 uint32_t value;
1742 int32_t len2;
1743 int8_t cs0 = choices[i];
1744 switch(cs0) {
1745 case ASCII:
1746 if(sourceChar <= 0x7f) {
1747 targetValue = (uint32_t)sourceChar;
1748 len = 1;
1749 cs = cs0;
1750 g = 0;
1751 }
1752 break;
1753 case ISO8859_1:
1754 if(GR96_START <= sourceChar && sourceChar <= GR96_END) {
1755 targetValue = (uint32_t)sourceChar - 0x80;
1756 len = 1;
1757 cs = cs0;
1758 g = 2;
1759 }
1760 break;
1761 case HWKANA_7BIT:
1762 if((uint32_t)(sourceChar - HWKANA_START) <= (HWKANA_END - HWKANA_START)) {
1763 if(converterData->version==3) {
1764 /* JIS7: use G1 (SO) */
1765 /* Shift U+FF61..U+FF9F to bytes 21..5F. */
1766 targetValue = (uint32_t)(sourceChar - (HWKANA_START - 0x21));
1767 len = 1;
1768 pFromU2022State->cs[1] = cs = cs0; /* do not output an escape sequence */
1769 g = 1;
1770 } else if(converterData->version==4) {
1771 /* JIS8: use 8-bit bytes with any single-byte charset, see escape sequence output below */
1772 /* Shift U+FF61..U+FF9F to bytes A1..DF. */
1773 targetValue = (uint32_t)(sourceChar - (HWKANA_START - 0xa1));
1774 len = 1;
1775
1776 cs = pFromU2022State->cs[0];
1777 if(IS_JP_DBCS(cs)) {
1778 /* switch from a DBCS charset to JISX201 */
1779 cs = (int8_t)JISX201;
1780 }
1781 /* else stay in the current G0 charset */
1782 g = 0;
1783 }
1784 /* else do not use HWKANA_7BIT with other versions */
1785 }
1786 break;
1787 case JISX201:
1788 /* G0 SBCS */
1789 value = jisx201FromU(sourceChar);
1790 if(value <= 0x7f) {
1791 targetValue = value;
1792 len = 1;
1793 cs = cs0;
1794 g = 0;
1795 useFallback = FALSE;
1796 }
1797 break;
1798 case JISX208:
1799 /* G0 DBCS from Shift-JIS table */
1800 len2 = MBCS_FROM_UCHAR32_ISO2022(
1801 converterData->myConverterArray[cs0],
1802 sourceChar, &value,
1803 useFallback, MBCS_OUTPUT_2);
1804 if(len2 == 2 || (len2 == -2 && len == 0)) { /* only accept DBCS: abs(len)==2 */
1805 value = _2022FromSJIS(value);
1806 if(value != 0) {
1807 targetValue = value;
1808 len = len2;
1809 cs = cs0;
1810 g = 0;
1811 useFallback = FALSE;
1812 }
1813 } else if(len == 0 && useFallback &&
1814 (uint32_t)(sourceChar - HWKANA_START) <= (HWKANA_END - HWKANA_START)) {
1815 targetValue = hwkana_fb[sourceChar - HWKANA_START];
1816 len = -2;
1817 cs = cs0;
1818 g = 0;
1819 useFallback = FALSE;
1820 }
1821 break;
1822 case ISO8859_7:
1823 /* G0 SBCS forced to 7-bit output */
1824 len2 = MBCS_SINGLE_FROM_UCHAR32(
1825 converterData->myConverterArray[cs0],
1826 sourceChar, &value,
1827 useFallback);
1828 if(len2 != 0 && !(len2 < 0 && len != 0) && GR96_START <= value && value <= GR96_END) {
1829 targetValue = value - 0x80;
1830 len = len2;
1831 cs = cs0;
1832 g = 2;
1833 useFallback = FALSE;
1834 }
1835 break;
1836 default:
1837 /* G0 DBCS */
1838 len2 = MBCS_FROM_UCHAR32_ISO2022(
1839 converterData->myConverterArray[cs0],
1840 sourceChar, &value,
1841 useFallback, MBCS_OUTPUT_2);
1842 if(len2 == 2 || (len2 == -2 && len == 0)) { /* only accept DBCS: abs(len)==2 */
1843 if(cs0 == KSC5601) {
1844 /*
1845 * Check for valid bytes for the encoding scheme.
1846 * This is necessary because the sub-converter (windows-949)
1847 * has a broader encoding scheme than is valid for 2022.
1848 */
1849 value = _2022FromGR94DBCS(value);
1850 if(value == 0) {
1851 break;
1852 }
1853 }
1854 targetValue = value;
1855 len = len2;
1856 cs = cs0;
1857 g = 0;
1858 useFallback = FALSE;
1859 }
1860 break;
1861 }
1862 }
1863
1864 if(len != 0) {
1865 if(len < 0) {
1866 len = -len; /* fallback */
1867 }
1868 outLen = 0; /* count output bytes */
1869
1870 /* write SI if necessary (only for JIS7) */
1871 if(pFromU2022State->g == 1 && g == 0) {
1872 buffer[outLen++] = UCNV_SI;
1873 pFromU2022State->g = 0;
1874 }
1875
1876 /* write the designation sequence if necessary */
1877 if(cs != pFromU2022State->cs[g]) {
1878 int32_t escLen = escSeqCharsLen[cs];
1879 uprv_memcpy(buffer + outLen, escSeqChars[cs], escLen);
1880 outLen += escLen;
1881 pFromU2022State->cs[g] = cs;
1882
1883 /* invalidate the choices[] */
1884 choiceCount = 0;
1885 }
1886
1887 /* write the shift sequence if necessary */
1888 if(g != pFromU2022State->g) {
1889 switch(g) {
1890 /* case 0 handled before writing escapes */
1891 case 1:
1892 buffer[outLen++] = UCNV_SO;
1893 pFromU2022State->g = 1;
1894 break;
1895 default: /* case 2 */
1896 buffer[outLen++] = 0x1b;
1897 buffer[outLen++] = 0x4e;
1898 break;
1899 /* no case 3: no SS3 in ISO-2022-JP-x */
1900 }
1901 }
1902
1903 /* write the output bytes */
1904 if(len == 1) {
1905 buffer[outLen++] = (char)targetValue;
1906 } else /* len == 2 */ {
1907 buffer[outLen++] = (char)(targetValue >> 8);
1908 buffer[outLen++] = (char)targetValue;
1909 }
1910 } else {
1911 /*
1912 * if we cannot find the character after checking all codepages
1913 * then this is an error
1914 */
1915 *err = U_INVALID_CHAR_FOUND;
1916 cnv->fromUChar32=sourceChar;
1917 break;
1918 }
1919
1920 if(sourceChar == CR || sourceChar == LF) {
1921 /* reset the G2 state at the end of a line (conversion got us into ASCII or JISX201 already) */
1922 pFromU2022State->cs[2] = 0;
1923 choiceCount = 0;
1924 }
1925
1926 /* output outLen>0 bytes in buffer[] */
1927 if(outLen == 1) {
1928 *target++ = buffer[0];
1929 if(offsets) {
1930 *offsets++ = (int32_t)(source - args->source - 1); /* -1: known to be ASCII */
1931 }
1932 } else if(outLen == 2 && (target + 2) <= targetLimit) {
1933 *target++ = buffer[0];
1934 *target++ = buffer[1];
1935 if(offsets) {
1936 int32_t sourceIndex = (int32_t)(source - args->source - U16_LENGTH(sourceChar));
1937 *offsets++ = sourceIndex;
1938 *offsets++ = sourceIndex;
1939 }
1940 } else {
1941 fromUWriteUInt8(
1942 cnv,
1943 buffer, outLen,
1944 &target, (const char *)targetLimit,
1945 &offsets, (int32_t)(source - args->source - U16_LENGTH(sourceChar)),
1946 err);
1947 if(U_FAILURE(*err)) {
1948 break;
1949 }
1950 }
1951 } /* end if(myTargetIndex<myTargetLength) */
1952 else{
1953 *err =U_BUFFER_OVERFLOW_ERROR;
1954 break;
1955 }
1956
1957 }/* end while(mySourceIndex<mySourceLength) */
1958
1959 /*
1960 * the end of the input stream and detection of truncated input
1961 * are handled by the framework, but for ISO-2022-JP conversion
1962 * we need to be in ASCII mode at the very end
1963 *
1964 * conditions:
1965 * successful
1966 * in SO mode or not in ASCII mode
1967 * end of input and no truncated input
1968 */
1969 if( U_SUCCESS(*err) &&
1970 (pFromU2022State->g!=0 || pFromU2022State->cs[0]!=ASCII) &&
1971 args->flush && source>=sourceLimit && cnv->fromUChar32==0
1972 ) {
1973 int32_t sourceIndex;
1974
1975 outLen = 0;
1976
1977 if(pFromU2022State->g != 0) {
1978 buffer[outLen++] = UCNV_SI;
1979 pFromU2022State->g = 0;
1980 }
1981
1982 if(pFromU2022State->cs[0] != ASCII) {
1983 int32_t escLen = escSeqCharsLen[ASCII];
1984 uprv_memcpy(buffer + outLen, escSeqChars[ASCII], escLen);
1985 outLen += escLen;
1986 pFromU2022State->cs[0] = (int8_t)ASCII;
1987 }
1988
1989 /* get the source index of the last input character */
1990 /*
1991 * TODO this would be simpler and more reliable if we used a pair
1992 * of sourceIndex/prevSourceIndex like in ucnvmbcs.c
1993 * so that we could simply use the prevSourceIndex here;
1994 * this code gives an incorrect result for the rare case of an unmatched
1995 * trail surrogate that is alone in the last buffer of the text stream
1996 */
1997 sourceIndex=(int32_t)(source-args->source);
1998 if(sourceIndex>0) {
1999 --sourceIndex;
2000 if( U16_IS_TRAIL(args->source[sourceIndex]) &&
2001 (sourceIndex==0 || U16_IS_LEAD(args->source[sourceIndex-1]))
2002 ) {
2003 --sourceIndex;
2004 }
2005 } else {
2006 sourceIndex=-1;
2007 }
2008
2009 fromUWriteUInt8(
2010 cnv,
2011 buffer, outLen,
2012 &target, (const char *)targetLimit,
2013 &offsets, sourceIndex,
2014 err);
2015 }
2016
2017 /*save the state and return */
2018 args->source = source;
2019 args->target = (char*)target;
2020 }
2021
2022 /*************** to unicode *******************/
2023
2024 static void
2025 UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC(UConverterToUnicodeArgs *args,
2026 UErrorCode* err){
2027 char tempBuf[2];
2028 const char *mySource = (char *) args->source;
2029 UChar *myTarget = args->target;
2030 const char *mySourceLimit = args->sourceLimit;
2031 uint32_t targetUniChar = 0x0000;
2032 uint32_t mySourceChar = 0x0000;
2033 uint32_t tmpSourceChar = 0x0000;
2034 UConverterDataISO2022* myData;
2035 ISO2022State *pToU2022State;
2036 StateEnum cs;
2037
2038 myData=(UConverterDataISO2022*)(args->converter->extraInfo);
2039 pToU2022State = &myData->toU2022State;
2040
2041 if(myData->key != 0) {
2042 /* continue with a partial escape sequence */
2043 goto escape;
2044 } else if(args->converter->toULength == 1 && mySource < mySourceLimit && myTarget < args->targetLimit) {
2045 /* continue with a partial double-byte character */
2046 mySourceChar = args->converter->toUBytes[0];
2047 args->converter->toULength = 0;
2048 cs = (StateEnum)pToU2022State->cs[pToU2022State->g];
2049 targetUniChar = missingCharMarker;
2050 goto getTrailByte;
2051 }
2052
2053 while(mySource < mySourceLimit){
2054
2055 targetUniChar =missingCharMarker;
2056
2057 if(myTarget < args->targetLimit){
2058
2059 mySourceChar= (unsigned char) *mySource++;
2060
2061 switch(mySourceChar) {
2062 case UCNV_SI:
2063 if(myData->version==3) {
2064 pToU2022State->g=0;
2065 continue;
2066 } else {
2067 /* only JIS7 uses SI/SO, not ISO-2022-JP-x */
2068 myData->isEmptySegment = FALSE; /* reset this, we have a different error */
2069 break;
2070 }
2071
2072 case UCNV_SO:
2073 if(myData->version==3) {
2074 /* JIS7: switch to G1 half-width Katakana */
2075 pToU2022State->cs[1] = (int8_t)HWKANA_7BIT;
2076 pToU2022State->g=1;
2077 continue;
2078 } else {
2079 /* only JIS7 uses SI/SO, not ISO-2022-JP-x */
2080 myData->isEmptySegment = FALSE; /* reset this, we have a different error */
2081 break;
2082 }
2083
2084 case ESC_2022:
2085 mySource--;
2086 escape:
2087 {
2088 const char * mySourceBefore = mySource;
2089 int8_t toULengthBefore = args->converter->toULength;
2090
2091 changeState_2022(args->converter,&(mySource),
2092 mySourceLimit, ISO_2022_JP,err);
2093
2094 /* If in ISO-2022-JP only and we successully completed an escape sequence, but previous segment was empty, create an error */
2095 if(myData->version==0 && myData->key==0 && U_SUCCESS(*err) && myData->isEmptySegment) {
2096 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
2097 args->converter->toUCallbackReason = UCNV_IRREGULAR;
2098 args->converter->toULength = (int8_t)(toULengthBefore + (mySource - mySourceBefore));
2099 }
2100 }
2101
2102 /* invalid or illegal escape sequence */
2103 if(U_FAILURE(*err)){
2104 args->target = myTarget;
2105 args->source = mySource;
2106 myData->isEmptySegment = FALSE; /* Reset to avoid future spurious errors */
2107 return;
2108 }
2109 /* If we successfully completed an escape sequence, we begin a new segment, empty so far */
2110 if(myData->key==0) {
2111 myData->isEmptySegment = TRUE;
2112 }
2113 continue;
2114
2115 /* ISO-2022-JP does not use single-byte (C1) SS2 and SS3 */
2116
2117 case CR:
2118 /*falls through*/
2119 case LF:
2120 /* automatically reset to single-byte mode */
2121 if((StateEnum)pToU2022State->cs[0] != ASCII && (StateEnum)pToU2022State->cs[0] != JISX201) {
2122 pToU2022State->cs[0] = (int8_t)ASCII;
2123 }
2124 pToU2022State->cs[2] = 0;
2125 pToU2022State->g = 0;
2126 /* falls through */
2127 default:
2128 /* convert one or two bytes */
2129 myData->isEmptySegment = FALSE;
2130 cs = (StateEnum)pToU2022State->cs[pToU2022State->g];
2131 if( (uint8_t)(mySourceChar - 0xa1) <= (0xdf - 0xa1) && myData->version==4 &&
2132 !IS_JP_DBCS(cs)
2133 ) {
2134 /* 8-bit halfwidth katakana in any single-byte mode for JIS8 */
2135 targetUniChar = mySourceChar + (HWKANA_START - 0xa1);
2136
2137 /* return from a single-shift state to the previous one */
2138 if(pToU2022State->g >= 2) {
2139 pToU2022State->g=pToU2022State->prevG;
2140 }
2141 } else switch(cs) {
2142 case ASCII:
2143 if(mySourceChar <= 0x7f) {
2144 targetUniChar = mySourceChar;
2145 }
2146 break;
2147 case ISO8859_1:
2148 if(mySourceChar <= 0x7f) {
2149 targetUniChar = mySourceChar + 0x80;
2150 }
2151 /* return from a single-shift state to the previous one */
2152 pToU2022State->g=pToU2022State->prevG;
2153 break;
2154 case ISO8859_7:
2155 if(mySourceChar <= 0x7f) {
2156 /* convert mySourceChar+0x80 to use a normal 8-bit table */
2157 targetUniChar =
2158 _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(
2159 myData->myConverterArray[cs],
2160 mySourceChar + 0x80);
2161 }
2162 /* return from a single-shift state to the previous one */
2163 pToU2022State->g=pToU2022State->prevG;
2164 break;
2165 case JISX201:
2166 if(mySourceChar <= 0x7f) {
2167 targetUniChar = jisx201ToU(mySourceChar);
2168 }
2169 break;
2170 case HWKANA_7BIT:
2171 if((uint8_t)(mySourceChar - 0x21) <= (0x5f - 0x21)) {
2172 /* 7-bit halfwidth Katakana */
2173 targetUniChar = mySourceChar + (HWKANA_START - 0x21);
2174 }
2175 break;
2176 default:
2177 /* G0 DBCS */
2178 if(mySource < mySourceLimit) {
2179 int leadIsOk, trailIsOk;
2180 uint8_t trailByte;
2181 getTrailByte:
2182 trailByte = (uint8_t)*mySource;
2183 /*
2184 * Ticket 5691: consistent illegal sequences:
2185 * - We include at least the first byte in the illegal sequence.
2186 * - If any of the non-initial bytes could be the start of a character,
2187 * we stop the illegal sequence before the first one of those.
2188 *
2189 * In ISO-2022 DBCS, if the second byte is in the 21..7e range or is
2190 * an ESC/SO/SI, we report only the first byte as the illegal sequence.
2191 * Otherwise we convert or report the pair of bytes.
2192 */
2193 leadIsOk = (uint8_t)(mySourceChar - 0x21) <= (0x7e - 0x21);
2194 trailIsOk = (uint8_t)(trailByte - 0x21) <= (0x7e - 0x21);
2195 if (leadIsOk && trailIsOk) {
2196 ++mySource;
2197 tmpSourceChar = (mySourceChar << 8) | trailByte;
2198 if(cs == JISX208) {
2199 _2022ToSJIS((uint8_t)mySourceChar, trailByte, tempBuf);
2200 mySourceChar = tmpSourceChar;
2201 } else {
2202 /* Copy before we modify tmpSourceChar so toUnicodeCallback() sees the correct bytes. */
2203 mySourceChar = tmpSourceChar;
2204 if (cs == KSC5601) {
2205 tmpSourceChar += 0x8080; /* = _2022ToGR94DBCS(tmpSourceChar) */
2206 }
2207 tempBuf[0] = (char)(tmpSourceChar >> 8);
2208 tempBuf[1] = (char)(tmpSourceChar);
2209 }
2210 targetUniChar = ucnv_MBCSSimpleGetNextUChar(myData->myConverterArray[cs], tempBuf, 2, FALSE);
2211 } else if (!(trailIsOk || IS_2022_CONTROL(trailByte))) {
2212 /* report a pair of illegal bytes if the second byte is not a DBCS starter */
2213 ++mySource;
2214 /* add another bit so that the code below writes 2 bytes in case of error */
2215 mySourceChar = 0x10000 | (mySourceChar << 8) | trailByte;
2216 }
2217 } else {
2218 args->converter->toUBytes[0] = (uint8_t)mySourceChar;
2219 args->converter->toULength = 1;
2220 goto endloop;
2221 }
2222 } /* End of inner switch */
2223 break;
2224 } /* End of outer switch */
2225 if(targetUniChar < (missingCharMarker-1/*0xfffe*/)){
2226 if(args->offsets){
2227 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
2228 }
2229 *(myTarget++)=(UChar)targetUniChar;
2230 }
2231 else if(targetUniChar > missingCharMarker){
2232 /* disassemble the surrogate pair and write to output*/
2233 targetUniChar-=0x0010000;
2234 *myTarget = (UChar)(0xd800+(UChar)(targetUniChar>>10));
2235 if(args->offsets){
2236 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
2237 }
2238 ++myTarget;
2239 if(myTarget< args->targetLimit){
2240 *myTarget = (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff));
2241 if(args->offsets){
2242 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
2243 }
2244 ++myTarget;
2245 }else{
2246 args->converter->UCharErrorBuffer[args->converter->UCharErrorBufferLength++]=
2247 (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff));
2248 }
2249
2250 }
2251 else{
2252 /* Call the callback function*/
2253 toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err);
2254 break;
2255 }
2256 }
2257 else{ /* goes with "if(myTarget < args->targetLimit)" way up near top of function */
2258 *err =U_BUFFER_OVERFLOW_ERROR;
2259 break;
2260 }
2261 }
2262 endloop:
2263 args->target = myTarget;
2264 args->source = mySource;
2265 }
2266
2267
2268 /***************************************************************
2269 * Rules for ISO-2022-KR encoding
2270 * i) The KSC5601 designator sequence should appear only once in a file,
2271 * at the begining of a line before any KSC5601 characters. This usually
2272 * means that it appears by itself on the first line of the file
2273 * ii) There are only 2 shifting sequences SO to shift into double byte mode
2274 * and SI to shift into single byte mode
2275 */
2276 static void
2277 UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(UConverterFromUnicodeArgs* args, UErrorCode* err){
2278
2279 UConverter* saveConv = args->converter;
2280 UConverterDataISO2022 *myConverterData=(UConverterDataISO2022*)saveConv->extraInfo;
2281 args->converter=myConverterData->currentConverter;
2282
2283 myConverterData->currentConverter->fromUChar32 = saveConv->fromUChar32;
2284 ucnv_MBCSFromUnicodeWithOffsets(args,err);
2285 saveConv->fromUChar32 = myConverterData->currentConverter->fromUChar32;
2286
2287 if(*err == U_BUFFER_OVERFLOW_ERROR) {
2288 if(myConverterData->currentConverter->charErrorBufferLength > 0) {
2289 uprv_memcpy(
2290 saveConv->charErrorBuffer,
2291 myConverterData->currentConverter->charErrorBuffer,
2292 myConverterData->currentConverter->charErrorBufferLength);
2293 }
2294 saveConv->charErrorBufferLength = myConverterData->currentConverter->charErrorBufferLength;
2295 myConverterData->currentConverter->charErrorBufferLength = 0;
2296 }
2297 args->converter=saveConv;
2298 }
2299
2300 static void
2301 UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC(UConverterFromUnicodeArgs* args, UErrorCode* err){
2302
2303 const UChar *source = args->source;
2304 const UChar *sourceLimit = args->sourceLimit;
2305 unsigned char *target = (unsigned char *) args->target;
2306 unsigned char *targetLimit = (unsigned char *) args->targetLimit;
2307 int32_t* offsets = args->offsets;
2308 uint32_t targetByteUnit = 0x0000;
2309 UChar32 sourceChar = 0x0000;
2310 UBool isTargetByteDBCS;
2311 UBool oldIsTargetByteDBCS;
2312 UConverterDataISO2022 *converterData;
2313 UConverterSharedData* sharedData;
2314 UBool useFallback;
2315 int32_t length =0;
2316
2317 converterData=(UConverterDataISO2022*)args->converter->extraInfo;
2318 /* if the version is 1 then the user is requesting
2319 * conversion with ibm-25546 pass the arguments to
2320 * MBCS converter and return
2321 */
2322 if(converterData->version==1){
2323 UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(args,err);
2324 return;
2325 }
2326
2327 /* initialize data */
2328 sharedData = converterData->currentConverter->sharedData;
2329 useFallback = args->converter->useFallback;
2330 isTargetByteDBCS=(UBool)args->converter->fromUnicodeStatus;
2331 oldIsTargetByteDBCS = isTargetByteDBCS;
2332
2333 isTargetByteDBCS = (UBool) args->converter->fromUnicodeStatus;
2334 if((sourceChar = args->converter->fromUChar32)!=0 && target <targetLimit) {
2335 goto getTrail;
2336 }
2337 while(source < sourceLimit){
2338
2339 targetByteUnit = missingCharMarker;
2340
2341 if(target < (unsigned char*) args->targetLimit){
2342 sourceChar = *source++;
2343
2344 /* do not convert SO/SI/ESC */
2345 if(IS_2022_CONTROL(sourceChar)) {
2346 /* callback(illegal) */
2347 *err=U_ILLEGAL_CHAR_FOUND;
2348 args->converter->fromUChar32=sourceChar;
2349 break;
2350 }
2351
2352 length = MBCS_FROM_UCHAR32_ISO2022(sharedData,sourceChar,&targetByteUnit,useFallback,MBCS_OUTPUT_2);
2353 if(length < 0) {
2354 length = -length; /* fallback */
2355 }
2356 /* only DBCS or SBCS characters are expected*/
2357 /* DB characters with high bit set to 1 are expected */
2358 if( length > 2 || length==0 ||
2359 (length == 1 && targetByteUnit > 0x7f) ||
2360 (length == 2 &&
2361 ((uint16_t)(targetByteUnit - 0xa1a1) > (0xfefe - 0xa1a1) ||
2362 (uint8_t)(targetByteUnit - 0xa1) > (0xfe - 0xa1)))
2363 ) {
2364 targetByteUnit=missingCharMarker;
2365 }
2366 if (targetByteUnit != missingCharMarker){
2367
2368 oldIsTargetByteDBCS = isTargetByteDBCS;
2369 isTargetByteDBCS = (UBool)(targetByteUnit>0x00FF);
2370 /* append the shift sequence */
2371 if (oldIsTargetByteDBCS != isTargetByteDBCS ){
2372
2373 if (isTargetByteDBCS)
2374 *target++ = UCNV_SO;
2375 else
2376 *target++ = UCNV_SI;
2377 if(offsets)
2378 *(offsets++) = (int32_t)(source - args->source-1);
2379 }
2380 /* write the targetUniChar to target */
2381 if(targetByteUnit <= 0x00FF){
2382 if( target < targetLimit){
2383 *(target++) = (unsigned char) targetByteUnit;
2384 if(offsets){
2385 *(offsets++) = (int32_t)(source - args->source-1);
2386 }
2387
2388 }else{
2389 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) (targetByteUnit);
2390 *err = U_BUFFER_OVERFLOW_ERROR;
2391 }
2392 }else{
2393 if(target < targetLimit){
2394 *(target++) =(unsigned char) ((targetByteUnit>>8) -0x80);
2395 if(offsets){
2396 *(offsets++) = (int32_t)(source - args->source-1);
2397 }
2398 if(target < targetLimit){
2399 *(target++) =(unsigned char) (targetByteUnit -0x80);
2400 if(offsets){
2401 *(offsets++) = (int32_t)(source - args->source-1);
2402 }
2403 }else{
2404 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) (targetByteUnit -0x80);
2405 *err = U_BUFFER_OVERFLOW_ERROR;
2406 }
2407 }else{
2408 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) ((targetByteUnit>>8) -0x80);
2409 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = (unsigned char) (targetByteUnit-0x80);
2410 *err = U_BUFFER_OVERFLOW_ERROR;
2411 }
2412 }
2413
2414 }
2415 else{
2416 /* oops.. the code point is unassingned
2417 * set the error and reason
2418 */
2419
2420 /*check if the char is a First surrogate*/
2421 if(UTF_IS_SURROGATE(sourceChar)) {
2422 if(UTF_IS_SURROGATE_FIRST(sourceChar)) {
2423 getTrail:
2424 /*look ahead to find the trail surrogate*/
2425 if(source < sourceLimit) {
2426 /* test the following code unit */
2427 UChar trail=(UChar) *source;
2428 if(UTF_IS_SECOND_SURROGATE(trail)) {
2429 source++;
2430 sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail);
2431 *err = U_INVALID_CHAR_FOUND;
2432 /* convert this surrogate code point */
2433 /* exit this condition tree */
2434 } else {
2435 /* this is an unmatched lead code unit (1st surrogate) */
2436 /* callback(illegal) */
2437 *err=U_ILLEGAL_CHAR_FOUND;
2438 }
2439 } else {
2440 /* no more input */
2441 *err = U_ZERO_ERROR;
2442 }
2443 } else {
2444 /* this is an unmatched trail code unit (2nd surrogate) */
2445 /* callback(illegal) */
2446 *err=U_ILLEGAL_CHAR_FOUND;
2447 }
2448 } else {
2449 /* callback(unassigned) for a BMP code point */
2450 *err = U_INVALID_CHAR_FOUND;
2451 }
2452
2453 args->converter->fromUChar32=sourceChar;
2454 break;
2455 }
2456 } /* end if(myTargetIndex<myTargetLength) */
2457 else{
2458 *err =U_BUFFER_OVERFLOW_ERROR;
2459 break;
2460 }
2461
2462 }/* end while(mySourceIndex<mySourceLength) */
2463
2464 /*
2465 * the end of the input stream and detection of truncated input
2466 * are handled by the framework, but for ISO-2022-KR conversion
2467 * we need to be in ASCII mode at the very end
2468 *
2469 * conditions:
2470 * successful
2471 * not in ASCII mode
2472 * end of input and no truncated input
2473 */
2474 if( U_SUCCESS(*err) &&
2475 isTargetByteDBCS &&
2476 args->flush && source>=sourceLimit && args->converter->fromUChar32==0
2477 ) {
2478 int32_t sourceIndex;
2479
2480 /* we are switching to ASCII */
2481 isTargetByteDBCS=FALSE;
2482
2483 /* get the source index of the last input character */
2484 /*
2485 * TODO this would be simpler and more reliable if we used a pair
2486 * of sourceIndex/prevSourceIndex like in ucnvmbcs.c
2487 * so that we could simply use the prevSourceIndex here;
2488 * this code gives an incorrect result for the rare case of an unmatched
2489 * trail surrogate that is alone in the last buffer of the text stream
2490 */
2491 sourceIndex=(int32_t)(source-args->source);
2492 if(sourceIndex>0) {
2493 --sourceIndex;
2494 if( U16_IS_TRAIL(args->source[sourceIndex]) &&
2495 (sourceIndex==0 || U16_IS_LEAD(args->source[sourceIndex-1]))
2496 ) {
2497 --sourceIndex;
2498 }
2499 } else {
2500 sourceIndex=-1;
2501 }
2502
2503 fromUWriteUInt8(
2504 args->converter,
2505 SHIFT_IN_STR, 1,
2506 &target, (const char *)targetLimit,
2507 &offsets, sourceIndex,
2508 err);
2509 }
2510
2511 /*save the state and return */
2512 args->source = source;
2513 args->target = (char*)target;
2514 args->converter->fromUnicodeStatus = (uint32_t)isTargetByteDBCS;
2515 }
2516
2517 /************************ To Unicode ***************************************/
2518
2519 static void
2520 UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(UConverterToUnicodeArgs *args,
2521 UErrorCode* err){
2522 char const* sourceStart;
2523 UConverterDataISO2022* myData=(UConverterDataISO2022*)(args->converter->extraInfo);
2524
2525 UConverterToUnicodeArgs subArgs;
2526 int32_t minArgsSize;
2527
2528 /* set up the subconverter arguments */
2529 if(args->size<sizeof(UConverterToUnicodeArgs)) {
2530 minArgsSize = args->size;
2531 } else {
2532 minArgsSize = (int32_t)sizeof(UConverterToUnicodeArgs);
2533 }
2534
2535 uprv_memcpy(&subArgs, args, minArgsSize);
2536 subArgs.size = (uint16_t)minArgsSize;
2537 subArgs.converter = myData->currentConverter;
2538
2539 /* remember the original start of the input for offsets */
2540 sourceStart = args->source;
2541
2542 if(myData->key != 0) {
2543 /* continue with a partial escape sequence */
2544 goto escape;
2545 }
2546
2547 while(U_SUCCESS(*err) && args->source < args->sourceLimit) {
2548 /*Find the end of the buffer e.g : Next Escape Seq | end of Buffer*/
2549 subArgs.source = args->source;
2550 subArgs.sourceLimit = getEndOfBuffer_2022(&(args->source), args->sourceLimit, args->flush);
2551 if(subArgs.source != subArgs.sourceLimit) {
2552 /*
2553 * get the current partial byte sequence
2554 *
2555 * it needs to be moved between the public and the subconverter
2556 * so that the conversion framework, which only sees the public
2557 * converter, can handle truncated and illegal input etc.
2558 */
2559 if(args->converter->toULength > 0) {
2560 uprv_memcpy(subArgs.converter->toUBytes, args->converter->toUBytes, args->converter->toULength);
2561 }
2562 subArgs.converter->toULength = args->converter->toULength;
2563
2564 /*
2565 * Convert up to the end of the input, or to before the next escape character.
2566 * Does not handle conversion extensions because the preToU[] state etc.
2567 * is not copied.
2568 */
2569 ucnv_MBCSToUnicodeWithOffsets(&subArgs, err);
2570
2571 if(args->offsets != NULL && sourceStart != args->source) {
2572 /* update offsets to base them on the actual start of the input */
2573 int32_t *offsets = args->offsets;
2574 UChar *target = args->target;
2575 int32_t delta = (int32_t)(args->source - sourceStart);
2576 while(target < subArgs.target) {
2577 if(*offsets >= 0) {
2578 *offsets += delta;
2579 }
2580 ++offsets;
2581 ++target;
2582 }
2583 }
2584 args->source = subArgs.source;
2585 args->target = subArgs.target;
2586 args->offsets = subArgs.offsets;
2587
2588 /* copy input/error/overflow buffers */
2589 if(subArgs.converter->toULength > 0) {
2590 uprv_memcpy(args->converter->toUBytes, subArgs.converter->toUBytes, subArgs.converter->toULength);
2591 }
2592 args->converter->toULength = subArgs.converter->toULength;
2593
2594 if(*err == U_BUFFER_OVERFLOW_ERROR) {
2595 if(subArgs.converter->UCharErrorBufferLength > 0) {
2596 uprv_memcpy(args->converter->UCharErrorBuffer, subArgs.converter->UCharErrorBuffer,
2597 subArgs.converter->UCharErrorBufferLength);
2598 }
2599 args->converter->UCharErrorBufferLength=subArgs.converter->UCharErrorBufferLength;
2600 subArgs.converter->UCharErrorBufferLength = 0;
2601 }
2602 }
2603
2604 if (U_FAILURE(*err) || (args->source == args->sourceLimit)) {
2605 return;
2606 }
2607
2608 escape:
2609 changeState_2022(args->converter,
2610 &(args->source),
2611 args->sourceLimit,
2612 ISO_2022_KR,
2613 err);
2614 }
2615 }
2616
2617 static void
2618 UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC(UConverterToUnicodeArgs *args,
2619 UErrorCode* err){
2620 char tempBuf[2];
2621 const char *mySource = ( char *) args->source;
2622 UChar *myTarget = args->target;
2623 const char *mySourceLimit = args->sourceLimit;
2624 UChar32 targetUniChar = 0x0000;
2625 UChar mySourceChar = 0x0000;
2626 UConverterDataISO2022* myData;
2627 UConverterSharedData* sharedData ;
2628 UBool useFallback;
2629
2630 myData=(UConverterDataISO2022*)(args->converter->extraInfo);
2631 if(myData->version==1){
2632 UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC_IBM(args,err);
2633 return;
2634 }
2635
2636 /* initialize state */
2637 sharedData = myData->currentConverter->sharedData;
2638 useFallback = args->converter->useFallback;
2639
2640 if(myData->key != 0) {
2641 /* continue with a partial escape sequence */
2642 goto escape;
2643 } else if(args->converter->toULength == 1 && mySource < mySourceLimit && myTarget < args->targetLimit) {
2644 /* continue with a partial double-byte character */
2645 mySourceChar = args->converter->toUBytes[0];
2646 args->converter->toULength = 0;
2647 goto getTrailByte;
2648 }
2649
2650 while(mySource< mySourceLimit){
2651
2652 if(myTarget < args->targetLimit){
2653
2654 mySourceChar= (unsigned char) *mySource++;
2655
2656 if(mySourceChar==UCNV_SI){
2657 myData->toU2022State.g = 0;
2658 if (myData->isEmptySegment) {
2659 myData->isEmptySegment = FALSE; /* we are handling it, reset to avoid future spurious errors */
2660 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
2661 args->converter->toUCallbackReason = UCNV_IRREGULAR;
2662 args->converter->toUBytes[0] = (uint8_t)mySourceChar;
2663 args->converter->toULength = 1;
2664 args->target = myTarget;
2665 args->source = mySource;
2666 return;
2667 }
2668 /*consume the source */
2669 continue;
2670 }else if(mySourceChar==UCNV_SO){
2671 myData->toU2022State.g = 1;
2672 myData->isEmptySegment = TRUE; /* Begin a new segment, empty so far */
2673 /*consume the source */
2674 continue;
2675 }else if(mySourceChar==ESC_2022){
2676 mySource--;
2677 escape:
2678 myData->isEmptySegment = FALSE; /* Any invalid ESC sequences will be detected separately, so just reset this */
2679 changeState_2022(args->converter,&(mySource),
2680 mySourceLimit, ISO_2022_KR, err);
2681 if(U_FAILURE(*err)){
2682 args->target = myTarget;
2683 args->source = mySource;
2684 return;
2685 }
2686 continue;
2687 }
2688
2689 myData->isEmptySegment = FALSE; /* Any invalid char errors will be detected separately, so just reset this */
2690 if(myData->toU2022State.g == 1) {
2691 if(mySource < mySourceLimit) {
2692 int leadIsOk, trailIsOk;
2693 uint8_t trailByte;
2694 getTrailByte:
2695 targetUniChar = missingCharMarker;
2696 trailByte = (uint8_t)*mySource;
2697 /*
2698 * Ticket 5691: consistent illegal sequences:
2699 * - We include at least the first byte in the illegal sequence.
2700 * - If any of the non-initial bytes could be the start of a character,
2701 * we stop the illegal sequence before the first one of those.
2702 *
2703 * In ISO-2022 DBCS, if the second byte is in the 21..7e range or is
2704 * an ESC/SO/SI, we report only the first byte as the illegal sequence.
2705 * Otherwise we convert or report the pair of bytes.
2706 */
2707 leadIsOk = (uint8_t)(mySourceChar - 0x21) <= (0x7e - 0x21);
2708 trailIsOk = (uint8_t)(trailByte - 0x21) <= (0x7e - 0x21);
2709 if (leadIsOk && trailIsOk) {
2710 ++mySource;
2711 tempBuf[0] = (char)(mySourceChar + 0x80);
2712 tempBuf[1] = (char)(trailByte + 0x80);
2713 targetUniChar = ucnv_MBCSSimpleGetNextUChar(sharedData, tempBuf, 2, useFallback);
2714 mySourceChar = (mySourceChar << 8) | trailByte;
2715 } else if (!(trailIsOk || IS_2022_CONTROL(trailByte))) {
2716 /* report a pair of illegal bytes if the second byte is not a DBCS starter */
2717 ++mySource;
2718 /* add another bit so that the code below writes 2 bytes in case of error */
2719 mySourceChar = 0x10000 | (mySourceChar << 8) | trailByte;
2720 }
2721 } else {
2722 args->converter->toUBytes[0] = (uint8_t)mySourceChar;
2723 args->converter->toULength = 1;
2724 break;
2725 }
2726 }
2727 else if(mySourceChar <= 0x7f) {
2728 targetUniChar = ucnv_MBCSSimpleGetNextUChar(sharedData, mySource - 1, 1, useFallback);
2729 } else {
2730 targetUniChar = 0xffff;
2731 }
2732 if(targetUniChar < 0xfffe){
2733 if(args->offsets) {
2734 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
2735 }
2736 *(myTarget++)=(UChar)targetUniChar;
2737 }
2738 else {
2739 /* Call the callback function*/
2740 toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err);
2741 break;
2742 }
2743 }
2744 else{
2745 *err =U_BUFFER_OVERFLOW_ERROR;
2746 break;
2747 }
2748 }
2749 args->target = myTarget;
2750 args->source = mySource;
2751 }
2752
2753 /*************************** END ISO2022-KR *********************************/
2754
2755 /*************************** ISO-2022-CN *********************************
2756 *
2757 * Rules for ISO-2022-CN Encoding:
2758 * i) The designator sequence must appear once on a line before any instance
2759 * of character set it designates.
2760 * ii) If two lines contain characters from the same character set, both lines
2761 * must include the designator sequence.
2762 * iii) Once the designator sequence is known, a shifting sequence has to be found
2763 * to invoke the shifting
2764 * iv) All lines start in ASCII and end in ASCII.
2765 * v) Four shifting sequences are employed for this purpose:
2766 *
2767 * Sequcence ASCII Eq Charsets
2768 * ---------- ------- ---------
2769 * SI <SI> US-ASCII
2770 * SO <SO> CNS-11643-1992 Plane 1, GB2312, ISO-IR-165
2771 * SS2 <ESC>N CNS-11643-1992 Plane 2
2772 * SS3 <ESC>O CNS-11643-1992 Planes 3-7
2773 *
2774 * vi)
2775 * SOdesignator : ESC "$" ")" finalchar_for_SO
2776 * SS2designator : ESC "$" "*" finalchar_for_SS2
2777 * SS3designator : ESC "$" "+" finalchar_for_SS3
2778 *
2779 * ESC $ ) A Indicates the bytes following SO are Chinese
2780 * characters as defined in GB 2312-80, until
2781 * another SOdesignation appears
2782 *
2783 *
2784 * ESC $ ) E Indicates the bytes following SO are as defined
2785 * in ISO-IR-165 (for details, see section 2.1),
2786 * until another SOdesignation appears
2787 *
2788 * ESC $ ) G Indicates the bytes following SO are as defined
2789 * in CNS 11643-plane-1, until another
2790 * SOdesignation appears
2791 *
2792 * ESC $ * H Indicates the two bytes immediately following
2793 * SS2 is a Chinese character as defined in CNS
2794 * 11643-plane-2, until another SS2designation
2795 * appears
2796 * (Meaning <ESC>N must preceed every 2 byte
2797 * sequence.)
2798 *
2799 * ESC $ + I Indicates the immediate two bytes following SS3
2800 * is a Chinese character as defined in CNS
2801 * 11643-plane-3, until another SS3designation
2802 * appears
2803 * (Meaning <ESC>O must preceed every 2 byte
2804 * sequence.)
2805 *
2806 * ESC $ + J Indicates the immediate two bytes following SS3
2807 * is a Chinese character as defined in CNS
2808 * 11643-plane-4, until another SS3designation
2809 * appears
2810 * (In English: <ESC>O must preceed every 2 byte
2811 * sequence.)
2812 *
2813 * ESC $ + K Indicates the immediate two bytes following SS3
2814 * is a Chinese character as defined in CNS
2815 * 11643-plane-5, until another SS3designation
2816 * appears
2817 *
2818 * ESC $ + L Indicates the immediate two bytes following SS3
2819 * is a Chinese character as defined in CNS
2820 * 11643-plane-6, until another SS3designation
2821 * appears
2822 *
2823 * ESC $ + M Indicates the immediate two bytes following SS3
2824 * is a Chinese character as defined in CNS
2825 * 11643-plane-7, until another SS3designation
2826 * appears
2827 *
2828 * As in ISO-2022-CN, each line starts in ASCII, and ends in ASCII, and
2829 * has its own designation information before any Chinese characters
2830 * appear
2831 *
2832 */
2833
2834 /* The following are defined this way to make the strings truely readonly */
2835 static const char GB_2312_80_STR[] = "\x1B\x24\x29\x41";
2836 static const char ISO_IR_165_STR[] = "\x1B\x24\x29\x45";
2837 static const char CNS_11643_1992_Plane_1_STR[] = "\x1B\x24\x29\x47";
2838 static const char CNS_11643_1992_Plane_2_STR[] = "\x1B\x24\x2A\x48";
2839 static const char CNS_11643_1992_Plane_3_STR[] = "\x1B\x24\x2B\x49";
2840 static const char CNS_11643_1992_Plane_4_STR[] = "\x1B\x24\x2B\x4A";
2841 static const char CNS_11643_1992_Plane_5_STR[] = "\x1B\x24\x2B\x4B";
2842 static const char CNS_11643_1992_Plane_6_STR[] = "\x1B\x24\x2B\x4C";
2843 static const char CNS_11643_1992_Plane_7_STR[] = "\x1B\x24\x2B\x4D";
2844
2845 /********************** ISO2022-CN Data **************************/
2846 static const char* const escSeqCharsCN[10] ={
2847 SHIFT_IN_STR, /* ASCII */
2848 GB_2312_80_STR,
2849 ISO_IR_165_STR,
2850 CNS_11643_1992_Plane_1_STR,
2851 CNS_11643_1992_Plane_2_STR,
2852 CNS_11643_1992_Plane_3_STR,
2853 CNS_11643_1992_Plane_4_STR,
2854 CNS_11643_1992_Plane_5_STR,
2855 CNS_11643_1992_Plane_6_STR,
2856 CNS_11643_1992_Plane_7_STR
2857 };
2858
2859 static void
2860 UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC(UConverterFromUnicodeArgs* args, UErrorCode* err){
2861 UConverter *cnv = args->converter;
2862 UConverterDataISO2022 *converterData;
2863 ISO2022State *pFromU2022State;
2864 uint8_t *target = (uint8_t *) args->target;
2865 const uint8_t *targetLimit = (const uint8_t *) args->targetLimit;
2866 const UChar* source = args->source;
2867 const UChar* sourceLimit = args->sourceLimit;
2868 int32_t* offsets = args->offsets;
2869 UChar32 sourceChar;
2870 char buffer[8];
2871 int32_t len;
2872 int8_t choices[3];
2873 int32_t choiceCount;
2874 uint32_t targetValue = 0;
2875 UBool useFallback;
2876
2877 /* set up the state */
2878 converterData = (UConverterDataISO2022*)cnv->extraInfo;
2879 pFromU2022State = &converterData->fromU2022State;
2880
2881 choiceCount = 0;
2882
2883 /* check if the last codepoint of previous buffer was a lead surrogate*/
2884 if((sourceChar = cnv->fromUChar32)!=0 && target< targetLimit) {
2885 goto getTrail;
2886 }
2887
2888 while( source < sourceLimit){
2889 if(target < targetLimit){
2890
2891 sourceChar = *(source++);
2892 /*check if the char is a First surrogate*/
2893 if(UTF_IS_SURROGATE(sourceChar)) {
2894 if(UTF_IS_SURROGATE_FIRST(sourceChar)) {
2895 getTrail:
2896 /*look ahead to find the trail surrogate*/
2897 if(source < sourceLimit) {
2898 /* test the following code unit */
2899 UChar trail=(UChar) *source;
2900 if(UTF_IS_SECOND_SURROGATE(trail)) {
2901 source++;
2902 sourceChar=UTF16_GET_PAIR_VALUE(sourceChar, trail);
2903 cnv->fromUChar32=0x00;
2904 /* convert this supplementary code point */
2905 /* exit this condition tree */
2906 } else {
2907 /* this is an unmatched lead code unit (1st surrogate) */
2908 /* callback(illegal) */
2909 *err=U_ILLEGAL_CHAR_FOUND;
2910 cnv->fromUChar32=sourceChar;
2911 break;
2912 }
2913 } else {
2914 /* no more input */
2915 cnv->fromUChar32=sourceChar;
2916 break;
2917 }
2918 } else {
2919 /* this is an unmatched trail code unit (2nd surrogate) */
2920 /* callback(illegal) */
2921 *err=U_ILLEGAL_CHAR_FOUND;
2922 cnv->fromUChar32=sourceChar;
2923 break;
2924 }
2925 }
2926
2927 /* do the conversion */
2928 if(sourceChar <= 0x007f ){
2929 /* do not convert SO/SI/ESC */
2930 if(IS_2022_CONTROL(sourceChar)) {
2931 /* callback(illegal) */
2932 *err=U_ILLEGAL_CHAR_FOUND;
2933 cnv->fromUChar32=sourceChar;
2934 break;
2935 }
2936
2937 /* US-ASCII */
2938 if(pFromU2022State->g == 0) {
2939 buffer[0] = (char)sourceChar;
2940 len = 1;
2941 } else {
2942 buffer[0] = UCNV_SI;
2943 buffer[1] = (char)sourceChar;
2944 len = 2;
2945 pFromU2022State->g = 0;
2946 choiceCount = 0;
2947 }
2948 if(sourceChar == CR || sourceChar == LF) {
2949 /* reset the state at the end of a line */
2950 uprv_memset(pFromU2022State, 0, sizeof(ISO2022State));
2951 choiceCount = 0;
2952 }
2953 }
2954 else{
2955 /* convert U+0080..U+10ffff */
2956 int32_t i;
2957 int8_t cs, g;
2958
2959 if(choiceCount == 0) {
2960 /* try the current SO/G1 converter first */
2961 choices[0] = pFromU2022State->cs[1];
2962
2963 /* default to GB2312_1 if none is designated yet */
2964 if(choices[0] == 0) {
2965 choices[0] = GB2312_1;
2966 }
2967
2968 if(converterData->version == 0) {
2969 /* ISO-2022-CN */
2970
2971 /* try the other SO/G1 converter; a CNS_11643_1 lookup may result in any plane */
2972 if(choices[0] == GB2312_1) {
2973 choices[1] = (int8_t)CNS_11643_1;
2974 } else {
2975 choices[1] = (int8_t)GB2312_1;
2976 }
2977
2978 choiceCount = 2;
2979 } else if (converterData->version == 1) {
2980 /* ISO-2022-CN-EXT */
2981
2982 /* try one of the other converters */
2983 switch(choices[0]) {
2984 case GB2312_1:
2985 choices[1] = (int8_t)CNS_11643_1;
2986 choices[2] = (int8_t)ISO_IR_165;
2987 break;
2988 case ISO_IR_165:
2989 choices[1] = (int8_t)GB2312_1;
2990 choices[2] = (int8_t)CNS_11643_1;
2991 break;
2992 default: /* CNS_11643_x */
2993 choices[1] = (int8_t)GB2312_1;
2994 choices[2] = (int8_t)ISO_IR_165;
2995 break;
2996 }
2997
2998 choiceCount = 3;
2999 } else {
3000 choices[0] = (int8_t)CNS_11643_1;
3001 choices[1] = (int8_t)GB2312_1;
3002 }
3003 }
3004
3005 cs = g = 0;
3006 /*
3007 * len==0: no mapping found yet
3008 * len<0: found a fallback result: continue looking for a roundtrip but no further fallbacks
3009 * len>0: found a roundtrip result, done
3010 */
3011 len = 0;
3012 /*
3013 * We will turn off useFallback after finding a fallback,
3014 * but we still get fallbacks from PUA code points as usual.
3015 * Therefore, we will also need to check that we don't overwrite
3016 * an early fallback with a later one.
3017 */
3018 useFallback = cnv->useFallback;
3019
3020 for(i = 0; i < choiceCount && len <= 0; ++i) {
3021 int8_t cs0 = choices[i];
3022 if(cs0 > 0) {
3023 uint32_t value;
3024 int32_t len2;
3025 if(cs0 >= CNS_11643_0) {
3026 len2 = MBCS_FROM_UCHAR32_ISO2022(
3027 converterData->myConverterArray[CNS_11643],
3028 sourceChar,
3029 &value,
3030 useFallback,
3031 MBCS_OUTPUT_3);
3032 if(len2 == 3 || (len2 == -3 && len == 0)) {
3033 targetValue = value;
3034 cs = (int8_t)(CNS_11643_0 + (value >> 16) - 0x80);
3035 if(len2 >= 0) {
3036 len = 2;
3037 } else {
3038 len = -2;
3039 useFallback = FALSE;
3040 }
3041 if(cs == CNS_11643_1) {
3042 g = 1;
3043 } else if(cs == CNS_11643_2) {
3044 g = 2;
3045 } else /* plane 3..7 */ if(converterData->version == 1) {
3046 g = 3;
3047 } else {
3048 /* ISO-2022-CN (without -EXT) does not support plane 3..7 */
3049 len = 0;
3050 }
3051 }
3052 } else {
3053 /* GB2312_1 or ISO-IR-165 */
3054 len2 = MBCS_FROM_UCHAR32_ISO2022(
3055 converterData->myConverterArray[cs0],
3056 sourceChar,
3057 &value,
3058 useFallback,
3059 MBCS_OUTPUT_2);
3060 if(len2 == 2 || (len2 == -2 && len == 0)) {
3061 targetValue = value;
3062 len = len2;
3063 cs = cs0;
3064 g = 1;
3065 useFallback = FALSE;
3066 }
3067 }
3068 }
3069 }
3070
3071 if(len != 0) {
3072 len = 0; /* count output bytes; it must have been abs(len) == 2 */
3073
3074 /* write the designation sequence if necessary */
3075 if(cs != pFromU2022State->cs[g]) {
3076 if(cs < CNS_11643) {
3077 uprv_memcpy(buffer, escSeqCharsCN[cs], 4);
3078 } else {
3079 uprv_memcpy(buffer, escSeqCharsCN[CNS_11643 + (cs - CNS_11643_1)], 4);
3080 }
3081 len = 4;
3082 pFromU2022State->cs[g] = cs;
3083 if(g == 1) {
3084 /* changing the SO/G1 charset invalidates the choices[] */
3085 choiceCount = 0;
3086 }
3087 }
3088
3089 /* write the shift sequence if necessary */
3090 if(g != pFromU2022State->g) {
3091 switch(g) {
3092 case 1:
3093 buffer[len++] = UCNV_SO;
3094
3095 /* set the new state only if it is the locking shift SO/G1, not for SS2 or SS3 */
3096 pFromU2022State->g = 1;
3097 break;
3098 case 2:
3099 buffer[len++] = 0x1b;
3100 buffer[len++] = 0x4e;
3101 break;
3102 default: /* case 3 */
3103 buffer[len++] = 0x1b;
3104 buffer[len++] = 0x4f;
3105 break;
3106 }
3107 }
3108
3109 /* write the two output bytes */
3110 buffer[len++] = (char)(targetValue >> 8);
3111 buffer[len++] = (char)targetValue;
3112 } else {
3113 /* if we cannot find the character after checking all codepages
3114 * then this is an error
3115 */
3116 *err = U_INVALID_CHAR_FOUND;
3117 cnv->fromUChar32=sourceChar;
3118 break;
3119 }
3120 }
3121
3122 /* output len>0 bytes in buffer[] */
3123 if(len == 1) {
3124 *target++ = buffer[0];
3125 if(offsets) {
3126 *offsets++ = (int32_t)(source - args->source - 1); /* -1: known to be ASCII */
3127 }
3128 } else if(len == 2 && (target + 2) <= targetLimit) {
3129 *target++ = buffer[0];
3130 *target++ = buffer[1];
3131 if(offsets) {
3132 int32_t sourceIndex = (int32_t)(source - args->source - U16_LENGTH(sourceChar));
3133 *offsets++ = sourceIndex;
3134 *offsets++ = sourceIndex;
3135 }
3136 } else {
3137 fromUWriteUInt8(
3138 cnv,
3139 buffer, len,
3140 &target, (const char *)targetLimit,
3141 &offsets, (int32_t)(source - args->source - U16_LENGTH(sourceChar)),
3142 err);
3143 if(U_FAILURE(*err)) {
3144 break;
3145 }
3146 }
3147 } /* end if(myTargetIndex<myTargetLength) */
3148 else{
3149 *err =U_BUFFER_OVERFLOW_ERROR;
3150 break;
3151 }
3152
3153 }/* end while(mySourceIndex<mySourceLength) */
3154
3155 /*
3156 * the end of the input stream and detection of truncated input
3157 * are handled by the framework, but for ISO-2022-CN conversion
3158 * we need to be in ASCII mode at the very end
3159 *
3160 * conditions:
3161 * successful
3162 * not in ASCII mode
3163 * end of input and no truncated input
3164 */
3165 if( U_SUCCESS(*err) &&
3166 pFromU2022State->g!=0 &&
3167 args->flush && source>=sourceLimit && cnv->fromUChar32==0
3168 ) {
3169 int32_t sourceIndex;
3170
3171 /* we are switching to ASCII */
3172 pFromU2022State->g=0;
3173
3174 /* get the source index of the last input character */
3175 /*
3176 * TODO this would be simpler and more reliable if we used a pair
3177 * of sourceIndex/prevSourceIndex like in ucnvmbcs.c
3178 * so that we could simply use the prevSourceIndex here;
3179 * this code gives an incorrect result for the rare case of an unmatched
3180 * trail surrogate that is alone in the last buffer of the text stream
3181 */
3182 sourceIndex=(int32_t)(source-args->source);
3183 if(sourceIndex>0) {
3184 --sourceIndex;
3185 if( U16_IS_TRAIL(args->source[sourceIndex]) &&
3186 (sourceIndex==0 || U16_IS_LEAD(args->source[sourceIndex-1]))
3187 ) {
3188 --sourceIndex;
3189 }
3190 } else {
3191 sourceIndex=-1;
3192 }
3193
3194 fromUWriteUInt8(
3195 cnv,
3196 SHIFT_IN_STR, 1,
3197 &target, (const char *)targetLimit,
3198 &offsets, sourceIndex,
3199 err);
3200 }
3201
3202 /*save the state and return */
3203 args->source = source;
3204 args->target = (char*)target;
3205 }
3206
3207
3208 static void
3209 UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC(UConverterToUnicodeArgs *args,
3210 UErrorCode* err){
3211 char tempBuf[3];
3212 const char *mySource = (char *) args->source;
3213 UChar *myTarget = args->target;
3214 const char *mySourceLimit = args->sourceLimit;
3215 uint32_t targetUniChar = 0x0000;
3216 uint32_t mySourceChar = 0x0000;
3217 UConverterDataISO2022* myData;
3218 ISO2022State *pToU2022State;
3219
3220 myData=(UConverterDataISO2022*)(args->converter->extraInfo);
3221 pToU2022State = &myData->toU2022State;
3222
3223 if(myData->key != 0) {
3224 /* continue with a partial escape sequence */
3225 goto escape;
3226 } else if(args->converter->toULength == 1 && mySource < mySourceLimit && myTarget < args->targetLimit) {
3227 /* continue with a partial double-byte character */
3228 mySourceChar = args->converter->toUBytes[0];
3229 args->converter->toULength = 0;
3230 targetUniChar = missingCharMarker;
3231 goto getTrailByte;
3232 }
3233
3234 while(mySource < mySourceLimit){
3235
3236 targetUniChar =missingCharMarker;
3237
3238 if(myTarget < args->targetLimit){
3239
3240 mySourceChar= (unsigned char) *mySource++;
3241
3242 switch(mySourceChar){
3243 case UCNV_SI:
3244 pToU2022State->g=0;
3245 if (myData->isEmptySegment) {
3246 myData->isEmptySegment = FALSE; /* we are handling it, reset to avoid future spurious errors */
3247 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
3248 args->converter->toUCallbackReason = UCNV_IRREGULAR;
3249 args->converter->toUBytes[0] = mySourceChar;
3250 args->converter->toULength = 1;
3251 args->target = myTarget;
3252 args->source = mySource;
3253 return;
3254 }
3255 continue;
3256
3257 case UCNV_SO:
3258 if(pToU2022State->cs[1] != 0) {
3259 pToU2022State->g=1;
3260 myData->isEmptySegment = TRUE; /* Begin a new segment, empty so far */
3261 continue;
3262 } else {
3263 /* illegal to have SO before a matching designator */
3264 myData->isEmptySegment = FALSE; /* Handling a different error, reset this to avoid future spurious errs */
3265 break;
3266 }
3267
3268 case ESC_2022:
3269 mySource--;
3270 escape:
3271 {
3272 const char * mySourceBefore = mySource;
3273 int8_t toULengthBefore = args->converter->toULength;
3274
3275 changeState_2022(args->converter,&(mySource),
3276 mySourceLimit, ISO_2022_CN,err);
3277
3278 /* After SO there must be at least one character before a designator (designator error handled separately) */
3279 if(myData->key==0 && U_SUCCESS(*err) && myData->isEmptySegment) {
3280 *err = U_ILLEGAL_ESCAPE_SEQUENCE;
3281 args->converter->toUCallbackReason = UCNV_IRREGULAR;
3282 args->converter->toULength = (int8_t)(toULengthBefore + (mySource - mySourceBefore));
3283 }
3284 }
3285
3286 /* invalid or illegal escape sequence */
3287 if(U_FAILURE(*err)){
3288 args->target = myTarget;
3289 args->source = mySource;
3290 myData->isEmptySegment = FALSE; /* Reset to avoid future spurious errors */
3291 return;
3292 }
3293 continue;
3294
3295 /* ISO-2022-CN does not use single-byte (C1) SS2 and SS3 */
3296
3297 case CR:
3298 /*falls through*/
3299 case LF:
3300 uprv_memset(pToU2022State, 0, sizeof(ISO2022State));
3301 /* falls through */
3302 default:
3303 /* convert one or two bytes */
3304 myData->isEmptySegment = FALSE;
3305 if(pToU2022State->g != 0) {
3306 if(mySource < mySourceLimit) {
3307 UConverterSharedData *cnv;
3308 StateEnum tempState;
3309 int32_t tempBufLen;
3310 int leadIsOk, trailIsOk;
3311 uint8_t trailByte;
3312 getTrailByte:
3313 trailByte = (uint8_t)*mySource;
3314 /*
3315 * Ticket 5691: consistent illegal sequences:
3316 * - We include at least the first byte in the illegal sequence.
3317 * - If any of the non-initial bytes could be the start of a character,
3318 * we stop the illegal sequence before the first one of those.
3319 *
3320 * In ISO-2022 DBCS, if the second byte is in the 21..7e range or is
3321 * an ESC/SO/SI, we report only the first byte as the illegal sequence.
3322 * Otherwise we convert or report the pair of bytes.
3323 */
3324 leadIsOk = (uint8_t)(mySourceChar - 0x21) <= (0x7e - 0x21);
3325 trailIsOk = (uint8_t)(trailByte - 0x21) <= (0x7e - 0x21);
3326 if (leadIsOk && trailIsOk) {
3327 ++mySource;
3328 tempState = (StateEnum)pToU2022State->cs[pToU2022State->g];
3329 if(tempState >= CNS_11643_0) {
3330 cnv = myData->myConverterArray[CNS_11643];
3331 tempBuf[0] = (char) (0x80+(tempState-CNS_11643_0));
3332 tempBuf[1] = (char) (mySourceChar);
3333 tempBuf[2] = (char) trailByte;
3334 tempBufLen = 3;
3335
3336 }else{
3337 cnv = myData->myConverterArray[tempState];
3338 tempBuf[0] = (char) (mySourceChar);
3339 tempBuf[1] = (char) trailByte;
3340 tempBufLen = 2;
3341 }
3342 targetUniChar = ucnv_MBCSSimpleGetNextUChar(cnv, tempBuf, tempBufLen, FALSE);
3343 mySourceChar = (mySourceChar << 8) | trailByte;
3344 } else if (!(trailIsOk || IS_2022_CONTROL(trailByte))) {
3345 /* report a pair of illegal bytes if the second byte is not a DBCS starter */
3346 ++mySource;
3347 /* add another bit so that the code below writes 2 bytes in case of error */
3348 mySourceChar = 0x10000 | (mySourceChar << 8) | trailByte;
3349 }
3350 if(pToU2022State->g>=2) {
3351 /* return from a single-shift state to the previous one */
3352 pToU2022State->g=pToU2022State->prevG;
3353 }
3354 } else {
3355 args->converter->toUBytes[0] = (uint8_t)mySourceChar;
3356 args->converter->toULength = 1;
3357 goto endloop;
3358 }
3359 }
3360 else{
3361 if(mySourceChar <= 0x7f) {
3362 targetUniChar = (UChar) mySourceChar;
3363 }
3364 }
3365 break;
3366 }
3367 if(targetUniChar < (missingCharMarker-1/*0xfffe*/)){
3368 if(args->offsets){
3369 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
3370 }
3371 *(myTarget++)=(UChar)targetUniChar;
3372 }
3373 else if(targetUniChar > missingCharMarker){
3374 /* disassemble the surrogate pair and write to output*/
3375 targetUniChar-=0x0010000;
3376 *myTarget = (UChar)(0xd800+(UChar)(targetUniChar>>10));
3377 if(args->offsets){
3378 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
3379 }
3380 ++myTarget;
3381 if(myTarget< args->targetLimit){
3382 *myTarget = (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff));
3383 if(args->offsets){
3384 args->offsets[myTarget - args->target] = (int32_t)(mySource - args->source - (mySourceChar <= 0xff ? 1 : 2));
3385 }
3386 ++myTarget;
3387 }else{
3388 args->converter->UCharErrorBuffer[args->converter->UCharErrorBufferLength++]=
3389 (UChar)(0xdc00+(UChar)(targetUniChar&0x3ff));
3390 }
3391
3392 }
3393 else{
3394 /* Call the callback function*/
3395 toUnicodeCallback(args->converter,mySourceChar,targetUniChar,err);
3396 break;
3397 }
3398 }
3399 else{
3400 *err =U_BUFFER_OVERFLOW_ERROR;
3401 break;
3402 }
3403 }
3404 endloop:
3405 args->target = myTarget;
3406 args->source = mySource;
3407 }
3408
3409 static void
3410 _ISO_2022_WriteSub(UConverterFromUnicodeArgs *args, int32_t offsetIndex, UErrorCode *err) {
3411 UConverter *cnv = args->converter;
3412 UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) cnv->extraInfo;
3413 ISO2022State *pFromU2022State=&myConverterData->fromU2022State;
3414 char *p, *subchar;
3415 char buffer[8];
3416 int32_t length;
3417
3418 subchar=(char *)cnv->subChars;
3419 length=cnv->subCharLen; /* assume length==1 for most variants */
3420
3421 p = buffer;
3422 switch(myConverterData->locale[0]){
3423 case 'j':
3424 {
3425 int8_t cs;
3426
3427 if(pFromU2022State->g == 1) {
3428 /* JIS7: switch from G1 to G0 */
3429 pFromU2022State->g = 0;
3430 *p++ = UCNV_SI;
3431 }
3432
3433 cs = pFromU2022State->cs[0];
3434 if(cs != ASCII && cs != JISX201) {
3435 /* not in ASCII or JIS X 0201: switch to ASCII */
3436 pFromU2022State->cs[0] = (int8_t)ASCII;
3437 *p++ = '\x1b';
3438 *p++ = '\x28';
3439 *p++ = '\x42';
3440 }
3441
3442 *p++ = subchar[0];
3443 break;
3444 }
3445 case 'c':
3446 if(pFromU2022State->g != 0) {
3447 /* not in ASCII mode: switch to ASCII */
3448 pFromU2022State->g = 0;
3449 *p++ = UCNV_SI;
3450 }
3451 *p++ = subchar[0];
3452 break;
3453 case 'k':
3454 if(myConverterData->version == 0) {
3455 if(length == 1) {
3456 if((UBool)args->converter->fromUnicodeStatus) {
3457 /* in DBCS mode: switch to SBCS */
3458 args->converter->fromUnicodeStatus = 0;
3459 *p++ = UCNV_SI;
3460 }
3461 *p++ = subchar[0];
3462 } else /* length == 2*/ {
3463 if(!(UBool)args->converter->fromUnicodeStatus) {
3464 /* in SBCS mode: switch to DBCS */
3465 args->converter->fromUnicodeStatus = 1;
3466 *p++ = UCNV_SO;
3467 }
3468 *p++ = subchar[0];
3469 *p++ = subchar[1];
3470 }
3471 break;
3472 } else {
3473 /* save the subconverter's substitution string */
3474 uint8_t *currentSubChars = myConverterData->currentConverter->subChars;
3475 int8_t currentSubCharLen = myConverterData->currentConverter->subCharLen;
3476
3477 /* set our substitution string into the subconverter */
3478 myConverterData->currentConverter->subChars = (uint8_t *)subchar;
3479 myConverterData->currentConverter->subCharLen = (int8_t)length;
3480
3481 /* let the subconverter write the subchar, set/retrieve fromUChar32 state */
3482 args->converter = myConverterData->currentConverter;
3483 myConverterData->currentConverter->fromUChar32 = cnv->fromUChar32;
3484 ucnv_cbFromUWriteSub(args, 0, err);
3485 cnv->fromUChar32 = myConverterData->currentConverter->fromUChar32;
3486 args->converter = cnv;
3487
3488 /* restore the subconverter's substitution string */
3489 myConverterData->currentConverter->subChars = currentSubChars;
3490 myConverterData->currentConverter->subCharLen = currentSubCharLen;
3491
3492 if(*err == U_BUFFER_OVERFLOW_ERROR) {
3493 if(myConverterData->currentConverter->charErrorBufferLength > 0) {
3494 uprv_memcpy(
3495 cnv->charErrorBuffer,
3496 myConverterData->currentConverter->charErrorBuffer,
3497 myConverterData->currentConverter->charErrorBufferLength);
3498 }
3499 cnv->charErrorBufferLength = myConverterData->currentConverter->charErrorBufferLength;
3500 myConverterData->currentConverter->charErrorBufferLength = 0;
3501 }
3502 return;
3503 }
3504 default:
3505 /* not expected */
3506 break;
3507 }
3508 ucnv_cbFromUWriteBytes(args,
3509 buffer, (int32_t)(p - buffer),
3510 offsetIndex, err);
3511 }
3512
3513 /*
3514 * Structure for cloning an ISO 2022 converter into a single memory block.
3515 * ucnv_safeClone() of the converter will align the entire cloneStruct,
3516 * and then ucnv_safeClone() of the sub-converter may additionally align
3517 * currentConverter inside the cloneStruct, for which we need the deadSpace
3518 * after currentConverter.
3519 * This is because UAlignedMemory may be larger than the actually
3520 * necessary alignment size for the platform.
3521 * The other cloneStruct fields will not be moved around,
3522 * and are aligned properly with cloneStruct's alignment.
3523 */
3524 struct cloneStruct
3525 {
3526 UConverter cnv;
3527 UConverter currentConverter;
3528 UAlignedMemory deadSpace;
3529 UConverterDataISO2022 mydata;
3530 };
3531
3532
3533 static UConverter *
3534 _ISO_2022_SafeClone(
3535 const UConverter *cnv,
3536 void *stackBuffer,
3537 int32_t *pBufferSize,
3538 UErrorCode *status)
3539 {
3540 struct cloneStruct * localClone;
3541 UConverterDataISO2022 *cnvData;
3542 int32_t i, size;
3543
3544 if (*pBufferSize == 0) { /* 'preflighting' request - set needed size into *pBufferSize */
3545 *pBufferSize = (int32_t)sizeof(struct cloneStruct);
3546 return NULL;
3547 }
3548
3549 cnvData = (UConverterDataISO2022 *)cnv->extraInfo;
3550 localClone = (struct cloneStruct *)stackBuffer;
3551
3552 /* ucnv.c/ucnv_safeClone() copied the main UConverter already */
3553
3554 uprv_memcpy(&localClone->mydata, cnvData, sizeof(UConverterDataISO2022));
3555 localClone->cnv.extraInfo = &localClone->mydata; /* set pointer to extra data */
3556 localClone->cnv.isExtraLocal = TRUE;
3557
3558 /* share the subconverters */
3559
3560 if(cnvData->currentConverter != NULL) {
3561 size = (int32_t)(sizeof(UConverter) + sizeof(UAlignedMemory)); /* include size of padding */
3562 localClone->mydata.currentConverter =
3563 ucnv_safeClone(cnvData->currentConverter,
3564 &localClone->currentConverter,
3565 &size, status);
3566 if(U_FAILURE(*status)) {
3567 return NULL;
3568 }
3569 }
3570
3571 for(i=0; i<UCNV_2022_MAX_CONVERTERS; ++i) {
3572 if(cnvData->myConverterArray[i] != NULL) {
3573 ucnv_incrementRefCount(cnvData->myConverterArray[i]);
3574 }
3575 }
3576
3577 return &localClone->cnv;
3578 }
3579
3580 static void
3581 _ISO_2022_GetUnicodeSet(const UConverter *cnv,
3582 const USetAdder *sa,
3583 UConverterUnicodeSet which,
3584 UErrorCode *pErrorCode)
3585 {
3586 int32_t i;
3587 UConverterDataISO2022* cnvData;
3588
3589 if (U_FAILURE(*pErrorCode)) {
3590 return;
3591 }
3592 #ifdef U_ENABLE_GENERIC_ISO_2022
3593 if (cnv->sharedData == &_ISO2022Data) {
3594 /* We use UTF-8 in this case */
3595 sa->addRange(sa->set, 0, 0xd7FF);
3596 sa->addRange(sa->set, 0xE000, 0x10FFFF);
3597 return;
3598 }
3599 #endif
3600
3601 cnvData = (UConverterDataISO2022*)cnv->extraInfo;
3602
3603 /* open a set and initialize it with code points that are algorithmically round-tripped */
3604 switch(cnvData->locale[0]){
3605 case 'j':
3606 /* include JIS X 0201 which is hardcoded */
3607 sa->add(sa->set, 0xa5);
3608 sa->add(sa->set, 0x203e);
3609 if(jpCharsetMasks[cnvData->version]&CSM(ISO8859_1)) {
3610 /* include Latin-1 for some variants of JP */
3611 sa->addRange(sa->set, 0, 0xff);
3612 } else {
3613 /* include ASCII for JP */
3614 sa->addRange(sa->set, 0, 0x7f);
3615 }
3616 if(cnvData->version==3 || cnvData->version==4 || which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
3617 /*
3618 * Do not test (jpCharsetMasks[cnvData->version]&CSM(HWKANA_7BIT))!=0
3619 * because the bit is on for all JP versions although only versions 3 & 4 (JIS7 & JIS8)
3620 * use half-width Katakana.
3621 * This is because all ISO-2022-JP variants are lenient in that they accept (in toUnicode)
3622 * half-width Katakana via the ESC ( I sequence.
3623 * However, we only emit (fromUnicode) half-width Katakana according to the
3624 * definition of each variant.
3625 *
3626 * When including fallbacks,
3627 * we need to include half-width Katakana Unicode code points for all JP variants because
3628 * JIS X 0208 has hardcoded fallbacks for them (which map to full-width Katakana).
3629 */
3630 /* include half-width Katakana for JP */
3631 sa->addRange(sa->set, HWKANA_START, HWKANA_END);
3632 }
3633 break;
3634 case 'c':
3635 case 'z':
3636 /* include ASCII for CN */
3637 sa->addRange(sa->set, 0, 0x7f);
3638 break;
3639 case 'k':
3640 /* there is only one converter for KR, and it is not in the myConverterArray[] */
3641 cnvData->currentConverter->sharedData->impl->getUnicodeSet(
3642 cnvData->currentConverter, sa, which, pErrorCode);
3643 /* the loop over myConverterArray[] will simply not find another converter */
3644 break;
3645 default:
3646 break;
3647 }
3648
3649 #if 0 /* Replaced by ucnv_MBCSGetFilteredUnicodeSetForUnicode() until we implement ucnv_getUnicodeSet() with reverse fallbacks. */
3650 if( (cnvData->locale[0]=='c' || cnvData->locale[0]=='z') &&
3651 cnvData->version==0 && i==CNS_11643
3652 ) {
3653 /* special handling for non-EXT ISO-2022-CN: add only code points for CNS planes 1 and 2 */
3654 ucnv_MBCSGetUnicodeSetForBytes(
3655 cnvData->myConverterArray[i],
3656 sa, UCNV_ROUNDTRIP_SET,
3657 0, 0x81, 0x82,
3658 pErrorCode);
3659 }
3660 #endif
3661
3662 for (i=0; i<UCNV_2022_MAX_CONVERTERS; i++) {
3663 UConverterSetFilter filter;
3664 if(cnvData->myConverterArray[i]!=NULL) {
3665 if( (cnvData->locale[0]=='c' || cnvData->locale[0]=='z') &&
3666 cnvData->version==0 && i==CNS_11643
3667 ) {
3668 /*
3669 * Version-specific for CN:
3670 * CN version 0 does not map CNS planes 3..7 although
3671 * they are all available in the CNS conversion table;
3672 * CN version 1 (-EXT) does map them all.
3673 * The two versions create different Unicode sets.
3674 */
3675 filter=UCNV_SET_FILTER_2022_CN;
3676 } else if(cnvData->locale[0]=='j' && i==JISX208) {
3677 /*
3678 * Only add code points that map to Shift-JIS codes
3679 * corresponding to JIS X 0208.
3680 */
3681 filter=UCNV_SET_FILTER_SJIS;
3682 } else if(i==KSC5601) {
3683 /*
3684 * Some of the KSC 5601 tables (convrtrs.txt has this aliases on multiple tables)
3685 * are broader than GR94.
3686 */
3687 filter=UCNV_SET_FILTER_GR94DBCS;
3688 } else {
3689 filter=UCNV_SET_FILTER_NONE;
3690 }
3691 ucnv_MBCSGetFilteredUnicodeSetForUnicode(cnvData->myConverterArray[i], sa, which, filter, pErrorCode);
3692 }
3693 }
3694
3695 /*
3696 * ISO 2022 converters must not convert SO/SI/ESC despite what
3697 * sub-converters do by themselves.
3698 * Remove these characters from the set.
3699 */
3700 sa->remove(sa->set, 0x0e);
3701 sa->remove(sa->set, 0x0f);
3702 sa->remove(sa->set, 0x1b);
3703
3704 /* ISO 2022 converters do not convert C1 controls either */
3705 sa->removeRange(sa->set, 0x80, 0x9f);
3706 }
3707
3708 static const UConverterImpl _ISO2022Impl={
3709 UCNV_ISO_2022,
3710
3711 NULL,
3712 NULL,
3713
3714 _ISO2022Open,
3715 _ISO2022Close,
3716 _ISO2022Reset,
3717
3718 #ifdef U_ENABLE_GENERIC_ISO_2022
3719 T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC,
3720 T_UConverter_toUnicode_ISO_2022_OFFSETS_LOGIC,
3721 ucnv_fromUnicode_UTF8,
3722 ucnv_fromUnicode_UTF8_OFFSETS_LOGIC,
3723 #else
3724 NULL,
3725 NULL,
3726 NULL,
3727 NULL,
3728 #endif
3729 NULL,
3730
3731 NULL,
3732 _ISO2022getName,
3733 _ISO_2022_WriteSub,
3734 _ISO_2022_SafeClone,
3735 _ISO_2022_GetUnicodeSet
3736 };
3737 static const UConverterStaticData _ISO2022StaticData={
3738 sizeof(UConverterStaticData),
3739 "ISO_2022",
3740 2022,
3741 UCNV_IBM,
3742 UCNV_ISO_2022,
3743 1,
3744 3, /* max 3 bytes per UChar from UTF-8 (4 bytes from surrogate _pair_) */
3745 { 0x1a, 0, 0, 0 },
3746 1,
3747 FALSE,
3748 FALSE,
3749 0,
3750 0,
3751 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
3752 };
3753 const UConverterSharedData _ISO2022Data={
3754 sizeof(UConverterSharedData),
3755 ~((uint32_t) 0),
3756 NULL,
3757 NULL,
3758 &_ISO2022StaticData,
3759 FALSE,
3760 &_ISO2022Impl,
3761 0
3762 };
3763
3764 /*************JP****************/
3765 static const UConverterImpl _ISO2022JPImpl={
3766 UCNV_ISO_2022,
3767
3768 NULL,
3769 NULL,
3770
3771 _ISO2022Open,
3772 _ISO2022Close,
3773 _ISO2022Reset,
3774
3775 UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC,
3776 UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC,
3777 UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC,
3778 UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC,
3779 NULL,
3780
3781 NULL,
3782 _ISO2022getName,
3783 _ISO_2022_WriteSub,
3784 _ISO_2022_SafeClone,
3785 _ISO_2022_GetUnicodeSet
3786 };
3787 static const UConverterStaticData _ISO2022JPStaticData={
3788 sizeof(UConverterStaticData),
3789 "ISO_2022_JP",
3790 0,
3791 UCNV_IBM,
3792 UCNV_ISO_2022,
3793 1,
3794 6, /* max 6 bytes per UChar: 4-byte escape sequence + DBCS */
3795 { 0x1a, 0, 0, 0 },
3796 1,
3797 FALSE,
3798 FALSE,
3799 0,
3800 0,
3801 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
3802 };
3803 static const UConverterSharedData _ISO2022JPData={
3804 sizeof(UConverterSharedData),
3805 ~((uint32_t) 0),
3806 NULL,
3807 NULL,
3808 &_ISO2022JPStaticData,
3809 FALSE,
3810 &_ISO2022JPImpl,
3811 0
3812 };
3813
3814 /************* KR ***************/
3815 static const UConverterImpl _ISO2022KRImpl={
3816 UCNV_ISO_2022,
3817
3818 NULL,
3819 NULL,
3820
3821 _ISO2022Open,
3822 _ISO2022Close,
3823 _ISO2022Reset,
3824
3825 UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC,
3826 UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC,
3827 UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC,
3828 UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC,
3829 NULL,
3830
3831 NULL,
3832 _ISO2022getName,
3833 _ISO_2022_WriteSub,
3834 _ISO_2022_SafeClone,
3835 _ISO_2022_GetUnicodeSet
3836 };
3837 static const UConverterStaticData _ISO2022KRStaticData={
3838 sizeof(UConverterStaticData),
3839 "ISO_2022_KR",
3840 0,
3841 UCNV_IBM,
3842 UCNV_ISO_2022,
3843 1,
3844 3, /* max 3 bytes per UChar: SO+DBCS */
3845 { 0x1a, 0, 0, 0 },
3846 1,
3847 FALSE,
3848 FALSE,
3849 0,
3850 0,
3851 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
3852 };
3853 static const UConverterSharedData _ISO2022KRData={
3854 sizeof(UConverterSharedData),
3855 ~((uint32_t) 0),
3856 NULL,
3857 NULL,
3858 &_ISO2022KRStaticData,
3859 FALSE,
3860 &_ISO2022KRImpl,
3861 0
3862 };
3863
3864 /*************** CN ***************/
3865 static const UConverterImpl _ISO2022CNImpl={
3866
3867 UCNV_ISO_2022,
3868
3869 NULL,
3870 NULL,
3871
3872 _ISO2022Open,
3873 _ISO2022Close,
3874 _ISO2022Reset,
3875
3876 UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC,
3877 UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC,
3878 UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC,
3879 UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC,
3880 NULL,
3881
3882 NULL,
3883 _ISO2022getName,
3884 _ISO_2022_WriteSub,
3885 _ISO_2022_SafeClone,
3886 _ISO_2022_GetUnicodeSet
3887 };
3888 static const UConverterStaticData _ISO2022CNStaticData={
3889 sizeof(UConverterStaticData),
3890 "ISO_2022_CN",
3891 0,
3892 UCNV_IBM,
3893 UCNV_ISO_2022,
3894 1,
3895 8, /* max 8 bytes per UChar: 4-byte CNS designator + 2 bytes for SS2/SS3 + DBCS */
3896 { 0x1a, 0, 0, 0 },
3897 1,
3898 FALSE,
3899 FALSE,
3900 0,
3901 0,
3902 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
3903 };
3904 static const UConverterSharedData _ISO2022CNData={
3905 sizeof(UConverterSharedData),
3906 ~((uint32_t) 0),
3907 NULL,
3908 NULL,
3909 &_ISO2022CNStaticData,
3910 FALSE,
3911 &_ISO2022CNImpl,
3912 0
3913 };
3914
3915
3916
3917 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
mirror of ICU