]> git.saurik.com Git - apple/icu.git/blob - icuSources/i18n/ucol_tok.cpp
projects / apple / icu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ICU-400.37.tar.gz
[apple/icu.git] / icuSources / i18n / ucol_tok.cpp
1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2001-2008, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: ucol_tok.cpp
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created 02/22/2001
14 * created by: Vladimir Weinstein
15 *
16 * This module reads a tailoring rule string and produces a list of
17 * tokens that will be turned into collation elements
18 *
19 */
20
21 #include "unicode/utypes.h"
22
23 #if !UCONFIG_NO_COLLATION
24
25 #include "unicode/ustring.h"
26 #include "unicode/uchar.h"
27 #include "unicode/uniset.h"
28
29 #include "ucol_tok.h"
30 #include "ucol_bld.h"
31 #include "cmemory.h"
32 #include "util.h"
33
34 U_CDECL_BEGIN
35 static int32_t U_CALLCONV
36 uhash_hashTokens(const UHashTok k)
37 {
38 int32_t hash = 0;
39 //uint32_t key = (uint32_t)k.integer;
40 UColToken *key = (UColToken *)k.pointer;
41 if (key != 0) {
42 //int32_t len = (key & 0xFF000000)>>24;
43 int32_t len = (key->source & 0xFF000000)>>24;
44 int32_t inc = ((len - 32) / 32) + 1;
45
46 //const UChar *p = (key & 0x00FFFFFF) + rulesToParse;
47 const UChar *p = (key->source & 0x00FFFFFF) + key->rulesToParse;
48 const UChar *limit = p + len;
49
50 while (p<limit) {
51 hash = (hash * 37) + *p;
52 p += inc;
53 }
54 }
55 return hash;
56 }
57
58 static UBool U_CALLCONV
59 uhash_compareTokens(const UHashTok key1, const UHashTok key2)
60 {
61 //uint32_t p1 = (uint32_t) key1.integer;
62 //uint32_t p2 = (uint32_t) key2.integer;
63 UColToken *p1 = (UColToken *)key1.pointer;
64 UColToken *p2 = (UColToken *)key2.pointer;
65 const UChar *s1 = (p1->source & 0x00FFFFFF) + p1->rulesToParse;
66 const UChar *s2 = (p2->source & 0x00FFFFFF) + p2->rulesToParse;
67 uint32_t s1L = ((p1->source & 0xFF000000) >> 24);
68 uint32_t s2L = ((p2->source & 0xFF000000) >> 24);
69 const UChar *end = s1+s1L-1;
70
71 if (p1 == p2) {
72 return TRUE;
73 }
74 if (p1->source == 0 || p2->source == 0) {
75 return FALSE;
76 }
77 if(s1L != s2L) {
78 return FALSE;
79 }
80 if(p1->source == p2->source) {
81 return TRUE;
82 }
83 while((s1 < end) && *s1 == *s2) {
84 ++s1;
85 ++s2;
86 }
87 if(*s1 == *s2) {
88 return TRUE;
89 } else {
90 return FALSE;
91 }
92 }
93 U_CDECL_END
94
95 /*static inline void U_CALLCONV
96 uhash_freeBlockWrapper(void *obj) {
97 uhash_freeBlock(obj);
98 }*/
99
100
101 typedef struct {
102 uint32_t startCE;
103 uint32_t startContCE;
104 uint32_t limitCE;
105 uint32_t limitContCE;
106 } indirectBoundaries;
107
108 /* these values are used for finding CE values for indirect positioning. */
109 /* Indirect positioning is a mechanism for allowing resets on symbolic */
110 /* values. It only works for resets and you cannot tailor indirect names */
111 /* An indirect name can define either an anchor point or a range. An */
112 /* anchor point behaves in exactly the same way as a code point in reset */
113 /* would, except that it cannot be tailored. A range (we currently only */
114 /* know for the [top] range will explicitly set the upper bound for */
115 /* generated CEs, thus allowing for better control over how many CEs can */
116 /* be squeezed between in the range without performance penalty. */
117 /* In that respect, we use [top] for tailoring of locales that use CJK */
118 /* characters. Other indirect values are currently a pure convenience, */
119 /* they can be used to assure that the CEs will be always positioned in */
120 /* the same place relative to a point with known properties (e.g. first */
121 /* primary ignorable). */
122 static indirectBoundaries ucolIndirectBoundaries[15];
123 /*
124 static indirectBoundaries ucolIndirectBoundaries[11] = {
125 { UCOL_RESET_TOP_VALUE, 0,
126 UCOL_NEXT_TOP_VALUE, 0 },
127 { UCOL_FIRST_PRIMARY_IGNORABLE, 0,
128 0, 0 },
129 { UCOL_LAST_PRIMARY_IGNORABLE, UCOL_LAST_PRIMARY_IGNORABLE_CONT,
130 0, 0 },
131 { UCOL_FIRST_SECONDARY_IGNORABLE, 0,
132 0, 0 },
133 { UCOL_LAST_SECONDARY_IGNORABLE, 0,
134 0, 0 },
135 { UCOL_FIRST_TERTIARY_IGNORABLE, 0,
136 0, 0 },
137 { UCOL_LAST_TERTIARY_IGNORABLE, 0,
138 0, 0 },
139 { UCOL_FIRST_VARIABLE, 0,
140 0, 0 },
141 { UCOL_LAST_VARIABLE, 0,
142 0, 0 },
143 { UCOL_FIRST_NON_VARIABLE, 0,
144 0, 0 },
145 { UCOL_LAST_NON_VARIABLE, 0,
146 0, 0 },
147 };
148 */
149
150 static void setIndirectBoundaries(uint32_t indexR, uint32_t *start, uint32_t *end) {
151
152 // Set values for the top - TODO: once we have values for all the indirects, we are going
153 // to initalize here.
154 ucolIndirectBoundaries[indexR].startCE = start[0];
155 ucolIndirectBoundaries[indexR].startContCE = start[1];
156 if(end) {
157 ucolIndirectBoundaries[indexR].limitCE = end[0];
158 ucolIndirectBoundaries[indexR].limitContCE = end[1];
159 } else {
160 ucolIndirectBoundaries[indexR].limitCE = 0;
161 ucolIndirectBoundaries[indexR].limitContCE = 0;
162 }
163 }
164
165
166 static inline
167 void syntaxError(const UChar* rules,
168 int32_t pos,
169 int32_t rulesLen,
170 UParseError* parseError)
171 {
172 parseError->offset = pos;
173 parseError->line = 0 ; /* we are not using line numbers */
174
175 // for pre-context
176 int32_t start = (pos < U_PARSE_CONTEXT_LEN)? 0 : (pos - (U_PARSE_CONTEXT_LEN-1));
177 int32_t stop = pos;
178
179 u_memcpy(parseError->preContext,rules+start,stop-start);
180 //null terminate the buffer
181 parseError->preContext[stop-start] = 0;
182
183 //for post-context
184 start = pos+1;
185 stop = ((pos+U_PARSE_CONTEXT_LEN)<= rulesLen )? (pos+(U_PARSE_CONTEXT_LEN-1)) :
186 rulesLen;
187
188 if(start < stop) {
189 u_memcpy(parseError->postContext,rules+start,stop-start);
190 //null terminate the buffer
191 parseError->postContext[stop-start]= 0;
192 } else {
193 parseError->postContext[0] = 0;
194 }
195 }
196
197 static
198 void ucol_uprv_tok_setOptionInImage(UColOptionSet *opts, UColAttribute attrib, UColAttributeValue value) {
199 switch(attrib) {
200 case UCOL_HIRAGANA_QUATERNARY_MODE:
201 opts->hiraganaQ = value;
202 break;
203 case UCOL_FRENCH_COLLATION:
204 opts->frenchCollation = value;
205 break;
206 case UCOL_ALTERNATE_HANDLING:
207 opts->alternateHandling = value;
208 break;
209 case UCOL_CASE_FIRST:
210 opts->caseFirst = value;
211 break;
212 case UCOL_CASE_LEVEL:
213 opts->caseLevel = value;
214 break;
215 case UCOL_NORMALIZATION_MODE:
216 opts->normalizationMode = value;
217 break;
218 case UCOL_STRENGTH:
219 opts->strength = value;
220 break;
221 case UCOL_NUMERIC_COLLATION:
222 opts->numericCollation = value;
223 break;
224 case UCOL_ATTRIBUTE_COUNT:
225 default:
226 break;
227 }
228 }
229
230 #define UTOK_OPTION_COUNT 20
231
232 static UBool didInit = FALSE;
233 /* we can be strict, or we can be lenient */
234 /* I'd surely be lenient with the option arguments */
235 /* maybe even with options */
236 U_STRING_DECL(suboption_00, "non-ignorable", 13);
237 U_STRING_DECL(suboption_01, "shifted", 7);
238
239 U_STRING_DECL(suboption_02, "lower", 5);
240 U_STRING_DECL(suboption_03, "upper", 5);
241 U_STRING_DECL(suboption_04, "off", 3);
242 U_STRING_DECL(suboption_05, "on", 2);
243 U_STRING_DECL(suboption_06, "1", 1);
244 U_STRING_DECL(suboption_07, "2", 1);
245 U_STRING_DECL(suboption_08, "3", 1);
246 U_STRING_DECL(suboption_09, "4", 1);
247 U_STRING_DECL(suboption_10, "I", 1);
248
249 U_STRING_DECL(suboption_11, "primary", 7);
250 U_STRING_DECL(suboption_12, "secondary", 9);
251 U_STRING_DECL(suboption_13, "tertiary", 8);
252 U_STRING_DECL(suboption_14, "variable", 8);
253 U_STRING_DECL(suboption_15, "regular", 7);
254 U_STRING_DECL(suboption_16, "implicit", 8);
255 U_STRING_DECL(suboption_17, "trailing", 8);
256
257
258 U_STRING_DECL(option_00, "undefined", 9);
259 U_STRING_DECL(option_01, "rearrange", 9);
260 U_STRING_DECL(option_02, "alternate", 9);
261 U_STRING_DECL(option_03, "backwards", 9);
262 U_STRING_DECL(option_04, "variable top", 12);
263 U_STRING_DECL(option_05, "top", 3);
264 U_STRING_DECL(option_06, "normalization", 13);
265 U_STRING_DECL(option_07, "caseLevel", 9);
266 U_STRING_DECL(option_08, "caseFirst", 9);
267 U_STRING_DECL(option_09, "scriptOrder", 11);
268 U_STRING_DECL(option_10, "charsetname", 11);
269 U_STRING_DECL(option_11, "charset", 7);
270 U_STRING_DECL(option_12, "before", 6);
271 U_STRING_DECL(option_13, "hiraganaQ", 9);
272 U_STRING_DECL(option_14, "strength", 8);
273 U_STRING_DECL(option_15, "first", 5);
274 U_STRING_DECL(option_16, "last", 4);
275 U_STRING_DECL(option_17, "optimize", 8);
276 U_STRING_DECL(option_18, "suppressContractions", 20);
277 U_STRING_DECL(option_19, "numericOrdering", 15);
278
279
280 /*
281 [last variable] last variable value
282 [last primary ignorable] largest CE for primary ignorable
283 [last secondary ignorable] largest CE for secondary ignorable
284 [last tertiary ignorable] largest CE for tertiary ignorable
285 [top] guaranteed to be above all implicit CEs, for now and in the future (in 1.8)
286 */
287
288
289 static const ucolTokSuboption alternateSub[2] = {
290 {suboption_00, 13, UCOL_NON_IGNORABLE},
291 {suboption_01, 7, UCOL_SHIFTED}
292 };
293
294 static const ucolTokSuboption caseFirstSub[3] = {
295 {suboption_02, 5, UCOL_LOWER_FIRST},
296 {suboption_03, 5, UCOL_UPPER_FIRST},
297 {suboption_04, 3, UCOL_OFF},
298 };
299
300 static const ucolTokSuboption onOffSub[2] = {
301 {suboption_04, 3, UCOL_OFF},
302 {suboption_05, 2, UCOL_ON}
303 };
304
305 static const ucolTokSuboption frenchSub[1] = {
306 {suboption_07, 1, UCOL_ON}
307 };
308
309 static const ucolTokSuboption beforeSub[3] = {
310 {suboption_06, 1, UCOL_PRIMARY},
311 {suboption_07, 1, UCOL_SECONDARY},
312 {suboption_08, 1, UCOL_TERTIARY}
313 };
314
315 static const ucolTokSuboption strengthSub[5] = {
316 {suboption_06, 1, UCOL_PRIMARY},
317 {suboption_07, 1, UCOL_SECONDARY},
318 {suboption_08, 1, UCOL_TERTIARY},
319 {suboption_09, 1, UCOL_QUATERNARY},
320 {suboption_10, 1, UCOL_IDENTICAL},
321 };
322
323 static const ucolTokSuboption firstLastSub[7] = {
324 {suboption_11, 7, UCOL_PRIMARY},
325 {suboption_12, 9, UCOL_PRIMARY},
326 {suboption_13, 8, UCOL_PRIMARY},
327 {suboption_14, 8, UCOL_PRIMARY},
328 {suboption_15, 7, UCOL_PRIMARY},
329 {suboption_16, 8, UCOL_PRIMARY},
330 {suboption_17, 8, UCOL_PRIMARY},
331 };
332
333 enum OptionNumber {
334 OPTION_ALTERNATE_HANDLING = 0,
335 OPTION_FRENCH_COLLATION,
336 OPTION_CASE_LEVEL,
337 OPTION_CASE_FIRST,
338 OPTION_NORMALIZATION_MODE,
339 OPTION_HIRAGANA_QUATERNARY,
340 OPTION_STRENGTH,
341 OPTION_NUMERIC_COLLATION,
342 OPTION_NORMAL_OPTIONS_LIMIT = OPTION_NUMERIC_COLLATION,
343 OPTION_VARIABLE_TOP,
344 OPTION_REARRANGE,
345 OPTION_BEFORE,
346 OPTION_TOP,
347 OPTION_FIRST,
348 OPTION_LAST,
349 OPTION_OPTIMIZE,
350 OPTION_SUPPRESS_CONTRACTIONS,
351 OPTION_UNDEFINED,
352 OPTION_SCRIPT_ORDER,
353 OPTION_CHARSET_NAME,
354 OPTION_CHARSET
355 } ;
356
357 static const ucolTokOption rulesOptions[UTOK_OPTION_COUNT] = {
358 /*00*/ {option_02, 9, alternateSub, 2, UCOL_ALTERNATE_HANDLING}, /*"alternate" */
359 /*01*/ {option_03, 9, frenchSub, 1, UCOL_FRENCH_COLLATION}, /*"backwards" */
360 /*02*/ {option_07, 9, onOffSub, 2, UCOL_CASE_LEVEL}, /*"caseLevel" */
361 /*03*/ {option_08, 9, caseFirstSub, 3, UCOL_CASE_FIRST}, /*"caseFirst" */
362 /*04*/ {option_06, 13, onOffSub, 2, UCOL_NORMALIZATION_MODE}, /*"normalization" */
363 /*05*/ {option_13, 9, onOffSub, 2, UCOL_HIRAGANA_QUATERNARY_MODE}, /*"hiraganaQ" */
364 /*06*/ {option_14, 8, strengthSub, 5, UCOL_STRENGTH}, /*"strength" */
365 /*07*/ {option_19, 15, onOffSub, 2, UCOL_NUMERIC_COLLATION}, /*"numericOrdering"*/
366 /*08*/ {option_04, 12, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"variable top" */
367 /*09*/ {option_01, 9, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"rearrange" */
368 /*10*/ {option_12, 6, beforeSub, 3, UCOL_ATTRIBUTE_COUNT}, /*"before" */
369 /*11*/ {option_05, 3, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"top" */
370 /*12*/ {option_15, 5, firstLastSub, 7, UCOL_ATTRIBUTE_COUNT}, /*"first" */
371 /*13*/ {option_16, 4, firstLastSub, 7, UCOL_ATTRIBUTE_COUNT}, /*"last" */
372 /*14*/ {option_17, 8, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"optimize" */
373 /*15*/ {option_18, 20, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"suppressContractions" */
374 /*16*/ {option_00, 9, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"undefined" */
375 /*17*/ {option_09, 11, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"scriptOrder" */
376 /*18*/ {option_10, 11, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"charsetname" */
377 /*19*/ {option_11, 7, NULL, 0, UCOL_ATTRIBUTE_COUNT} /*"charset" */
378 };
379
380 static
381 int32_t u_strncmpNoCase(const UChar *s1,
382 const UChar *s2,
383 int32_t n)
384 {
385 if(n > 0) {
386 int32_t rc;
387 for(;;) {
388 rc = (int32_t)u_tolower(*s1) - (int32_t)u_tolower(*s2);
389 if(rc != 0 || *s1 == 0 || --n == 0) {
390 return rc;
391 }
392 ++s1;
393 ++s2;
394 }
395 }
396 return 0;
397 }
398
399 static
400 void ucol_uprv_tok_initData() {
401 if(!didInit) {
402 U_STRING_INIT(suboption_00, "non-ignorable", 13);
403 U_STRING_INIT(suboption_01, "shifted", 7);
404
405 U_STRING_INIT(suboption_02, "lower", 5);
406 U_STRING_INIT(suboption_03, "upper", 5);
407 U_STRING_INIT(suboption_04, "off", 3);
408 U_STRING_INIT(suboption_05, "on", 2);
409
410 U_STRING_INIT(suboption_06, "1", 1);
411 U_STRING_INIT(suboption_07, "2", 1);
412 U_STRING_INIT(suboption_08, "3", 1);
413 U_STRING_INIT(suboption_09, "4", 1);
414 U_STRING_INIT(suboption_10, "I", 1);
415
416 U_STRING_INIT(suboption_11, "primary", 7);
417 U_STRING_INIT(suboption_12, "secondary", 9);
418 U_STRING_INIT(suboption_13, "tertiary", 8);
419 U_STRING_INIT(suboption_14, "variable", 8);
420 U_STRING_INIT(suboption_15, "regular", 7);
421 U_STRING_INIT(suboption_16, "implicit", 8);
422 U_STRING_INIT(suboption_17, "trailing", 8);
423
424
425 U_STRING_INIT(option_00, "undefined", 9);
426 U_STRING_INIT(option_01, "rearrange", 9);
427 U_STRING_INIT(option_02, "alternate", 9);
428 U_STRING_INIT(option_03, "backwards", 9);
429 U_STRING_INIT(option_04, "variable top", 12);
430 U_STRING_INIT(option_05, "top", 3);
431 U_STRING_INIT(option_06, "normalization", 13);
432 U_STRING_INIT(option_07, "caseLevel", 9);
433 U_STRING_INIT(option_08, "caseFirst", 9);
434 U_STRING_INIT(option_09, "scriptOrder", 11);
435 U_STRING_INIT(option_10, "charsetname", 11);
436 U_STRING_INIT(option_11, "charset", 7);
437 U_STRING_INIT(option_12, "before", 6);
438 U_STRING_INIT(option_13, "hiraganaQ", 9);
439 U_STRING_INIT(option_14, "strength", 8);
440 U_STRING_INIT(option_15, "first", 5);
441 U_STRING_INIT(option_16, "last", 4);
442 U_STRING_INIT(option_17, "optimize", 8);
443 U_STRING_INIT(option_18, "suppressContractions", 20);
444 U_STRING_INIT(option_19, "numericOrdering", 15);
445 didInit = TRUE;
446 }
447 }
448
449
450 // This function reads basic options to set in the runtime collator
451 // used by data driven tests. Should not support build time options
452 U_CAPI const UChar * U_EXPORT2
453 ucol_tok_getNextArgument(const UChar *start, const UChar *end,
454 UColAttribute *attrib, UColAttributeValue *value,
455 UErrorCode *status)
456 {
457 uint32_t i = 0;
458 int32_t j=0;
459 UBool foundOption = FALSE;
460 const UChar *optionArg = NULL;
461
462 ucol_uprv_tok_initData();
463
464 while(start < end && u_isWhitespace(*start)) { /* eat whitespace */
465 start++;
466 }
467 if(start >= end) {
468 return NULL;
469 }
470 /* skip opening '[' */
471 if(*start == 0x005b) {
472 start++;
473 } else {
474 *status = U_ILLEGAL_ARGUMENT_ERROR; // no opening '['
475 return NULL;
476 }
477
478 while(i < UTOK_OPTION_COUNT) {
479 if(u_strncmpNoCase(start, rulesOptions[i].optionName, rulesOptions[i].optionLen) == 0) {
480 foundOption = TRUE;
481 if(end - start > rulesOptions[i].optionLen) {
482 optionArg = start+rulesOptions[i].optionLen+1; /* start of the options, skip space */
483 while(u_isWhitespace(*optionArg)) { /* eat whitespace */
484 optionArg++;
485 }
486 }
487 break;
488 }
489 i++;
490 }
491
492 if(!foundOption) {
493 *status = U_ILLEGAL_ARGUMENT_ERROR;
494 return NULL;
495 }
496
497 if(optionArg) {
498 for(j = 0; j<rulesOptions[i].subSize; j++) {
499 if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
500 //ucol_uprv_tok_setOptionInImage(src->opts, rulesOptions[i].attr, rulesOptions[i].subopts[j].attrVal);
501 *attrib = rulesOptions[i].attr;
502 *value = rulesOptions[i].subopts[j].attrVal;
503 optionArg += rulesOptions[i].subopts[j].subLen;
504 while(u_isWhitespace(*optionArg)) { /* eat whitespace */
505 optionArg++;
506 }
507 if(*optionArg == 0x005d) {
508 optionArg++;
509 return optionArg;
510 } else {
511 *status = U_ILLEGAL_ARGUMENT_ERROR;
512 return NULL;
513 }
514 }
515 }
516 }
517 *status = U_ILLEGAL_ARGUMENT_ERROR;
518 return NULL;
519 }
520
521 static
522 USet *ucol_uprv_tok_readAndSetUnicodeSet(const UChar *start, const UChar *end, UErrorCode *status) {
523 while(*start != 0x005b) { /* advance while we find the first '[' */
524 start++;
525 }
526 // now we need to get a balanced set of '[]'. The problem is that a set can have
527 // many, and *end point to the first closing '['
528 int32_t noOpenBraces = 1;
529 int32_t current = 1; // skip the opening brace
530 while(start+current < end && noOpenBraces != 0) {
531 if(start[current] == 0x005b) {
532 noOpenBraces++;
533 } else if(start[current] == 0x005D) { // closing brace
534 noOpenBraces--;
535 }
536 current++;
537 }
538
539 if(noOpenBraces != 0 || u_strchr(start+current, 0x005d /*']'*/) == NULL) {
540 *status = U_ILLEGAL_ARGUMENT_ERROR;
541 return NULL;
542 }
543 return uset_openPattern(start, current, status);
544 }
545
546 static
547 int32_t ucol_uprv_tok_readOption(const UChar *start, const UChar *end, const UChar **optionArg) {
548 int32_t i = 0;
549 ucol_uprv_tok_initData();
550
551 while(u_isWhitespace(*start)) { /* eat whitespace */
552 start++;
553 }
554 while(i < UTOK_OPTION_COUNT) {
555 if(u_strncmpNoCase(start, rulesOptions[i].optionName, rulesOptions[i].optionLen) == 0) {
556 if(end - start > rulesOptions[i].optionLen) {
557 *optionArg = start+rulesOptions[i].optionLen; /* start of the options*/
558 while(u_isWhitespace(**optionArg)) { /* eat whitespace */
559 (*optionArg)++;
560 }
561 }
562 break;
563 }
564 i++;
565 }
566 if(i == UTOK_OPTION_COUNT) {
567 i = -1; // didn't find an option
568 }
569 return i;
570 }
571
572
573 // reads and conforms to various options in rules
574 // end is the position of the first closing ']'
575 // However, some of the options take an UnicodeSet definition
576 // which needs to duplicate the closing ']'
577 // for example: '[copy [\uAC00-\uD7FF]]'
578 // These options will move end to the second ']' and the
579 // caller will set the current to it.
580 static
581 uint8_t ucol_uprv_tok_readAndSetOption(UColTokenParser *src, UErrorCode *status) {
582 const UChar* start = src->current;
583 int32_t i = 0;
584 int32_t j=0;
585 const UChar *optionArg = NULL;
586
587 uint8_t result = 0;
588
589 start++; /*skip opening '['*/
590 i = ucol_uprv_tok_readOption(start, src->end, &optionArg);
591 if(optionArg) {
592 src->current = optionArg;
593 }
594
595 if(i < 0) {
596 *status = U_ILLEGAL_ARGUMENT_ERROR;
597 } else {
598 int32_t noOpenBraces = 1;
599 switch(i) {
600 case OPTION_ALTERNATE_HANDLING:
601 case OPTION_FRENCH_COLLATION:
602 case OPTION_CASE_LEVEL:
603 case OPTION_CASE_FIRST:
604 case OPTION_NORMALIZATION_MODE:
605 case OPTION_HIRAGANA_QUATERNARY:
606 case OPTION_STRENGTH:
607 case OPTION_NUMERIC_COLLATION:
608 if(optionArg) {
609 for(j = 0; j<rulesOptions[i].subSize; j++) {
610 if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
611 ucol_uprv_tok_setOptionInImage(src->opts, rulesOptions[i].attr, rulesOptions[i].subopts[j].attrVal);
612 result = UCOL_TOK_SUCCESS;
613 }
614 }
615 }
616 if(result == 0) {
617 *status = U_ILLEGAL_ARGUMENT_ERROR;
618 }
619 break;
620 case OPTION_VARIABLE_TOP:
621 result = UCOL_TOK_SUCCESS | UCOL_TOK_VARIABLE_TOP;
622 break;
623 case OPTION_REARRANGE:
624 result = UCOL_TOK_SUCCESS;
625 break;
626 case OPTION_BEFORE:
627 if(optionArg) {
628 for(j = 0; j<rulesOptions[i].subSize; j++) {
629 if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
630 result = UCOL_TOK_SUCCESS | rulesOptions[i].subopts[j].attrVal + 1;
631 }
632 }
633 }
634 if(result == 0) {
635 *status = U_ILLEGAL_ARGUMENT_ERROR;
636 }
637 break;
638 case OPTION_TOP: /* we are going to have an array with structures of limit CEs */
639 /* index to this array will be src->parsedToken.indirectIndex*/
640 src->parsedToken.indirectIndex = 0;
641 result = UCOL_TOK_SUCCESS | UCOL_TOK_TOP;
642 break;
643 case OPTION_FIRST:
644 case OPTION_LAST: /* first, last */
645 for(j = 0; j<rulesOptions[i].subSize; j++) {
646 if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
647 // the calculation below assumes that OPTION_FIRST and OPTION_LAST are at i and i+1 and that the first
648 // element of indirect boundaries is reserved for top.
649 src->parsedToken.indirectIndex = (uint16_t)(i-OPTION_FIRST+1+j*2);
650 result = UCOL_TOK_SUCCESS | UCOL_TOK_TOP;;
651 }
652 }
653 if(result == 0) {
654 *status = U_ILLEGAL_ARGUMENT_ERROR;
655 }
656 break;
657 case OPTION_OPTIMIZE:
658 case OPTION_SUPPRESS_CONTRACTIONS: // copy and remove are handled before normalization
659 // we need to move end here
660 src->current++; // skip opening brace
661 while(src->current < src->end && noOpenBraces != 0) {
662 if(*src->current == 0x005b) {
663 noOpenBraces++;
664 } else if(*src->current == 0x005D) { // closing brace
665 noOpenBraces--;
666 }
667 src->current++;
668 }
669 result = UCOL_TOK_SUCCESS;
670 break;
671 default:
672 *status = U_UNSUPPORTED_ERROR;
673 break;
674 }
675 }
676 src->current = u_memchr(src->current, 0x005d, src->end-src->current);
677 return result;
678 }
679
680
681 inline void ucol_tok_addToExtraCurrent(UColTokenParser *src, const UChar *stuff, int32_t len, UErrorCode *status) {
682 if(src->extraCurrent+len >= src->extraEnd) {
683 /* reallocate */
684 UChar *newSrc = (UChar *)uprv_realloc(src->source, (src->extraEnd-src->source)*2*sizeof(UChar));
685 if(newSrc != NULL) {
686 src->current = newSrc + (src->current - src->source);
687 src->extraCurrent = newSrc + (src->extraCurrent - src->source);
688 src->end = newSrc + (src->end - src->source);
689 src->extraEnd = newSrc + (src->extraEnd-src->source)*2;
690 src->sourceCurrent = newSrc + (src->sourceCurrent-src->source);
691 src->source = newSrc;
692 } else {
693 *status = U_MEMORY_ALLOCATION_ERROR;
694 }
695 }
696 if(len == 1) {
697 *src->extraCurrent++ = *stuff;
698 } else {
699 uprv_memcpy(src->extraCurrent, stuff, len*sizeof(UChar));
700 src->extraCurrent += len;
701 }
702
703
704 }
705
706 inline UBool ucol_tok_doSetTop(UColTokenParser *src, UErrorCode *status) {
707 /*
708 top = TRUE;
709 */
710 UChar buff[5];
711 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
712 buff[0] = 0xFFFE;
713 buff[1] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE >> 16);
714 buff[2] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE & 0xFFFF);
715 if(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE == 0) {
716 src->parsedToken.charsLen = 3;
717 ucol_tok_addToExtraCurrent(src, buff, 3, status);
718 } else {
719 buff[3] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE >> 16);
720 buff[4] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE & 0xFFFF);
721 src->parsedToken.charsLen = 5;
722 ucol_tok_addToExtraCurrent(src, buff, 5, status);
723 }
724 return TRUE;
725 }
726
727 static UBool isCharNewLine(UChar c){
728 switch(c){
729 case 0x000A: /* LF */
730 case 0x000D: /* CR */
731 case 0x000C: /* FF */
732 case 0x0085: /* NEL */
733 case 0x2028: /* LS */
734 case 0x2029: /* PS */
735 return TRUE;
736 default:
737 return FALSE;
738 }
739 }
740
741 U_CAPI const UChar* U_EXPORT2
742 ucol_tok_parseNextToken(UColTokenParser *src,
743 UBool startOfRules,
744 UParseError *parseError,
745 UErrorCode *status)
746 {
747 /* parsing part */
748 UBool variableTop = FALSE;
749 UBool top = FALSE;
750 UBool inChars = TRUE;
751 UBool inQuote = FALSE;
752 UBool wasInQuote = FALSE;
753 uint8_t before = 0;
754 UBool isEscaped = FALSE;
755 // TODO: replace these variables with src->parsedToken counterparts
756 // no need to use them anymore since we have src->parsedToken.
757 // Ideally, token parser would be a nice class... Once, when I have
758 // more time (around 2020 probably).
759 uint32_t newExtensionLen = 0;
760 uint32_t extensionOffset = 0;
761 uint32_t newStrength = UCOL_TOK_UNSET;
762 UChar buff[10];
763
764 src->parsedToken.charsOffset = 0; src->parsedToken.charsLen = 0;
765 src->parsedToken.prefixOffset = 0; src->parsedToken.prefixLen = 0;
766 src->parsedToken.indirectIndex = 0;
767
768 while (src->current < src->end) {
769 UChar ch = *(src->current);
770
771 if (inQuote) {
772 if (ch == 0x0027/*'\''*/) {
773 inQuote = FALSE;
774 } else {
775 if ((src->parsedToken.charsLen == 0) || inChars) {
776 if(src->parsedToken.charsLen == 0) {
777 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
778 }
779 src->parsedToken.charsLen++;
780 } else {
781 if(newExtensionLen == 0) {
782 extensionOffset = (uint32_t)(src->extraCurrent - src->source);
783 }
784 newExtensionLen++;
785 }
786 }
787 }else if(isEscaped){
788 isEscaped =FALSE;
789 if (newStrength == UCOL_TOK_UNSET) {
790 *status = U_INVALID_FORMAT_ERROR;
791 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
792 return NULL;
793 // enabling rules to start with non-tokens a < b
794 // newStrength = UCOL_TOK_RESET;
795 }
796 if(ch != 0x0000 && src->current != src->end) {
797 if (inChars) {
798 if(src->parsedToken.charsLen == 0) {
799 src->parsedToken.charsOffset = (uint32_t)(src->current - src->source);
800 }
801 src->parsedToken.charsLen++;
802 } else {
803 if(newExtensionLen == 0) {
804 extensionOffset = (uint32_t)(src->current - src->source);
805 }
806 newExtensionLen++;
807 }
808 }
809 }else {
810 if(!uprv_isRuleWhiteSpace(ch)) {
811 /* Sets the strength for this entry */
812 switch (ch) {
813 case 0x003D/*'='*/ :
814 if (newStrength != UCOL_TOK_UNSET) {
815 goto EndOfLoop;
816 }
817
818 /* if we start with strength, we'll reset to top */
819 if(startOfRules == TRUE) {
820 src->parsedToken.indirectIndex = 5;
821 top = ucol_tok_doSetTop(src, status);
822 newStrength = UCOL_TOK_RESET;
823 goto EndOfLoop;
824 }
825 newStrength = UCOL_IDENTICAL;
826 break;
827
828 case 0x002C/*','*/:
829 if (newStrength != UCOL_TOK_UNSET) {
830 goto EndOfLoop;
831 }
832
833 /* if we start with strength, we'll reset to top */
834 if(startOfRules == TRUE) {
835 src->parsedToken.indirectIndex = 5;
836 top = ucol_tok_doSetTop(src, status);
837 newStrength = UCOL_TOK_RESET;
838 goto EndOfLoop;
839 }
840 newStrength = UCOL_TERTIARY;
841 break;
842
843 case 0x003B/*';'*/:
844 if (newStrength != UCOL_TOK_UNSET) {
845 goto EndOfLoop;
846 }
847
848 /* if we start with strength, we'll reset to top */
849 if(startOfRules == TRUE) {
850 src->parsedToken.indirectIndex = 5;
851 top = ucol_tok_doSetTop(src, status);
852 newStrength = UCOL_TOK_RESET;
853 goto EndOfLoop;
854 }
855 newStrength = UCOL_SECONDARY;
856 break;
857
858 case 0x003C/*'<'*/:
859 if (newStrength != UCOL_TOK_UNSET) {
860 goto EndOfLoop;
861 }
862
863 /* if we start with strength, we'll reset to top */
864 if(startOfRules == TRUE) {
865 src->parsedToken.indirectIndex = 5;
866 top = ucol_tok_doSetTop(src, status);
867 newStrength = UCOL_TOK_RESET;
868 goto EndOfLoop;
869 }
870 /* before this, do a scan to verify whether this is */
871 /* another strength */
872 if(*(src->current+1) == 0x003C) {
873 src->current++;
874 if(*(src->current+1) == 0x003C) {
875 src->current++; /* three in a row! */
876 newStrength = UCOL_TERTIARY;
877 } else { /* two in a row */
878 newStrength = UCOL_SECONDARY;
879 }
880 } else { /* just one */
881 newStrength = UCOL_PRIMARY;
882 }
883 break;
884
885 case 0x0026/*'&'*/:
886 if (newStrength != UCOL_TOK_UNSET) {
887 /**/
888 goto EndOfLoop;
889 }
890
891 newStrength = UCOL_TOK_RESET; /* PatternEntry::RESET = 0 */
892 break;
893
894 case 0x005b/*'['*/:
895 /* options - read an option, analyze it */
896 if(u_strchr(src->current, 0x005d /*']'*/) != NULL) {
897 uint8_t result = ucol_uprv_tok_readAndSetOption(src, status);
898 if(U_SUCCESS(*status)) {
899 if(result & UCOL_TOK_TOP) {
900 if(newStrength == UCOL_TOK_RESET) {
901 top = ucol_tok_doSetTop(src, status);
902 if(before) { // This is a combination of before and indirection like '&[before 2][first regular]<b'
903 src->parsedToken.charsLen+=2;
904 buff[0] = 0x002d;
905 buff[1] = before;
906 ucol_tok_addToExtraCurrent(src, buff, 2, status);
907 }
908
909 src->current++;
910 goto EndOfLoop;
911 } else {
912 *status = U_INVALID_FORMAT_ERROR;
913 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
914 }
915 } else if(result & UCOL_TOK_VARIABLE_TOP) {
916 if(newStrength != UCOL_TOK_RESET && newStrength != UCOL_TOK_UNSET) {
917 variableTop = TRUE;
918 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
919 src->parsedToken.charsLen = 1;
920 buff[0] = 0xFFFF;
921 ucol_tok_addToExtraCurrent(src, buff, 1, status);
922 src->current++;
923 goto EndOfLoop;
924 } else {
925 *status = U_INVALID_FORMAT_ERROR;
926 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
927 }
928 } else if (result & UCOL_TOK_BEFORE){
929 if(newStrength == UCOL_TOK_RESET) {
930 before = result & UCOL_TOK_BEFORE;
931 } else {
932 *status = U_INVALID_FORMAT_ERROR;
933 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
934
935 }
936 }
937 } else {
938 *status = U_INVALID_FORMAT_ERROR;
939 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
940 return NULL;
941 }
942 }
943 break;
944 case 0x0021/*! skip java thai modifier reordering*/:
945 break;
946 case 0x002F/*'/'*/:
947 wasInQuote = FALSE; /* if we were copying source characters, we want to stop now */
948 inChars = FALSE; /* we're now processing expansion */
949 break;
950 case 0x005C /* back slash for escaped chars */:
951 isEscaped = TRUE;
952 break;
953 /* found a quote, we're gonna start copying */
954 case 0x0027/*'\''*/:
955 if (newStrength == UCOL_TOK_UNSET) { /* quote is illegal until we have a strength */
956 *status = U_INVALID_FORMAT_ERROR;
957 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
958 return NULL;
959 // enabling rules to start with a non-token character a < b
960 // newStrength = UCOL_TOK_RESET;
961 }
962
963 inQuote = TRUE;
964
965 if(inChars) { /* we're doing characters */
966 if(wasInQuote == FALSE) {
967 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
968 }
969 if (src->parsedToken.charsLen != 0) {
970 ucol_tok_addToExtraCurrent(src, src->current - src->parsedToken.charsLen, src->parsedToken.charsLen, status);
971 }
972 src->parsedToken.charsLen++;
973 } else { /* we're doing an expansion */
974 if(wasInQuote == FALSE) {
975 extensionOffset = (uint32_t)(src->extraCurrent - src->source);
976 }
977 if (newExtensionLen != 0) {
978 ucol_tok_addToExtraCurrent(src, src->current - newExtensionLen, newExtensionLen, status);
979 }
980 newExtensionLen++;
981 }
982
983 wasInQuote = TRUE;
984
985 ch = *(++(src->current));
986 if(ch == 0x0027) { /* copy the double quote */
987 ucol_tok_addToExtraCurrent(src, &ch, 1, status);
988 inQuote = FALSE;
989 }
990 break;
991
992 /* '@' is french only if the strength is not currently set */
993 /* if it is, it's just a regular character in collation rules */
994 case 0x0040/*'@'*/:
995 if (newStrength == UCOL_TOK_UNSET) {
996 src->opts->frenchCollation = UCOL_ON;
997 break;
998 }
999
1000 case 0x007C /*|*/: /* this means we have actually been reading prefix part */
1001 // we want to store read characters to the prefix part and continue reading
1002 // the characters (proper way would be to restart reading the chars, but in
1003 // that case we would have to complicate the token hasher, which I do not
1004 // intend to play with. Instead, we will do prefixes when prefixes are due
1005 // (before adding the elements).
1006 src->parsedToken.prefixOffset = src->parsedToken.charsOffset;
1007 src->parsedToken.prefixLen = src->parsedToken.charsLen;
1008
1009 if(inChars) { /* we're doing characters */
1010 if(wasInQuote == FALSE) {
1011 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
1012 }
1013 if (src->parsedToken.charsLen != 0) {
1014 ucol_tok_addToExtraCurrent(src, src->current - src->parsedToken.charsLen, src->parsedToken.charsLen, status);
1015 }
1016 src->parsedToken.charsLen++;
1017 }
1018
1019 wasInQuote = TRUE;
1020
1021 do {
1022 ch = *(++(src->current));
1023 // skip whitespace between '|' and the character
1024 } while (uprv_isRuleWhiteSpace(ch));
1025 break;
1026
1027 //charsOffset = 0;
1028 //newCharsLen = 0;
1029 //break; // We want to store the whole prefix/character sequence. If we break
1030 // the '|' is going to get lost.
1031 case 0x0023 /*#*/: /* this is a comment, skip everything through the end of line */
1032 do {
1033 ch = *(++(src->current));
1034 } while (!isCharNewLine(ch));
1035
1036 break;
1037 default:
1038 if (newStrength == UCOL_TOK_UNSET) {
1039 *status = U_INVALID_FORMAT_ERROR;
1040 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
1041 return NULL;
1042 }
1043
1044 if (ucol_tok_isSpecialChar(ch) && (inQuote == FALSE)) {
1045 *status = U_INVALID_FORMAT_ERROR;
1046 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
1047 return NULL;
1048 }
1049
1050 if(ch == 0x0000 && src->current+1 == src->end) {
1051 break;
1052 }
1053
1054 if (inChars) {
1055 if(src->parsedToken.charsLen == 0) {
1056 src->parsedToken.charsOffset = (uint32_t)(src->current - src->source);
1057 }
1058 src->parsedToken.charsLen++;
1059 } else {
1060 if(newExtensionLen == 0) {
1061 extensionOffset = (uint32_t)(src->current - src->source);
1062 }
1063 newExtensionLen++;
1064 }
1065
1066 break;
1067 }
1068 }
1069 }
1070
1071 if(wasInQuote) {
1072 if(ch != 0x27) {
1073 if(inQuote || !uprv_isRuleWhiteSpace(ch)) {
1074 ucol_tok_addToExtraCurrent(src, &ch, 1, status);
1075 }
1076 }
1077 }
1078
1079 src->current++;
1080 }
1081
1082 EndOfLoop:
1083 wasInQuote = FALSE;
1084 if (newStrength == UCOL_TOK_UNSET) {
1085 return NULL;
1086 }
1087
1088 if (src->parsedToken.charsLen == 0 && top == FALSE) {
1089 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
1090 *status = U_INVALID_FORMAT_ERROR;
1091 return NULL;
1092 }
1093
1094 src->parsedToken.strength = newStrength;
1095 src->parsedToken.extensionOffset = extensionOffset;
1096 src->parsedToken.extensionLen = newExtensionLen;
1097 src->parsedToken.flags = (UCOL_TOK_VARIABLE_TOP * (variableTop?1:0)) | (UCOL_TOK_TOP * (top?1:0)) | before;
1098
1099 return src->current;
1100 }
1101
1102 /*
1103 Processing Description
1104 1 Build a ListList. Each list has a header, which contains two lists (positive
1105 and negative), a reset token, a baseCE, nextCE, and previousCE. The lists and
1106 reset may be null.
1107 2 As you process, you keep a LAST pointer that points to the last token you
1108 handled.
1109 */
1110
1111 static UColToken *ucol_tok_initAReset(UColTokenParser *src, UChar *expand, uint32_t *expandNext,
1112 UParseError *parseError, UErrorCode *status)
1113 {
1114 if(src->resultLen == src->listCapacity) {
1115 // Unfortunately, this won't work, as we store addresses of lhs in token
1116 src->listCapacity *= 2;
1117 src->lh = (UColTokListHeader *)uprv_realloc(src->lh, src->listCapacity*sizeof(UColTokListHeader));
1118 if(src->lh == NULL) {
1119 *status = U_MEMORY_ALLOCATION_ERROR;
1120 return NULL;
1121 }
1122 }
1123 /* do the reset thing */
1124 UColToken *sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
1125 /* test for NULL */
1126 if (sourceToken == NULL) {
1127 *status = U_MEMORY_ALLOCATION_ERROR;
1128 return NULL;
1129 }
1130 sourceToken->rulesToParse = src->source;
1131 sourceToken->source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
1132 sourceToken->expansion = src->parsedToken.extensionLen << 24 | src->parsedToken.extensionOffset;
1133
1134 sourceToken->debugSource = *(src->source + src->parsedToken.charsOffset);
1135 sourceToken->debugExpansion = *(src->source + src->parsedToken.extensionOffset);
1136
1137 // keep the flags around so that we know about before
1138 sourceToken->flags = src->parsedToken.flags;
1139
1140 if(src->parsedToken.prefixOffset != 0) {
1141 // this is a syntax error
1142 *status = U_INVALID_FORMAT_ERROR;
1143 syntaxError(src->source,src->parsedToken.charsOffset-1,src->parsedToken.charsOffset+src->parsedToken.charsLen,parseError);
1144 uprv_free(sourceToken);
1145 return 0;
1146 } else {
1147 sourceToken->prefix = 0;
1148 }
1149
1150 sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
1151 sourceToken->strength = UCOL_TOK_RESET;
1152 sourceToken->next = NULL;
1153 sourceToken->previous = NULL;
1154 sourceToken->noOfCEs = 0;
1155 sourceToken->noOfExpCEs = 0;
1156 sourceToken->listHeader = &src->lh[src->resultLen];
1157
1158 src->lh[src->resultLen].first = NULL;
1159 src->lh[src->resultLen].last = NULL;
1160 src->lh[src->resultLen].first = NULL;
1161 src->lh[src->resultLen].last = NULL;
1162
1163 src->lh[src->resultLen].reset = sourceToken;
1164
1165 /*
1166 3 Consider each item: relation, source, and expansion: e.g. ...< x / y ...
1167 First convert all expansions into normal form. Examples:
1168 If "xy" doesn't occur earlier in the list or in the UCA, convert &xy * c *
1169 d * ... into &x * c/y * d * ...
1170 Note: reset values can never have expansions, although they can cause the
1171 very next item to have one. They may be contractions, if they are found
1172 earlier in the list.
1173 */
1174 *expandNext = 0;
1175 if(expand != NULL) {
1176 /* check to see if there is an expansion */
1177 if(src->parsedToken.charsLen > 1) {
1178 uint32_t resetCharsOffset;
1179 resetCharsOffset = (uint32_t)(expand - src->source);
1180 sourceToken->source = ((resetCharsOffset - src->parsedToken.charsOffset ) << 24) | src->parsedToken.charsOffset;
1181 *expandNext = ((src->parsedToken.charsLen + src->parsedToken.charsOffset - resetCharsOffset)<<24) | (resetCharsOffset);
1182 }
1183 }
1184
1185 src->resultLen++;
1186
1187 uhash_put(src->tailored, sourceToken, sourceToken, status);
1188
1189 return sourceToken;
1190 }
1191
1192 static
1193 inline UColToken *getVirginBefore(UColTokenParser *src, UColToken *sourceToken, uint8_t strength, UParseError *parseError, UErrorCode *status) {
1194 if(U_FAILURE(*status)) {
1195 return NULL;
1196 }
1197 /* this is a virgin before - we need to fish the anchor from the UCA */
1198 collIterate s;
1199 uint32_t baseCE = UCOL_NOT_FOUND, baseContCE = UCOL_NOT_FOUND;
1200 uint32_t CE, SecondCE;
1201 uint32_t invPos;
1202 if(sourceToken != NULL) {
1203 uprv_init_collIterate(src->UCA, src->source+((sourceToken->source)&0xFFFFFF), 1, &s);
1204 } else {
1205 uprv_init_collIterate(src->UCA, src->source+src->parsedToken.charsOffset /**charsOffset*/, 1, &s);
1206 }
1207
1208 baseCE = ucol_getNextCE(src->UCA, &s, status) & 0xFFFFFF3F;
1209 baseContCE = ucol_getNextCE(src->UCA, &s, status);
1210 if(baseContCE == UCOL_NO_MORE_CES) {
1211 baseContCE = 0;
1212 }
1213
1214
1215 UCAConstants *consts = (UCAConstants *)((uint8_t *)src->UCA->image + src->UCA->image->UCAConsts);
1216 uint32_t ch = 0;
1217 uint32_t expandNext = 0;
1218 UColToken key;
1219
1220 if((baseCE & 0xFF000000) >= (consts->UCA_PRIMARY_IMPLICIT_MIN<<24) && (baseCE & 0xFF000000) <= (consts->UCA_PRIMARY_IMPLICIT_MAX<<24) ) { /* implicits - */
1221 uint32_t primary = baseCE & UCOL_PRIMARYMASK | (baseContCE & UCOL_PRIMARYMASK) >> 16;
1222 uint32_t raw = uprv_uca_getRawFromImplicit(primary);
1223 ch = uprv_uca_getCodePointFromRaw(raw-1);
1224 uint32_t primaryCE = uprv_uca_getImplicitFromRaw(raw-1);
1225 CE = primaryCE & UCOL_PRIMARYMASK | 0x0505;
1226 SecondCE = (primaryCE << 16) & UCOL_PRIMARYMASK | UCOL_CONTINUATION_MARKER;
1227
1228 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
1229 *src->extraCurrent++ = 0xFFFE;
1230 *src->extraCurrent++ = (UChar)ch;
1231 src->parsedToken.charsLen++;
1232
1233 key.source = (src->parsedToken.charsLen/**newCharsLen*/ << 24) | src->parsedToken.charsOffset/**charsOffset*/;
1234 key.rulesToParse = src->source;
1235
1236 //sourceToken = (UColToken *)uhash_iget(src->tailored, (int32_t)key);
1237 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1238
1239 if(sourceToken == NULL) {
1240 src->lh[src->resultLen].baseCE = CE & 0xFFFFFF3F;
1241 if(isContinuation(SecondCE)) {
1242 src->lh[src->resultLen].baseContCE = SecondCE;
1243 } else {
1244 src->lh[src->resultLen].baseContCE = 0;
1245 }
1246 src->lh[src->resultLen].nextCE = 0;
1247 src->lh[src->resultLen].nextContCE = 0;
1248 src->lh[src->resultLen].previousCE = 0;
1249 src->lh[src->resultLen].previousContCE = 0;
1250
1251 src->lh[src->resultLen].indirect = FALSE;
1252
1253 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
1254 }
1255
1256 } else {
1257 invPos = ucol_inv_getPrevCE(src, baseCE, baseContCE, &CE, &SecondCE, strength);
1258
1259 // we got the previous CE. Now we need to see if the difference between
1260 // the two CEs is really of the requested strength.
1261 // if it's a bigger difference (we asked for secondary and got primary), we
1262 // need to modify the CE.
1263 if(ucol_getCEStrengthDifference(baseCE, baseContCE, CE, SecondCE) < strength) {
1264 // adjust the strength
1265 // now we are in the situation where our baseCE should actually be modified in
1266 // order to get the CE in the right position.
1267 if(strength == UCOL_SECONDARY) {
1268 CE = baseCE - 0x0200;
1269 } else { // strength == UCOL_TERTIARY
1270 CE = baseCE - 0x02;
1271 }
1272 if(baseContCE) {
1273 if(strength == UCOL_SECONDARY) {
1274 SecondCE = baseContCE - 0x0200;
1275 } else { // strength == UCOL_TERTIARY
1276 SecondCE = baseContCE - 0x02;
1277 }
1278 }
1279 }
1280
1281 #if 0
1282 // the code below relies on getting a code point from the inverse table, in order to be
1283 // able to merge the situations like &x < 9 &[before 1]a < d. This won't work:
1284 // 1. There are many code points that have the same CE
1285 // 2. The CE to codepoint table (things pointed to by CETable[3*invPos+2] are broken.
1286 // Also, in case when there is no equivalent strength before an element, we have to actually
1287 // construct one. For example, &[before 2]a << x won't result in x << a, because the element
1288 // before a is a primary difference.
1289
1290 //uint32_t *CETable = (uint32_t *)((uint8_t *)src->invUCA+src->invUCA->table);
1291
1292
1293 ch = CETable[3*invPos+2];
1294
1295 if((ch & UCOL_INV_SIZEMASK) != 0) {
1296 uint16_t *conts = (uint16_t *)((uint8_t *)src->invUCA+src->invUCA->conts);
1297 uint32_t offset = (ch & UCOL_INV_OFFSETMASK);
1298 ch = conts[offset];
1299 }
1300
1301 *src->extraCurrent++ = (UChar)ch;
1302 src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source - 1);
1303 src->parsedToken.charsLen = 1;
1304
1305 // We got an UCA before. However, this might have been tailored.
1306 // example:
1307 // &\u30ca = \u306a
1308 // &[before 3]\u306a<<<\u306a|\u309d
1309
1310
1311 // uint32_t key = (*newCharsLen << 24) | *charsOffset;
1312 key.source = (src->parsedToken.charsLen/**newCharsLen*/ << 24) | src->parsedToken.charsOffset/**charsOffset*/;
1313 key.rulesToParse = src->source;
1314
1315 //sourceToken = (UColToken *)uhash_iget(src->tailored, (int32_t)key);
1316 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1317 #endif
1318
1319 // here is how it should be. The situation such as &[before 1]a < x, should be
1320 // resolved exactly as if we wrote &a > x.
1321 // therefore, I don't really care if the UCA value before a has been changed.
1322 // However, I do care if the strength between my element and the previous element
1323 // is bigger then I wanted. So, if CE < baseCE and I wanted &[before 2], then i'll
1324 // have to construct the base CE.
1325
1326
1327
1328 // if we found a tailored thing, we have to use the UCA value and construct
1329 // a new reset token with constructed name
1330 //if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
1331 // character to which we want to anchor is already tailored.
1332 // We need to construct a new token which will be the anchor
1333 // point
1334 //*(src->extraCurrent-1) = 0xFFFE;
1335 //*src->extraCurrent++ = (UChar)ch;
1336 // grab before
1337 src->parsedToken.charsOffset -= 10;
1338 src->parsedToken.charsLen += 10;
1339 src->lh[src->resultLen].baseCE = CE & 0xFFFFFF3F;
1340 if(isContinuation(SecondCE)) {
1341 src->lh[src->resultLen].baseContCE = SecondCE;
1342 } else {
1343 src->lh[src->resultLen].baseContCE = 0;
1344 }
1345 src->lh[src->resultLen].nextCE = 0;
1346 src->lh[src->resultLen].nextContCE = 0;
1347 src->lh[src->resultLen].previousCE = 0;
1348 src->lh[src->resultLen].previousContCE = 0;
1349
1350 src->lh[src->resultLen].indirect = FALSE;
1351
1352 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
1353 //}
1354 }
1355
1356 return sourceToken;
1357
1358 }
1359
1360 uint32_t ucol_tok_assembleTokenList(UColTokenParser *src, UParseError *parseError, UErrorCode *status) {
1361 UColToken *lastToken = NULL;
1362 const UChar *parseEnd = NULL;
1363 uint32_t expandNext = 0;
1364 UBool variableTop = FALSE;
1365 UBool top = FALSE;
1366 uint16_t specs = 0;
1367 UColTokListHeader *ListList = NULL;
1368
1369 src->parsedToken.strength = UCOL_TOK_UNSET;
1370
1371 ListList = src->lh;
1372
1373 if(U_FAILURE(*status)) {
1374 return 0;
1375 }
1376
1377 while(src->current < src->end) {
1378 src->parsedToken.prefixOffset = 0;
1379
1380 parseEnd = ucol_tok_parseNextToken(src,
1381 (UBool)(lastToken == NULL),
1382 parseError,
1383 status);
1384
1385 specs = src->parsedToken.flags;
1386
1387
1388 variableTop = ((specs & UCOL_TOK_VARIABLE_TOP) != 0);
1389 top = ((specs & UCOL_TOK_TOP) != 0);
1390
1391 if(U_SUCCESS(*status) && parseEnd != NULL) {
1392 UColToken *sourceToken = NULL;
1393 //uint32_t key = 0;
1394 uint32_t lastStrength = UCOL_TOK_UNSET;
1395
1396 if(lastToken != NULL ) {
1397 lastStrength = lastToken->strength;
1398 }
1399
1400 //key = newCharsLen << 24 | charsOffset;
1401 UColToken key;
1402 key.source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
1403 key.rulesToParse = src->source;
1404
1405 /* 4 Lookup each source in the CharsToToken map, and find a sourceToken */
1406 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1407
1408 if(src->parsedToken.strength != UCOL_TOK_RESET) {
1409 if(lastToken == NULL) { /* this means that rules haven't started properly */
1410 *status = U_INVALID_FORMAT_ERROR;
1411 syntaxError(src->source,0,(int32_t)(src->end-src->source),parseError);
1412 return 0;
1413 }
1414 /* 6 Otherwise (when relation != reset) */
1415 if(sourceToken == NULL) {
1416 /* If sourceToken is null, create new one, */
1417 sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
1418 /* test for NULL */
1419 if (sourceToken == NULL) {
1420 *status = U_MEMORY_ALLOCATION_ERROR;
1421 return 0;
1422 }
1423 sourceToken->rulesToParse = src->source;
1424 sourceToken->source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
1425
1426 sourceToken->debugSource = *(src->source + src->parsedToken.charsOffset);
1427
1428 sourceToken->prefix = src->parsedToken.prefixLen << 24 | src->parsedToken.prefixOffset;
1429 sourceToken->debugPrefix = *(src->source + src->parsedToken.prefixOffset);
1430
1431 sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
1432 sourceToken->next = NULL;
1433 sourceToken->previous = NULL;
1434 sourceToken->noOfCEs = 0;
1435 sourceToken->noOfExpCEs = 0;
1436 // keep the flags around so that we know about before
1437 sourceToken->flags = src->parsedToken.flags;
1438 uhash_put(src->tailored, sourceToken, sourceToken, status);
1439 if(U_FAILURE(*status)) {
1440 return 0;
1441 }
1442 } else {
1443 /* we could have fished out a reset here */
1444 if(sourceToken->strength != UCOL_TOK_RESET && lastToken != sourceToken) {
1445 /* otherwise remove sourceToken from where it was. */
1446 if(sourceToken->next != NULL) {
1447 if(sourceToken->next->strength > sourceToken->strength) {
1448 sourceToken->next->strength = sourceToken->strength;
1449 }
1450 sourceToken->next->previous = sourceToken->previous;
1451 } else {
1452 sourceToken->listHeader->last = sourceToken->previous;
1453 }
1454
1455 if(sourceToken->previous != NULL) {
1456 sourceToken->previous->next = sourceToken->next;
1457 } else {
1458 sourceToken->listHeader->first = sourceToken->next;
1459 }
1460 sourceToken->next = NULL;
1461 sourceToken->previous = NULL;
1462 }
1463 }
1464
1465 sourceToken->strength = src->parsedToken.strength;
1466 sourceToken->listHeader = lastToken->listHeader;
1467
1468 /*
1469 1. Find the strongest strength in each list, and set strongestP and strongestN
1470 accordingly in the headers.
1471 */
1472 if(lastStrength == UCOL_TOK_RESET
1473 || sourceToken->listHeader->first == 0) {
1474 /* If LAST is a reset
1475 insert sourceToken in the list. */
1476 if(sourceToken->listHeader->first == 0) {
1477 sourceToken->listHeader->first = sourceToken;
1478 sourceToken->listHeader->last = sourceToken;
1479 } else { /* we need to find a place for us */
1480 /* and we'll get in front of the same strength */
1481 if(sourceToken->listHeader->first->strength <= sourceToken->strength) {
1482 sourceToken->next = sourceToken->listHeader->first;
1483 sourceToken->next->previous = sourceToken;
1484 sourceToken->listHeader->first = sourceToken;
1485 sourceToken->previous = NULL;
1486 } else {
1487 lastToken = sourceToken->listHeader->first;
1488 while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
1489 lastToken = lastToken->next;
1490 }
1491 if(lastToken->next != NULL) {
1492 lastToken->next->previous = sourceToken;
1493 } else {
1494 sourceToken->listHeader->last = sourceToken;
1495 }
1496 sourceToken->previous = lastToken;
1497 sourceToken->next = lastToken->next;
1498 lastToken->next = sourceToken;
1499 }
1500 }
1501 } else {
1502 /* Otherwise (when LAST is not a reset)
1503 if polarity (LAST) == polarity(relation), insert sourceToken after LAST,
1504 otherwise insert before.
1505 when inserting after or before, search to the next position with the same
1506 strength in that direction. (This is called postpone insertion). */
1507 if(sourceToken != lastToken) {
1508 if(lastToken->polarity == sourceToken->polarity) {
1509 while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
1510 lastToken = lastToken->next;
1511 }
1512 sourceToken->previous = lastToken;
1513 if(lastToken->next != NULL) {
1514 lastToken->next->previous = sourceToken;
1515 } else {
1516 sourceToken->listHeader->last = sourceToken;
1517 }
1518
1519 sourceToken->next = lastToken->next;
1520 lastToken->next = sourceToken;
1521 } else {
1522 while(lastToken->previous != NULL && lastToken->previous->strength > sourceToken->strength) {
1523 lastToken = lastToken->previous;
1524 }
1525 sourceToken->next = lastToken;
1526 if(lastToken->previous != NULL) {
1527 lastToken->previous->next = sourceToken;
1528 } else {
1529 sourceToken->listHeader->first = sourceToken;
1530 }
1531 sourceToken->previous = lastToken->previous;
1532 lastToken->previous = sourceToken;
1533 }
1534 } else { /* repeated one thing twice in rules, stay with the stronger strength */
1535 if(lastStrength < sourceToken->strength) {
1536 sourceToken->strength = lastStrength;
1537 }
1538 }
1539 }
1540
1541 /* if the token was a variable top, we're gonna put it in */
1542 if(variableTop == TRUE && src->varTop == NULL) {
1543 variableTop = FALSE;
1544 src->varTop = sourceToken;
1545 }
1546
1547 // Treat the expansions.
1548 // There are two types of expansions: explicit (x / y) and reset based propagating expansions
1549 // (&abc * d * e <=> &ab * d / c * e / c)
1550 // if both of them are in effect for a token, they are combined.
1551
1552 sourceToken->expansion = src->parsedToken.extensionLen << 24 | src->parsedToken.extensionOffset;
1553
1554 if(expandNext != 0) {
1555 if(sourceToken->strength == UCOL_PRIMARY) { /* primary strength kills off the implicit expansion */
1556 expandNext = 0;
1557 } else if(sourceToken->expansion == 0) { /* if there is no expansion, implicit is just added to the token */
1558 sourceToken->expansion = expandNext;
1559 } else { /* there is both explicit and implicit expansion. We need to make a combination */
1560 uprv_memcpy(src->extraCurrent, src->source + (expandNext & 0xFFFFFF), (expandNext >> 24)*sizeof(UChar));
1561 uprv_memcpy(src->extraCurrent+(expandNext >> 24), src->source + src->parsedToken.extensionOffset, src->parsedToken.extensionLen*sizeof(UChar));
1562 sourceToken->expansion = (uint32_t)(((expandNext >> 24) + src->parsedToken.extensionLen)<<24 | (uint32_t)(src->extraCurrent - src->source));
1563 src->extraCurrent += (expandNext >> 24) + src->parsedToken.extensionLen;
1564 }
1565 }
1566
1567 // This is just for debugging purposes
1568 if(sourceToken->expansion != 0) {
1569 sourceToken->debugExpansion = *(src->source + src->parsedToken.extensionOffset);
1570 } else {
1571 sourceToken->debugExpansion = 0;
1572 }
1573 // if the previous token was a reset before, the strength of this
1574 // token must match the strength of before. Otherwise we have an
1575 // undefined situation.
1576 // In other words, we currently have a cludge which we use to
1577 // represent &a >> x. This is written as &[before 2]a << x.
1578 if((lastToken->flags & UCOL_TOK_BEFORE) != 0) {
1579 uint8_t beforeStrength = (lastToken->flags & UCOL_TOK_BEFORE) - 1;
1580 if(beforeStrength != sourceToken->strength) {
1581 *status = U_INVALID_FORMAT_ERROR;
1582 syntaxError(src->source,0,(int32_t)(src->end-src->source),parseError);
1583 return 0;
1584 }
1585 }
1586 } else {
1587 if(lastToken != NULL && lastStrength == UCOL_TOK_RESET) {
1588 /* if the previous token was also a reset, */
1589 /*this means that we have two consecutive resets */
1590 /* and we want to remove the previous one if empty*/
1591 if(src->resultLen > 0 && ListList[src->resultLen-1].first == NULL) {
1592 src->resultLen--;
1593 }
1594 }
1595
1596 if(sourceToken == NULL) { /* this is a reset, but it might still be somewhere in the tailoring, in shorter form */
1597 uint32_t searchCharsLen = src->parsedToken.charsLen;
1598 while(searchCharsLen > 1 && sourceToken == NULL) {
1599 searchCharsLen--;
1600 //key = searchCharsLen << 24 | charsOffset;
1601 UColToken key;
1602 key.source = searchCharsLen << 24 | src->parsedToken.charsOffset;
1603 key.rulesToParse = src->source;
1604 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1605 }
1606 if(sourceToken != NULL) {
1607 expandNext = (src->parsedToken.charsLen - searchCharsLen) << 24 | (src->parsedToken.charsOffset + searchCharsLen);
1608 }
1609 }
1610
1611 if((specs & UCOL_TOK_BEFORE) != 0) { /* we're doing before */
1612 if(top == FALSE) { /* there is no indirection */
1613 uint8_t strength = (specs & UCOL_TOK_BEFORE) - 1;
1614 if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
1615 /* this is a before that is already ordered in the UCA - so we need to get the previous with good strength */
1616 while(sourceToken->strength > strength && sourceToken->previous != NULL) {
1617 sourceToken = sourceToken->previous;
1618 }
1619 /* here, either we hit the strength or NULL */
1620 if(sourceToken->strength == strength) {
1621 if(sourceToken->previous != NULL) {
1622 sourceToken = sourceToken->previous;
1623 } else { /* start of list */
1624 sourceToken = sourceToken->listHeader->reset;
1625 }
1626 } else { /* we hit NULL */
1627 /* we should be doing the else part */
1628 sourceToken = sourceToken->listHeader->reset;
1629 sourceToken = getVirginBefore(src, sourceToken, strength, parseError, status);
1630 }
1631 } else {
1632 sourceToken = getVirginBefore(src, sourceToken, strength, parseError, status);
1633 }
1634 } else { /* this is both before and indirection */
1635 top = FALSE;
1636 ListList[src->resultLen].previousCE = 0;
1637 ListList[src->resultLen].previousContCE = 0;
1638 ListList[src->resultLen].indirect = TRUE;
1639 /* we need to do slightly more work. we need to get the baseCE using the */
1640 /* inverse UCA & getPrevious. The next bound is not set, and will be decided */
1641 /* in ucol_bld */
1642 uint8_t strength = (specs & UCOL_TOK_BEFORE) - 1;
1643 uint32_t baseCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE;
1644 uint32_t baseContCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE;//&0xFFFFFF3F;
1645 uint32_t CE = UCOL_NOT_FOUND, SecondCE = UCOL_NOT_FOUND;
1646
1647 UCAConstants *consts = (UCAConstants *)((uint8_t *)src->UCA->image + src->UCA->image->UCAConsts);
1648 if((baseCE & 0xFF000000) >= (consts->UCA_PRIMARY_IMPLICIT_MIN<<24) && (baseCE & 0xFF000000) <= (consts->UCA_PRIMARY_IMPLICIT_MAX<<24) ) { /* implicits - */
1649 uint32_t primary = baseCE & UCOL_PRIMARYMASK | (baseContCE & UCOL_PRIMARYMASK) >> 16;
1650 uint32_t raw = uprv_uca_getRawFromImplicit(primary);
1651 uint32_t primaryCE = uprv_uca_getImplicitFromRaw(raw-1);
1652 CE = primaryCE & UCOL_PRIMARYMASK | 0x0505;
1653 SecondCE = (primaryCE << 16) & UCOL_PRIMARYMASK | UCOL_CONTINUATION_MARKER;
1654 } else {
1655 /*int32_t invPos = ucol_inv_getPrevCE(baseCE, baseContCE, &CE, &SecondCE, strength);*/
1656 ucol_inv_getPrevCE(src, baseCE, baseContCE, &CE, &SecondCE, strength);
1657 }
1658
1659 ListList[src->resultLen].baseCE = CE;
1660 ListList[src->resultLen].baseContCE = SecondCE;
1661 ListList[src->resultLen].nextCE = 0;
1662 ListList[src->resultLen].nextContCE = 0;
1663
1664 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
1665 }
1666 }
1667
1668
1669 /* 5 If the relation is a reset:
1670 If sourceToken is null
1671 Create new list, create new sourceToken, make the baseCE from source, put
1672 the sourceToken in ListHeader of the new list */
1673 if(sourceToken == NULL) {
1674 /*
1675 3 Consider each item: relation, source, and expansion: e.g. ...< x / y ...
1676 First convert all expansions into normal form. Examples:
1677 If "xy" doesn't occur earlier in the list or in the UCA, convert &xy * c *
1678 d * ... into &x * c/y * d * ...
1679 Note: reset values can never have expansions, although they can cause the
1680 very next item to have one. They may be contractions, if they are found
1681 earlier in the list.
1682 */
1683 if(top == FALSE) {
1684 collIterate s;
1685 uint32_t CE = UCOL_NOT_FOUND, SecondCE = UCOL_NOT_FOUND;
1686
1687 uprv_init_collIterate(src->UCA, src->source+src->parsedToken.charsOffset, src->parsedToken.charsLen, &s);
1688
1689 CE = ucol_getNextCE(src->UCA, &s, status);
1690 UChar *expand = s.pos;
1691 SecondCE = ucol_getNextCE(src->UCA, &s, status);
1692
1693 ListList[src->resultLen].baseCE = CE & 0xFFFFFF3F;
1694 if(isContinuation(SecondCE)) {
1695 ListList[src->resultLen].baseContCE = SecondCE;
1696 } else {
1697 ListList[src->resultLen].baseContCE = 0;
1698 }
1699 ListList[src->resultLen].nextCE = 0;
1700 ListList[src->resultLen].nextContCE = 0;
1701 ListList[src->resultLen].previousCE = 0;
1702 ListList[src->resultLen].previousContCE = 0;
1703 ListList[src->resultLen].indirect = FALSE;
1704 sourceToken = ucol_tok_initAReset(src, expand, &expandNext, parseError, status);
1705 } else { /* top == TRUE */
1706 /* just use the supplied values */
1707 top = FALSE;
1708 ListList[src->resultLen].previousCE = 0;
1709 ListList[src->resultLen].previousContCE = 0;
1710 ListList[src->resultLen].indirect = TRUE;
1711 ListList[src->resultLen].baseCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE;
1712 ListList[src->resultLen].baseContCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE;
1713 ListList[src->resultLen].nextCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].limitCE;
1714 ListList[src->resultLen].nextContCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].limitContCE;
1715
1716 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
1717
1718 }
1719 } else { /* reset to something already in rules */
1720 top = FALSE;
1721 }
1722 }
1723 /* 7 After all this, set LAST to point to sourceToken, and goto step 3. */
1724 lastToken = sourceToken;
1725 } else {
1726 if(U_FAILURE(*status)) {
1727 return 0;
1728 }
1729 }
1730 }
1731
1732 if(src->resultLen > 0 && ListList[src->resultLen-1].first == NULL) {
1733 src->resultLen--;
1734 }
1735 return src->resultLen;
1736 }
1737
1738 void ucol_tok_initTokenList(UColTokenParser *src, const UChar *rules, const uint32_t rulesLength, const UCollator *UCA, UErrorCode *status) {
1739 U_NAMESPACE_USE
1740
1741 uint32_t nSize = 0;
1742 uint32_t estimatedSize = (2*rulesLength+UCOL_TOK_EXTRA_RULE_SPACE_SIZE);
1743 if(U_FAILURE(*status)) {
1744 return;
1745 }
1746
1747 // set everything to zero, so that we can clean up gracefully
1748 uprv_memset(src, 0, sizeof(UColTokenParser));
1749
1750 // first we need to find options that don't like to be normalized,
1751 // like copy and remove...
1752 //const UChar *openBrace = rules;
1753 int32_t optionNumber = -1;
1754 const UChar *setStart;
1755 uint32_t i = 0;
1756 while(i < rulesLength) {
1757 if(rules[i] == 0x005B) {
1758 // while((openBrace = u_strchr(openBrace, 0x005B)) != NULL) { // find open braces
1759 //optionNumber = ucol_uprv_tok_readOption(openBrace+1, rules+rulesLength, &setStart);
1760 optionNumber = ucol_uprv_tok_readOption(rules+i+1, rules+rulesLength, &setStart);
1761 if(optionNumber == OPTION_OPTIMIZE) { /* copy - parts of UCA to tailoring */
1762 USet *newSet = ucol_uprv_tok_readAndSetUnicodeSet(setStart, rules+rulesLength, status);
1763 if(U_SUCCESS(*status)) {
1764 if(src->copySet == NULL) {
1765 src->copySet = newSet;
1766 } else {
1767 uset_addAll(src->copySet, newSet);
1768 uset_close(newSet);
1769 }
1770 } else {
1771 return;
1772 }
1773 } else if(optionNumber == OPTION_SUPPRESS_CONTRACTIONS) {
1774 USet *newSet = ucol_uprv_tok_readAndSetUnicodeSet(setStart, rules+rulesLength, status);
1775 if(U_SUCCESS(*status)) {
1776 if(src->removeSet == NULL) {
1777 src->removeSet = newSet;
1778 } else {
1779 uset_addAll(src->removeSet, newSet);
1780 uset_close(newSet);
1781 }
1782 } else {
1783 return;
1784 }
1785 }
1786 }
1787 //openBrace++;
1788 i++;
1789 }
1790
1791 src->source = (UChar *)uprv_malloc(estimatedSize*sizeof(UChar));
1792 /* test for NULL */
1793 if (src->source == NULL) {
1794 *status = U_MEMORY_ALLOCATION_ERROR;
1795 return;
1796 }
1797 uprv_memset(src->source, 0, estimatedSize*sizeof(UChar));
1798 nSize = unorm_normalize(rules, rulesLength, UNORM_NFD, 0, src->source, estimatedSize, status);
1799 if(nSize > estimatedSize || *status == U_BUFFER_OVERFLOW_ERROR) {
1800 *status = U_ZERO_ERROR;
1801 src->source = (UChar *)uprv_realloc(src->source, (nSize+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar));
1802 /* test for NULL */
1803 if (src->source == NULL) {
1804 *status = U_MEMORY_ALLOCATION_ERROR;
1805 return;
1806 }
1807 nSize = unorm_normalize(rules, rulesLength, UNORM_NFD, 0, src->source, nSize+UCOL_TOK_EXTRA_RULE_SPACE_SIZE, status);
1808 }
1809 src->current = src->source;
1810 src->end = src->source+nSize;
1811 src->sourceCurrent = src->source;
1812 src->extraCurrent = src->end+1; // Preserve terminating zero in the rule string so that option scanning works correctly
1813 src->extraEnd = src->source+estimatedSize; //src->end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE;
1814 src->varTop = NULL;
1815 src->UCA = UCA;
1816 src->invUCA = ucol_initInverseUCA(status);
1817 src->parsedToken.charsLen = 0;
1818 src->parsedToken.charsOffset = 0;
1819 src->parsedToken.extensionLen = 0;
1820 src->parsedToken.extensionOffset = 0;
1821 src->parsedToken.prefixLen = 0;
1822 src->parsedToken.prefixOffset = 0;
1823 src->parsedToken.flags = 0;
1824 src->parsedToken.strength = UCOL_TOK_UNSET;
1825 src->buildCCTabFlag = FALSE;
1826
1827 if(U_FAILURE(*status)) {
1828 return;
1829 }
1830 src->tailored = uhash_open(uhash_hashTokens, uhash_compareTokens, NULL, status);
1831 if(U_FAILURE(*status)) {
1832 return;
1833 }
1834 uhash_setValueDeleter(src->tailored, uhash_freeBlock);
1835
1836 src->opts = (UColOptionSet *)uprv_malloc(sizeof(UColOptionSet));
1837 /* test for NULL */
1838 if (src->opts == NULL) {
1839 *status = U_MEMORY_ALLOCATION_ERROR;
1840 return;
1841 }
1842
1843 uprv_memcpy(src->opts, UCA->options, sizeof(UColOptionSet));
1844
1845 // rulesToParse = src->source;
1846 src->lh = 0;
1847 src->listCapacity = 1024;
1848 src->lh = (UColTokListHeader *)uprv_malloc(src->listCapacity*sizeof(UColTokListHeader));
1849 //Test for NULL
1850 if (src->lh == NULL) {
1851 *status = U_MEMORY_ALLOCATION_ERROR;
1852 return;
1853 }
1854 uprv_memset(src->lh, 0, src->listCapacity*sizeof(UColTokListHeader));
1855 src->resultLen = 0;
1856
1857 UCAConstants *consts = (UCAConstants *)((uint8_t *)src->UCA->image + src->UCA->image->UCAConsts);
1858
1859 // UCOL_RESET_TOP_VALUE
1860 setIndirectBoundaries(0, consts->UCA_LAST_NON_VARIABLE, consts->UCA_FIRST_IMPLICIT);
1861 // UCOL_FIRST_PRIMARY_IGNORABLE
1862 setIndirectBoundaries(1, consts->UCA_FIRST_PRIMARY_IGNORABLE, 0);
1863 // UCOL_LAST_PRIMARY_IGNORABLE
1864 setIndirectBoundaries(2, consts->UCA_LAST_PRIMARY_IGNORABLE, 0);
1865 // UCOL_FIRST_SECONDARY_IGNORABLE
1866 setIndirectBoundaries(3, consts->UCA_FIRST_SECONDARY_IGNORABLE, 0);
1867 // UCOL_LAST_SECONDARY_IGNORABLE
1868 setIndirectBoundaries(4, consts->UCA_LAST_SECONDARY_IGNORABLE, 0);
1869 // UCOL_FIRST_TERTIARY_IGNORABLE
1870 setIndirectBoundaries(5, consts->UCA_FIRST_TERTIARY_IGNORABLE, 0);
1871 // UCOL_LAST_TERTIARY_IGNORABLE
1872 setIndirectBoundaries(6, consts->UCA_LAST_TERTIARY_IGNORABLE, 0);
1873 // UCOL_FIRST_VARIABLE
1874 setIndirectBoundaries(7, consts->UCA_FIRST_VARIABLE, 0);
1875 // UCOL_LAST_VARIABLE
1876 setIndirectBoundaries(8, consts->UCA_LAST_VARIABLE, 0);
1877 // UCOL_FIRST_NON_VARIABLE
1878 setIndirectBoundaries(9, consts->UCA_FIRST_NON_VARIABLE, 0);
1879 // UCOL_LAST_NON_VARIABLE
1880 setIndirectBoundaries(10, consts->UCA_LAST_NON_VARIABLE, consts->UCA_FIRST_IMPLICIT);
1881 // UCOL_FIRST_IMPLICIT
1882 setIndirectBoundaries(11, consts->UCA_FIRST_IMPLICIT, 0);
1883 // UCOL_LAST_IMPLICIT
1884 setIndirectBoundaries(12, consts->UCA_LAST_IMPLICIT, consts->UCA_FIRST_TRAILING);
1885 // UCOL_FIRST_TRAILING
1886 setIndirectBoundaries(13, consts->UCA_FIRST_TRAILING, 0);
1887 // UCOL_LAST_TRAILING
1888 setIndirectBoundaries(14, consts->UCA_LAST_TRAILING, 0);
1889 ucolIndirectBoundaries[14].limitCE = (consts->UCA_PRIMARY_SPECIAL_MIN<<24);
1890 }
1891
1892
1893 void ucol_tok_closeTokenList(UColTokenParser *src) {
1894 if(src->copySet != NULL) {
1895 uset_close(src->copySet);
1896 }
1897 if(src->removeSet != NULL) {
1898 uset_close(src->removeSet);
1899 }
1900 if(src->tailored != NULL) {
1901 uhash_close(src->tailored);
1902 }
1903 if(src->lh != NULL) {
1904 uprv_free(src->lh);
1905 }
1906 if(src->source != NULL) {
1907 uprv_free(src->source);
1908 }
1909 if(src->opts != NULL) {
1910 uprv_free(src->opts);
1911 }
1912 }
1913
1914 #endif /* #if !UCONFIG_NO_COLLATION */
1915
mirror of ICU