icuSources/i18n/ucol_sit.cpp

   1 /*
   2 *******************************************************************************
   3 *   Copyright (C) 2004, International Business Machines
   4 *   Corporation and others.  All Rights Reserved.
   5 *******************************************************************************
   6 *   file name:  ucol_sit.cpp
   7 *   encoding:   US-ASCII
   8 *   tab size:   8 (not used)
   9 *   indentation:4
  10 *
  11 * Modification history
  12 * Date        Name      Comments
  13 * 03/12/2004  weiv      Creation
  14 */
  15
  16 #include "utracimp.h"
  17 #include "ucol_imp.h"
  18 #include "ucol_tok.h"
  19 #include "unormimp.h"
  20 #include "cmemory.h"
  21 #include "cstring.h"
  22
  23 #if !UCONFIG_NO_COLLATION
  24
  25 enum OptionsList {
  26     UCOL_SIT_LANGUAGE = 0,
  27     UCOL_SIT_SCRIPT,
  28     UCOL_SIT_REGION,
  29     UCOL_SIT_VARIANT,
  30     UCOL_SIT_KEYWORD,
  31     UCOL_SIT_RFC3166BIS,
  32     UCOL_SIT_STRENGTH,
  33     UCOL_SIT_CASE_LEVEL,
  34     UCOL_SIT_CASE_FIRST,
  35     UCOL_SIT_NUMERIC_COLLATION,
  36     UCOL_SIT_ALTERNATE_HANDLING,
  37     UCOL_SIT_NORMALIZATION_MODE,
  38     UCOL_SIT_FRENCH_COLLATION,
  39     UCOL_SIT_HIRAGANA_QUATERNARY,
  40     UCOL_SIT_VARIABLE_TOP,
  41     UCOL_SIT_VARIABLE_TOP_VALUE,
  42     UCOL_SIT_ITEMS_COUNT
  43 };
  44
  45 /* list of locales for packing of a collator to an integer.
  46  * This list corresponds to ICU 3.0. If more collation bearing
  47  * locales are added in the future, this won't be a simple array
  48  * but a mapping allowing forward and reverse lookup would have to
  49  * be established. Currently, the mapping is from locale name to
  50  * index.
  51  */
  52 static const char* const locales[] = {
  53 /* 00 - 09 */ "ar", "be", "bg", "ca", "cs", "da", "de", "de__PHONEBOOK", "el", "en",
  54 /* 10 - 19 */ "en_BE", "eo", "es", "es__TRADITIONAL", "et", "fa", "fa_AF", "fi", "fo", "fr",
  55 /* 20 - 29 */ "gu", "he", "hi", "hi__DIRECT", "hr", "hu", "is", "it", "ja", "kk",
  56 /* 30 - 39 */ "kl", "kn", "ko", "lt", "lv", "mk", "mr", "mt", "nb", "nn",
  57 /* 40 - 49 */ "om", "pa", "pl", "ps", "ro", "root", "ru", "sh", "sk", "sl",
  58 /* 50 - 59 */ "sq", "sr", "sv", "ta", "te", "th", "tr", "uk", "vi", "zh",
  59 /* 60 - 64 */ "zh_HK", "zh_MO", "zh_TW", "zh_TW_STROKE", "zh__PINYIN"
  60 };
  61
  62 static const char* const keywords[] = {
  63 /* 00 */ "",
  64 /* 01 */ "direct",
  65 /* 02 */ "phonebook",
  66 /* 03 */ "pinyin",
  67 /* 04 */ "standard",
  68 /* 05 */ "stroke",
  69 /* 06 */ "traditional"
  70 };
  71
  72
  73 /* option starters chars. */
  74 static const char alternateHArg     = 'A';
  75 static const char variableTopValArg = 'B';
  76 static const char caseFirstArg      = 'C';
  77 static const char numericCollArg    = 'D';
  78 static const char caseLevelArg      = 'E';
  79 static const char frenchCollArg     = 'F';
  80 static const char hiraganaQArg      = 'H';
  81 static const char keywordArg        = 'K';
  82 static const char languageArg       = 'L';
  83 static const char normArg           = 'N';
  84 static const char regionArg         = 'R';
  85 static const char strengthArg       = 'S';
  86 static const char variableTopArg    = 'T';
  87 static const char variantArg        = 'V';
  88 static const char RFC3066Arg        = 'X';
  89 static const char scriptArg         = 'Z';
  90
  91 static const char collationKeyword[]  = "@collation=";
  92
  93 static const int32_t locElementCount = 5;
  94 static const int32_t locElementCapacity = 32;
  95 static const int32_t loc3066Capacity = 256;
  96 static const int32_t internalBufferSize = 512;
  97
  98 /* structure containing specification of a collator. Initialized
  99  * from a short string. Also used to construct a short string from a
 100  * collator instance
 101  */
 102 struct CollatorSpec {
 103     char locElements[locElementCount][locElementCapacity];
 104     char locale[loc3066Capacity];
 105     UColAttributeValue options[UCOL_ATTRIBUTE_COUNT];
 106     uint32_t variableTopValue;
 107     UChar variableTopString[locElementCapacity];
 108     int32_t variableTopStringLen;
 109     UBool variableTopSet;
 110     struct {
 111         const char *start;
 112         int32_t len;
 113     } entries[UCOL_SIT_ITEMS_COUNT];
 114 };
 115
 116
 117 /* structure for converting between character attribute
 118  * representation and real collation attribute value.
 119  */
 120 struct AttributeConversion {
 121     char letter;
 122     UColAttributeValue value;
 123 };
 124
 125 static const AttributeConversion conversions[12] = {
 126     { '1', UCOL_PRIMARY },
 127     { '2', UCOL_SECONDARY },
 128     { '3', UCOL_TERTIARY },
 129     { '4', UCOL_QUATERNARY },
 130     { 'D', UCOL_DEFAULT },
 131     { 'I', UCOL_IDENTICAL },
 132     { 'L', UCOL_LOWER_FIRST },
 133     { 'N', UCOL_NON_IGNORABLE },
 134     { 'O', UCOL_ON },
 135     { 'S', UCOL_SHIFTED },
 136     { 'U', UCOL_UPPER_FIRST },
 137     { 'X', UCOL_OFF }
 138 };
 139
 140
 141 static char
 142 ucol_sit_attributeValueToLetter(UColAttributeValue value, UErrorCode *status) {
 143     uint32_t i = 0;
 144     for(i = 0; i < sizeof(conversions)/sizeof(conversions[0]); i++) {
 145         if(conversions[i].value == value) {
 146             return conversions[i].letter;
 147         }
 148     }
 149     *status = U_ILLEGAL_ARGUMENT_ERROR;
 150     return 0;
 151 }
 152
 153 static UColAttributeValue
 154 ucol_sit_letterToAttributeValue(char letter, UErrorCode *status) {
 155     uint32_t i = 0;
 156     for(i = 0; i < sizeof(conversions)/sizeof(conversions[0]); i++) {
 157         if(conversions[i].letter == letter) {
 158             return conversions[i].value;
 159         }
 160     }
 161     *status = U_ILLEGAL_ARGUMENT_ERROR;
 162     return UCOL_DEFAULT;
 163 }
 164
 165 /* function prototype for functions used to parse a short string */
 166 U_CDECL_BEGIN
 167 typedef const char* U_CALLCONV
 168 ActionFunction(CollatorSpec *spec, uint32_t value1, const char* string,
 169                UErrorCode *status);
 170 U_CDECL_END
 171
 172 U_CDECL_BEGIN
 173 static const char* U_CALLCONV
 174 _processLocaleElement(CollatorSpec *spec, uint32_t value, const char* string,
 175                       UErrorCode *status)
 176 {
 177     int32_t len = 0;
 178     do {
 179         if(value == 0 || value == 4) {
 180             spec->locElements[value][len++] = uprv_tolower(*string);
 181         } else {
 182             spec->locElements[value][len++] = *string;
 183         }
 184     } while(*(++string) != '_' && *string && len < locElementCapacity);
 185     if(len >= locElementCapacity) {
 186         *status = U_BUFFER_OVERFLOW_ERROR;
 187         return string;
 188     }
 189     // don't skip the underscore at the end
 190     return string;
 191 }
 192 U_CDECL_END
 193
 194 U_CDECL_BEGIN
 195 static const char* U_CALLCONV
 196 _processRFC3066Locale(CollatorSpec *spec, uint32_t, const char* string,
 197                       UErrorCode *status)
 198 {
 199     char terminator = *string;
 200     string++;
 201     const char *end = uprv_strchr(string+1, terminator);
 202     if(end == NULL || end - string >= loc3066Capacity) {
 203         *status = U_BUFFER_OVERFLOW_ERROR;
 204         return string;
 205     } else {
 206         uprv_strncpy(spec->locale, string, end-string);
 207         return end+1;
 208     }
 209 }
 210
 211 U_CDECL_END
 212
 213 U_CDECL_BEGIN
 214 static const char* U_CALLCONV
 215 _processCollatorOption(CollatorSpec *spec, uint32_t option, const char* string,
 216                        UErrorCode *status)
 217 {
 218     spec->options[option] = ucol_sit_letterToAttributeValue(*string, status);
 219     if((*(++string) != '_' && *string) || U_FAILURE(*status)) {
 220         *status = U_ILLEGAL_ARGUMENT_ERROR;
 221     }
 222     return string;
 223 }
 224 U_CDECL_END
 225
 226
 227 static UChar
 228 readHexCodeUnit(const char **string, UErrorCode *status)
 229 {
 230     UChar result = 0;
 231     int32_t value = 0;
 232     char c;
 233     int32_t noDigits = 0;
 234     while((c = **string) != 0 && noDigits < 4) {
 235         if( c >= '0' && c <= '9') {
 236             value = c - '0';
 237         } else if ( c >= 'a' && c <= 'f') {
 238             value = c - 'a' + 10;
 239         } else if ( c >= 'A' && c <= 'F') {
 240             value = c - 'A' + 10;
 241         } else {
 242             *status = U_ILLEGAL_ARGUMENT_ERROR;
 243             return 0;
 244         }
 245         result = (result << 4) | (UChar)value;
 246         noDigits++;
 247         (*string)++;
 248     }
 249     // if the string was terminated before we read 4 digits, set an error
 250     if(noDigits < 4) {
 251         *status = U_ILLEGAL_ARGUMENT_ERROR;
 252     }
 253     return result;
 254 }
 255
 256 U_CDECL_BEGIN
 257 static const char* U_CALLCONV
 258 _processVariableTop(CollatorSpec *spec, uint32_t value1, const char* string, UErrorCode *status)
 259 {
 260     // get four digits
 261     int32_t i = 0;
 262     if(!value1) {
 263         while(U_SUCCESS(*status) && i < locElementCapacity && *string != 0 && *string != '_') {
 264             spec->variableTopString[i++] = readHexCodeUnit(&string, status);
 265         }
 266         spec->variableTopStringLen = i;
 267         if(i == locElementCapacity && (*string != 0 || *string != '_')) {
 268             *status = U_BUFFER_OVERFLOW_ERROR;
 269         }
 270     } else {
 271         spec->variableTopValue = readHexCodeUnit(&string, status);
 272     }
 273     if(U_SUCCESS(*status)) {
 274         spec->variableTopSet = TRUE;
 275     }
 276     return string;
 277 }
 278 U_CDECL_END
 279
 280
 281 /* Table for parsing short strings */
 282 struct ShortStringOptions {
 283     char optionStart;
 284     ActionFunction *action;
 285     uint32_t attr;
 286 };
 287
 288 static const ShortStringOptions options[UCOL_SIT_ITEMS_COUNT] =
 289 {
 290 /* 10 ALTERNATE_HANDLING */   {alternateHArg,     _processCollatorOption, UCOL_ALTERNATE_HANDLING }, // alternate  N, S, D
 291 /* 15 VARIABLE_TOP_VALUE */   {variableTopValArg, _processVariableTop,    1 },
 292 /* 08 CASE_FIRST */           {caseFirstArg,      _processCollatorOption, UCOL_CASE_FIRST }, // case first L, U, X, D
 293 /* 09 NUMERIC_COLLATION */    {numericCollArg,    _processCollatorOption, UCOL_NUMERIC_COLLATION }, // codan      O, X, D
 294 /* 07 CASE_LEVEL */           {caseLevelArg,      _processCollatorOption, UCOL_CASE_LEVEL }, // case level O, X, D
 295 /* 12 FRENCH_COLLATION */     {frenchCollArg,     _processCollatorOption, UCOL_FRENCH_COLLATION }, // french     O, X, D
 296 /* 13 HIRAGANA_QUATERNARY] */ {hiraganaQArg,      _processCollatorOption, UCOL_HIRAGANA_QUATERNARY_MODE }, // hiragana   O, X, D
 297 /* 04 KEYWORD */              {keywordArg,        _processLocaleElement,  4 }, // keyword
 298 /* 00 LANGUAGE */             {languageArg,       _processLocaleElement,  0 }, // language
 299 /* 11 NORMALIZATION_MODE */   {normArg,           _processCollatorOption, UCOL_NORMALIZATION_MODE }, // norm       O, X, D
 300 /* 02 REGION */               {regionArg,         _processLocaleElement,  2 }, // region
 301 /* 06 STRENGTH */             {strengthArg,       _processCollatorOption, UCOL_STRENGTH }, // strength   1, 2, 3, 4, I, D
 302 /* 14 VARIABLE_TOP */         {variableTopArg,    _processVariableTop,    0 },
 303 /* 03 VARIANT */              {variantArg,        _processLocaleElement,  3 }, // variant
 304 /* 05 RFC3066BIS */           {RFC3066Arg,        _processRFC3066Locale,  0 }, // rfc3066bis locale name
 305 /* 01 SCRIPT */               {scriptArg,         _processLocaleElement,  1 }  // script
 306 };
 307
 308
 309 static
 310 const char* ucol_sit_readOption(const char *start, CollatorSpec *spec,
 311                             UErrorCode *status)
 312 {
 313   int32_t i = 0;
 314
 315   for(i = 0; i < UCOL_SIT_ITEMS_COUNT; i++) {
 316       if(*start == options[i].optionStart) {
 317           spec->entries[i].start = start;
 318           const char* end = options[i].action(spec, options[i].attr, start+1, status);
 319           spec->entries[i].len = end - start;
 320           return end;
 321       }
 322   }
 323   *status = U_ILLEGAL_ARGUMENT_ERROR;
 324   return start;
 325 }
 326
 327 static
 328 void ucol_sit_initCollatorSpecs(CollatorSpec *spec)
 329 {
 330     // reset everything
 331     uprv_memset(spec, 0, sizeof(CollatorSpec));
 332     // set collation options to default
 333     int32_t i = 0;
 334     for(i = 0; i < UCOL_ATTRIBUTE_COUNT; i++) {
 335         spec->options[i] = UCOL_DEFAULT;
 336     }
 337 }
 338
 339 static const char*
 340 ucol_sit_readSpecs(CollatorSpec *s, const char *string,
 341                         UParseError *parseError, UErrorCode *status)
 342 {
 343     const char *definition = string;
 344     while(U_SUCCESS(*status) && *string) {
 345         string = ucol_sit_readOption(string, s, status);
 346         // advance over '_'
 347         while(*string && *string == '_') {
 348             string++;
 349         }
 350     }
 351     if(U_FAILURE(*status)) {
 352         parseError->offset = string - definition;
 353     }
 354     return string;
 355 }
 356
 357 static
 358 int32_t ucol_sit_dumpSpecs(CollatorSpec *s, char *destination, int32_t capacity, UErrorCode *status)
 359 {
 360     int32_t i = 0, j = 0;
 361     int32_t len = 0;
 362     char optName;
 363     if(U_SUCCESS(*status)) {
 364         for(i = 0; i < UCOL_SIT_ITEMS_COUNT; i++) {
 365             if(s->entries[i].start) {
 366                 if(len) {
 367                     if(len < capacity) {
 368                         uprv_strcat(destination, "_");
 369                     }
 370                     len++;
 371                 }
 372                 optName = *(s->entries[i].start);
 373                 if(optName == languageArg || optName == regionArg || optName == variantArg || optName == keywordArg) {
 374                     for(j = 0; j < s->entries[i].len; j++) {
 375                         if(len + j < capacity) {
 376                             destination[len+j] = uprv_toupper(*(s->entries[i].start+j));
 377                         }
 378                     }
 379                     len += s->entries[i].len;
 380                 } else {
 381                     len += s->entries[i].len;
 382                     if(len < capacity) {
 383                         uprv_strncat(destination,s->entries[i].start, s->entries[i].len);
 384                     }
 385                 }
 386             }
 387         }
 388         return len;
 389     } else {
 390         return 0;
 391     }
 392 }
 393
 394 static void
 395 ucol_sit_calculateWholeLocale(CollatorSpec *s) {
 396     // put the locale together, unless we have a done
 397     // locale
 398     if(s->locale[0] == 0) {
 399         // first the language
 400         uprv_strcat(s->locale, s->locElements[0]);
 401         // then the script, if present
 402         if(*(s->locElements[1])) {
 403             uprv_strcat(s->locale, "_");
 404             uprv_strcat(s->locale, s->locElements[1]);
 405         }
 406         // then the region, if present
 407         if(*(s->locElements[2])) {
 408             uprv_strcat(s->locale, "_");
 409             uprv_strcat(s->locale, s->locElements[2]);
 410         } else if(*(s->locElements[3])) { // if there is a variant, we need an underscore
 411             uprv_strcat(s->locale, "_");
 412         }
 413         // add variant, if there
 414         if(*(s->locElements[3])) {
 415             uprv_strcat(s->locale, "_");
 416             uprv_strcat(s->locale, s->locElements[3]);
 417         }
 418
 419         // if there is a collation keyword, add that too
 420         if(*(s->locElements[4])) {
 421             uprv_strcat(s->locale, collationKeyword);
 422             uprv_strcat(s->locale, s->locElements[4]);
 423         }
 424     }
 425 }
 426
 427 U_CAPI UCollator* U_EXPORT2
 428 ucol_openFromShortString( const char *definition,
 429                           UBool forceDefaults,
 430                           UParseError *parseError,
 431                           UErrorCode *status)
 432 {
 433     UTRACE_ENTRY_OC(UTRACE_UCOL_OPEN_FROM_SHORT_STRING);
 434     UTRACE_DATA1(UTRACE_INFO, "short string = \"%s\"", definition);
 435
 436     if(U_FAILURE(*status)) return 0;
 437
 438     UParseError internalParseError;
 439
 440     if(!parseError) {
 441         parseError = &internalParseError;
 442     }
 443     parseError->line = 0;
 444     parseError->offset = 0;
 445     parseError->preContext[0] = 0;
 446     parseError->postContext[0] = 0;
 447
 448
 449     // first we want to pick stuff out of short string.
 450     // we'll end up with an UCA version, locale and a bunch of
 451     // settings
 452
 453     // analyse the string in order to get everything we need.
 454     const char *string = definition;
 455     CollatorSpec s;
 456     ucol_sit_initCollatorSpecs(&s);
 457     string = ucol_sit_readSpecs(&s, definition, parseError, status);
 458     ucol_sit_calculateWholeLocale(&s);
 459
 460     char buffer[internalBufferSize];
 461     uprv_memset(buffer, 0, internalBufferSize);
 462     uloc_canonicalize(s.locale, buffer, internalBufferSize, status);
 463
 464     UCollator *result = ucol_open(s.locale, status);
 465     int32_t i = 0;
 466
 467     for(i = 0; i < UCOL_ATTRIBUTE_COUNT; i++) {
 468         if(s.options[i] != UCOL_DEFAULT) {
 469             if(forceDefaults || ucol_getAttribute(result, (UColAttribute)i, status) != s.options[i]) {
 470                 ucol_setAttribute(result, (UColAttribute)i, s.options[i], status);
 471             }
 472
 473             if(U_FAILURE(*status)) {
 474                 parseError->offset = string - definition;
 475                 ucol_close(result);
 476                 return NULL;
 477             }
 478
 479         }
 480     }
 481     if(s.variableTopSet) {
 482         if(s.variableTopString[0]) {
 483             ucol_setVariableTop(result, s.variableTopString, s.variableTopStringLen, status);
 484         } else { // we set by value, using 'B'
 485             ucol_restoreVariableTop(result, s.variableTopValue, status);
 486         }
 487     }
 488
 489
 490     if(U_FAILURE(*status)) { // here it can only be a bogus value
 491         ucol_close(result);
 492         result = NULL;
 493     }
 494
 495     UTRACE_EXIT_PTR_STATUS(result, *status);
 496     return result;
 497 }
 498
 499
 500 static void appendShortStringElement(const char *src, int32_t len, char *result, int32_t *resultSize, int32_t capacity, char arg)
 501 {
 502     if(len) {
 503         if(*resultSize) {
 504             if(*resultSize < capacity) {
 505                 uprv_strcat(result, "_");
 506             }
 507             (*resultSize)++;
 508         }
 509         *resultSize += len + 1;
 510         if(*resultSize < capacity) {
 511             uprv_strncat(result, &arg, 1);
 512             uprv_strncat(result, src, len);
 513         }
 514     }
 515 }
 516
 517 U_CAPI int32_t U_EXPORT2
 518 ucol_getShortDefinitionString(const UCollator *coll,
 519                               const char *locale,
 520                               char *dst,
 521                               int32_t capacity,
 522                               UErrorCode *status)
 523 {
 524     if(U_FAILURE(*status)) return 0;
 525     char buffer[internalBufferSize];
 526     uprv_memset(buffer, 0, internalBufferSize*sizeof(char));
 527     int32_t resultSize = 0;
 528     char tempbuff[internalBufferSize];
 529     char locBuff[internalBufferSize];
 530     uprv_memset(buffer, 0, internalBufferSize*sizeof(char));
 531     int32_t elementSize = 0;
 532     UBool isAvailable = 0;
 533     CollatorSpec s;
 534     ucol_sit_initCollatorSpecs(&s);
 535
 536     if(!locale) {
 537         locale = ucol_getLocale(coll, ULOC_VALID_LOCALE, status);
 538     }
 539     elementSize = ucol_getFunctionalEquivalent(locBuff, internalBufferSize, "collation", locale, &isAvailable, status);
 540
 541     if(elementSize) {
 542         // we should probably canonicalize here...
 543         elementSize = uloc_getLanguage(locBuff, tempbuff, internalBufferSize, status);
 544         appendShortStringElement(tempbuff, elementSize, buffer, &resultSize, capacity, languageArg);
 545         elementSize = uloc_getCountry(locBuff, tempbuff, internalBufferSize, status);
 546         appendShortStringElement(tempbuff, elementSize, buffer, &resultSize, capacity, regionArg);
 547         elementSize = uloc_getScript(locBuff, tempbuff, internalBufferSize, status);
 548         appendShortStringElement(tempbuff, elementSize, buffer, &resultSize, capacity, scriptArg);
 549         elementSize = uloc_getVariant(locBuff, tempbuff, internalBufferSize, status);
 550         appendShortStringElement(tempbuff, elementSize, buffer, &resultSize, capacity, variantArg);
 551         elementSize = uloc_getKeywordValue(locBuff, "collation", tempbuff, internalBufferSize, status);
 552         appendShortStringElement(tempbuff, elementSize, buffer, &resultSize, capacity, keywordArg);
 553     }
 554
 555     int32_t i = 0;
 556     UColAttributeValue attribute = UCOL_DEFAULT;
 557     for(i = 0; i < UCOL_SIT_ITEMS_COUNT; i++) {
 558         if(options[i].action == _processCollatorOption) {
 559             attribute = ucol_getAttributeOrDefault(coll, (UColAttribute)options[i].attr, status);
 560             if(attribute != UCOL_DEFAULT) {
 561                 char letter = ucol_sit_attributeValueToLetter(attribute, status);
 562                 appendShortStringElement(&letter, 1,
 563                     buffer, &resultSize, capacity, options[i].optionStart);
 564             }
 565         }
 566     }
 567     if(coll->variableTopValueisDefault == FALSE) {
 568         //s.variableTopValue = ucol_getVariableTop(coll, status);
 569         elementSize = T_CString_integerToString(tempbuff, coll->variableTopValue, 16);
 570         appendShortStringElement(tempbuff, elementSize, buffer, &resultSize, capacity, variableTopValArg);
 571     }
 572
 573     UParseError parseError;
 574     return ucol_normalizeShortDefinitionString(buffer, dst, capacity, &parseError, status);
 575 }
 576
 577 U_CAPI int32_t U_EXPORT2
 578 ucol_normalizeShortDefinitionString(const char *definition,
 579                                     char *destination,
 580                                     int32_t capacity,
 581                                     UParseError *parseError,
 582                                     UErrorCode *status)
 583 {
 584
 585     if(U_FAILURE(*status)) {
 586         return 0;
 587     }
 588
 589     if(destination) {
 590         uprv_memset(destination, 0, capacity*sizeof(char));
 591     }
 592
 593     UParseError pe;
 594     if(!parseError) {
 595         parseError = &pe;
 596     }
 597
 598     // validate
 599     CollatorSpec s;
 600     ucol_sit_initCollatorSpecs(&s);
 601     ucol_sit_readSpecs(&s, definition, parseError, status);
 602     return ucol_sit_dumpSpecs(&s, destination, capacity, status);
 603 }
 604
 605 // structure for packing the bits of the attributes in the
 606 // identifier number.
 607 // locale is packed separately
 608 struct bitPacking {
 609     char letter;
 610     uint32_t offset;
 611     uint32_t width;
 612     UColAttribute attribute;
 613     UColAttributeValue values[6];
 614 };
 615
 616 static const bitPacking attributesToBits[UCOL_ATTRIBUTE_COUNT] = {
 617     /* french */        { frenchCollArg,    29, 2, UCOL_FRENCH_COLLATION,         { UCOL_DEFAULT, UCOL_OFF, UCOL_ON }},
 618     /* alternate */     { alternateHArg,    27, 2, UCOL_ALTERNATE_HANDLING,       { UCOL_DEFAULT, UCOL_NON_IGNORABLE, UCOL_SHIFTED }},
 619     /* case first */    { caseFirstArg,     25, 2, UCOL_CASE_FIRST,               { UCOL_DEFAULT, UCOL_OFF, UCOL_LOWER_FIRST, UCOL_UPPER_FIRST }},
 620     /* case level */    { caseLevelArg,     23, 2, UCOL_CASE_LEVEL,               { UCOL_DEFAULT, UCOL_OFF, UCOL_ON }},
 621     /* normalization */ { normArg,          21, 2, UCOL_NORMALIZATION_MODE,       { UCOL_DEFAULT, UCOL_OFF, UCOL_ON }},
 622     /* strength */      { strengthArg,      18, 3, UCOL_STRENGTH,                 { UCOL_DEFAULT, UCOL_PRIMARY, UCOL_SECONDARY, UCOL_TERTIARY, UCOL_QUATERNARY, UCOL_IDENTICAL }},
 623     /* hiragana */      { hiraganaQArg,     16, 2, UCOL_HIRAGANA_QUATERNARY_MODE, { UCOL_DEFAULT, UCOL_OFF, UCOL_ON }},
 624     /* numeric coll */  { numericCollArg,   14, 2, UCOL_NUMERIC_COLLATION,        { UCOL_DEFAULT, UCOL_OFF, UCOL_ON }}
 625 };
 626
 627 static const uint32_t keywordShift =   9;
 628 static const uint32_t keywordWidth =   5;
 629 static const uint32_t localeShift =    0;
 630 static const uint32_t localeWidth =    7;
 631
 632
 633 static uint32_t ucol_sit_putLocaleInIdentifier(uint32_t result, const char* locale, UErrorCode* status) {
 634     char buffer[internalBufferSize], keywordBuffer[internalBufferSize],
 635         baseName[internalBufferSize], localeBuffer[internalBufferSize];
 636     int32_t len = 0, keywordLen = 0,
 637         baseNameLen = 0, localeLen = 0;
 638     uint32_t i = 0;
 639     UBool isAvailable = FALSE;
 640     if(locale) {
 641         len = uloc_canonicalize(locale, buffer, internalBufferSize, status);
 642         localeLen = ucol_getFunctionalEquivalent(localeBuffer, internalBufferSize, "collation", buffer, &isAvailable, status);
 643         keywordLen = uloc_getKeywordValue(buffer, "collation", keywordBuffer, internalBufferSize, status);
 644         baseNameLen = uloc_getBaseName(buffer, baseName, internalBufferSize, status);
 645
 646         /*Binary search for the map entry for normal cases */
 647
 648         uint32_t   low     = 0;
 649         uint32_t   high    = sizeof(locales)/sizeof(locales[0]);
 650         uint32_t   mid     = high;
 651         uint32_t   oldmid  = 0;
 652         int32_t    compVal = 0;
 653
 654
 655         while (high > low)  /*binary search*/{
 656
 657             mid = (high+low) >> 1; /*Finds median*/
 658
 659             if (mid == oldmid)
 660                 return UCOL_SIT_COLLATOR_NOT_ENCODABLE; // we didn't find it
 661
 662             compVal = uprv_strcmp(baseName, locales[mid]);
 663             if (compVal < 0){
 664                 high = mid;
 665             }
 666             else if (compVal > 0){
 667                 low = mid;
 668             }
 669             else /*we found it*/{
 670                 break;
 671             }
 672             oldmid = mid;
 673         }
 674
 675         result |= (mid & ((1 << localeWidth) - 1)) << localeShift;
 676     }
 677
 678     if(keywordLen) {
 679         for(i = 1; i < sizeof(keywords)/sizeof(keywords[0]); i++) {
 680             if(uprv_strcmp(keywords[i], keywordBuffer) == 0) {
 681                 result |= (i & ((1 << keywordWidth) - 1)) << keywordShift;
 682                 break;
 683             }
 684         }
 685     }
 686     return result;
 687 }
 688
 689 U_CAPI uint32_t U_EXPORT2
 690 ucol_collatorToIdentifier(const UCollator *coll,
 691                           const char *locale,
 692                           UErrorCode *status)
 693 {
 694     uint32_t result = 0;
 695     uint32_t i = 0, j = 0;
 696     UColAttributeValue attrValue = UCOL_DEFAULT;
 697
 698     // if variable top is not default, we need to use strings
 699     if(coll->variableTopValueisDefault != TRUE) {
 700         return UCOL_SIT_COLLATOR_NOT_ENCODABLE;
 701     }
 702
 703     if(locale == NULL) {
 704         locale = ucol_getLocale(coll, ULOC_VALID_LOCALE, status);
 705     }
 706
 707     result = ucol_sit_putLocaleInIdentifier(result, locale, status);
 708
 709     for(i = 0; i < sizeof(attributesToBits)/sizeof(attributesToBits[0]); i++) {
 710         attrValue = ucol_getAttributeOrDefault(coll, attributesToBits[i].attribute, status);
 711         j = 0;
 712         while(attributesToBits[i].values[j] != attrValue) {
 713             j++;
 714         }
 715         result |= (j & ((1 << attributesToBits[i].width) - 1)) << attributesToBits[i].offset;
 716     }
 717
 718     return result;
 719 }
 720
 721 U_CAPI UCollator* U_EXPORT2
 722 ucol_openFromIdentifier(uint32_t identifier,
 723                         UBool forceDefaults,
 724                         UErrorCode *status)
 725 {
 726     uint32_t i = 0;
 727     int32_t value = 0, keyword = 0;
 728     char locale[internalBufferSize];
 729
 730     value = (identifier >> localeShift) & ((1 << localeWidth) - 1);
 731     keyword = (identifier >> keywordShift) & ((1 << keywordWidth) - 1);
 732
 733     uprv_strcpy(locale, locales[value]);
 734
 735     if(keyword) {
 736         uprv_strcat(locale, collationKeyword);
 737         uprv_strcat(locale, keywords[keyword]);
 738     }
 739
 740     UColAttributeValue attrValue = UCOL_DEFAULT;
 741
 742     UCollator *result = ucol_open(locale, status);
 743
 744     // variable top is not set in the identifier, so we can easily skip that on
 745
 746     for(i = 0; i < sizeof(attributesToBits)/sizeof(attributesToBits[0]); i++) {
 747         value = (identifier >> attributesToBits[i].offset) & ((1 << attributesToBits[i].width) - 1);
 748         attrValue = attributesToBits[i].values[value];
 749         // the collator is all default, so we will set only the values that will differ from
 750         // the default values.
 751         if(attrValue != UCOL_DEFAULT) {
 752             if(forceDefaults ||
 753                 ucol_getAttribute(result, attributesToBits[i].attribute, status) != attrValue) {
 754                 ucol_setAttribute(result, attributesToBits[i].attribute, attrValue, status);
 755             }
 756         }
 757     }
 758
 759     return result;
 760 }
 761
 762 U_CAPI int32_t U_EXPORT2
 763 ucol_identifierToShortString(uint32_t identifier,
 764                              char *buffer,
 765                              int32_t capacity,
 766                              UBool forceDefaults,
 767                              UErrorCode *status)
 768 {
 769     int32_t locIndex = (identifier >> localeShift) & ((1 << localeWidth) - 1);
 770     int32_t keywordIndex = (identifier >> keywordShift) & ((1 << keywordWidth) - 1);
 771     CollatorSpec s;
 772     ucol_sit_initCollatorSpecs(&s);
 773     uprv_strcpy(s.locale, locales[locIndex]);
 774     if(keywordIndex) {
 775         uprv_strcat(s.locale, collationKeyword);
 776         uprv_strcat(s.locale, keywords[keywordIndex]);
 777     }
 778     UCollator *coll = ucol_openFromIdentifier(identifier, forceDefaults, status);
 779     int32_t resultLen = ucol_getShortDefinitionString(coll, s.locale, buffer, capacity, status);
 780     ucol_close(coll);
 781     return resultLen;
 782
 783 #if 0
 784     // TODO: Crumy, crumy, crumy... Very hard to currently go algorithmically from
 785     // identifier to short string. Do rethink
 786     if(forceDefaults == FALSE) {
 787         UCollator *coll = ucol_openFromIdentifier(identifier, FALSE, status);
 788         int32_t resultLen = ucol_getShortDefinitionString(coll, s.locale, buffer, capacity, status);
 789         ucol_close(coll);
 790         return resultLen;
 791     } else { // forceDefaults == TRUE
 792         char letter;
 793         UColAttributeValue value;
 794         int32_t i = 0;
 795         for(i = 0; i < sizeof(attributesToBits)/sizeof(attributesToBits[0]); i++) {
 796             value = attributesToBits[i].values[(identifier >> attributesToBits[i].offset) & ((1 << attributesToBits[i].width) - 1)];
 797             if(value != UCOL_DEFAULT) {
 798                 uprv_strcat(buffer, "_");
 799                 uprv_strncat(buffer, &attributesToBits[i].letter, 1);
 800                 letter = ucol_sit_attributeValueToLetter(value, status);
 801                 uprv_strncat(buffer, &letter, 1);
 802             }
 803         }
 804         return ucol_sit_dumpSpecs(&s, buffer, capacity, status);
 805     }
 806 #endif
 807 }
 808
 809 U_CAPI uint32_t U_EXPORT2
 810 ucol_shortStringToIdentifier(const char *definition,
 811                              UBool forceDefaults,
 812                              UErrorCode *status)
 813 {
 814     UParseError parseError;
 815     CollatorSpec s;
 816     uint32_t result = 0;
 817     uint32_t i = 0, j = 0;
 818     ucol_sit_initCollatorSpecs(&s);
 819
 820     ucol_sit_readSpecs(&s, definition, &parseError, status);
 821     ucol_sit_calculateWholeLocale(&s);
 822
 823     char locBuffer[internalBufferSize];
 824     UBool isAvailable = FALSE;
 825     UColAttributeValue attrValue = UCOL_DEFAULT;
 826
 827     ucol_getFunctionalEquivalent(locBuffer, internalBufferSize, "collation", s.locale, &isAvailable, status);
 828
 829     if(forceDefaults == FALSE) {
 830         UCollator *coll = ucol_openFromShortString(definition, FALSE, &parseError, status);
 831         result = ucol_collatorToIdentifier(coll, locBuffer, status);
 832         ucol_close(coll);
 833     } else { // forceDefaults == TRUE
 834         result = ucol_sit_putLocaleInIdentifier(result, locBuffer, status);
 835
 836         for(i = 0; i < sizeof(attributesToBits)/sizeof(attributesToBits[0]); i++) {
 837             attrValue = s.options[i];
 838             j = 0;
 839             while(attributesToBits[i].values[j] != attrValue) {
 840                 j++;
 841             }
 842             result |= (j & ((1 << attributesToBits[i].width) - 1)) << attributesToBits[i].offset;
 843         }
 844
 845     }
 846     return result;
 847
 848 }
 849
 850 U_CAPI UColAttributeValue  U_EXPORT2
 851 ucol_getAttributeOrDefault(const UCollator *coll, UColAttribute attr, UErrorCode *status)
 852 {
 853     if(U_FAILURE(*status) || coll == NULL) {
 854       return UCOL_DEFAULT;
 855     }
 856     switch(attr) {
 857     case UCOL_NUMERIC_COLLATION:
 858         return coll->numericCollationisDefault?UCOL_DEFAULT:coll->numericCollation;
 859     case UCOL_HIRAGANA_QUATERNARY_MODE:
 860         return coll->hiraganaQisDefault?UCOL_DEFAULT:coll->hiraganaQ;
 861     case UCOL_FRENCH_COLLATION: /* attribute for direction of secondary weights*/
 862         return coll->frenchCollationisDefault?UCOL_DEFAULT:coll->frenchCollation;
 863     case UCOL_ALTERNATE_HANDLING: /* attribute for handling variable elements*/
 864         return coll->alternateHandlingisDefault?UCOL_DEFAULT:coll->alternateHandling;
 865     case UCOL_CASE_FIRST: /* who goes first, lower case or uppercase */
 866         return coll->caseFirstisDefault?UCOL_DEFAULT:coll->caseFirst;
 867     case UCOL_CASE_LEVEL: /* do we have an extra case level */
 868         return coll->caseLevelisDefault?UCOL_DEFAULT:coll->caseLevel;
 869     case UCOL_NORMALIZATION_MODE: /* attribute for normalization */
 870         return coll->normalizationModeisDefault?UCOL_DEFAULT:coll->normalizationMode;
 871     case UCOL_STRENGTH:         /* attribute for strength */
 872         return coll->strengthisDefault?UCOL_DEFAULT:coll->strength;
 873     case UCOL_ATTRIBUTE_COUNT:
 874     default:
 875         *status = U_ILLEGAL_ARGUMENT_ERROR;
 876         break;
 877     }
 878     return UCOL_DEFAULT;
 879 }
 880
 881
 882 struct contContext {
 883     const UCollator *coll;
 884     USet            *conts;
 885     USet            *removedContractions;
 886     UErrorCode      *status;
 887 };
 888
 889
 890
 891 static void
 892 addContraction(const UCollator *coll, USet *contractions, UChar *buffer, int32_t bufLen,
 893                uint32_t CE, int32_t rightIndex, UErrorCode *status)
 894 {
 895     if(rightIndex == bufLen-1) {
 896         *status = U_INTERNAL_PROGRAM_ERROR;
 897         return;
 898     }
 899     const UChar *UCharOffset = (UChar *)coll->image+getContractOffset(CE);
 900     uint32_t newCE = *(coll->contractionCEs + (UCharOffset - coll->contractionIndex));
 901     // we might have a contraction that ends from previous level
 902     if(newCE != UCOL_NOT_FOUND && rightIndex > 1) {
 903             uset_addString(contractions, buffer, rightIndex);
 904     }
 905
 906     UCharOffset++;
 907     while(*UCharOffset != 0xFFFF) {
 908         newCE = *(coll->contractionCEs + (UCharOffset - coll->contractionIndex));
 909         buffer[rightIndex] = *UCharOffset;
 910         if(isSpecial(newCE) && getCETag(newCE) == CONTRACTION_TAG) {
 911             addContraction(coll, contractions, buffer, bufLen, newCE, rightIndex + 1, status);
 912         } else {
 913             uset_addString(contractions, buffer, rightIndex + 1);
 914         }
 915         UCharOffset++;
 916     }
 917 }
 918
 919 U_CDECL_BEGIN
 920 static UBool U_CALLCONV
 921 _processContractions(const void *context, UChar32 start, UChar32 limit, uint32_t CE)
 922 {
 923     UErrorCode *status = ((contContext *)context)->status;
 924     USet *unsafe = ((contContext *)context)->conts;
 925     USet *removed = ((contContext *)context)->removedContractions;
 926     const UCollator *coll = ((contContext *)context)->coll;
 927     UChar contraction[internalBufferSize];
 928     if(isSpecial(CE) && getCETag(CE) == CONTRACTION_TAG) {
 929         while(start < limit && U_SUCCESS(*status)) {
 930             // if there are suppressed contractions, we don't
 931             // want to add them.
 932             if(removed && uset_contains(removed, start)) {
 933                 start++;
 934                 continue;
 935             }
 936             // we start our contraction from middle, since we don't know if it
 937             // will grow toward right or left
 938             contraction[0] = (UChar)start;
 939             addContraction(coll, unsafe, contraction, internalBufferSize, CE, 1, status);
 940             start++;
 941         }
 942     }
 943     if(U_FAILURE(*status)) {
 944         return FALSE;
 945     } else {
 946         return TRUE;
 947     }
 948 }
 949
 950 static int32_t U_CALLCONV
 951 _getTrieFoldingOffset(uint32_t data)
 952 {
 953     return (int32_t)(data&0xFFFFFF);
 954 }
 955 U_CDECL_END
 956
 957
 958
 959 /**
 960  * Get a set containing the contractions defined by the collator. The set includes
 961  * both the UCA contractions and the contractions defined by the collator
 962  * @param coll collator
 963  * @param conts the set to hold the result
 964  * @param status to hold the error code
 965  * @return the size of the contraction set
 966  *
 967  * @draft ICU 3.0
 968  */
 969 U_CAPI int32_t U_EXPORT2
 970 ucol_getContractions( const UCollator *coll,
 971                   USet *contractions,
 972                   UErrorCode *status)
 973 {
 974     if(U_FAILURE(*status)) {
 975         return 0;
 976     }
 977     if(coll == NULL || contractions == NULL) {
 978         *status = U_ILLEGAL_ARGUMENT_ERROR;
 979         return 0;
 980     }
 981
 982     uset_clear(contractions);
 983     int32_t rulesLen = 0;
 984     const UChar* rules = ucol_getRules(coll, &rulesLen);
 985     UColTokenParser src;
 986     ucol_tok_initTokenList(&src, rules, rulesLen, coll->UCA, status);
 987
 988     contContext c = { NULL, contractions, src.removeSet, status };
 989
 990     coll->mapping->getFoldingOffset = _getTrieFoldingOffset;
 991
 992     // TODO: if you're supressing contractions in the tailoring
 993     // you want to remove (or rather not include) contractions
 994     // from the UCA.
 995     // Probably want to pass a set of contraction starters that
 996     // are suppressed. However, we don't want a dependency on
 997     // the builder, so this is going to be hard to pull off.
 998
 999     // Add the UCA contractions
1000     c.coll = coll->UCA;
1001     utrie_enum(coll->UCA->mapping, NULL, _processContractions, &c);
1002
1003     // This is collator specific. Add contractions from a collator
1004     c.coll = coll;
1005     c.removedContractions =  NULL;
1006     utrie_enum(coll->mapping, NULL, _processContractions, &c);
1007     ucol_tok_closeTokenList(&src);
1008
1009     return uset_getItemCount(contractions);
1010
1011 }
1012
1013
1014 U_CAPI int32_t U_EXPORT2
1015 ucol_getUnsafeSet( const UCollator *coll,
1016                   USet *unsafe,
1017                   UErrorCode *status)
1018 {
1019     UChar buffer[internalBufferSize];
1020     int32_t len = 0;
1021
1022     uset_clear(unsafe);
1023
1024     // cccpattern = "[[:^tccc=0:][:^lccc=0:]]", unfortunately variant
1025     static const UChar cccpattern[25] = { 0x5b, 0x5b, 0x3a, 0x5e, 0x74, 0x63, 0x63, 0x63, 0x3d, 0x30, 0x3a, 0x5d,
1026                                     0x5b, 0x3a, 0x5e, 0x6c, 0x63, 0x63, 0x63, 0x3d, 0x30, 0x3a, 0x5d, 0x5d, 0x00 };
1027
1028     // add chars that fail the fcd check
1029     uset_applyPattern(unsafe, cccpattern, 24, USET_IGNORE_SPACE, status);
1030
1031     // add Thai/Lao prevowels
1032     uset_addRange(unsafe, 0xe40, 0xe44);
1033     uset_addRange(unsafe, 0xec0, 0xec4);
1034     // add lead/trail surrogates
1035     uset_addRange(unsafe, 0xd800, 0xdfff);
1036
1037     USet *contractions = uset_open(0,0);
1038
1039     int32_t i = 0, j = 0;
1040     int32_t contsSize = ucol_getContractions(coll, contractions, status);
1041     UChar32 c = 0;
1042     // Contraction set consists only of strings
1043     // to get unsafe code points, we need to
1044     // break the strings apart and add them to the unsafe set
1045     for(i = 0; i < contsSize; i++) {
1046         len = uset_getItem(contractions, i, NULL, NULL, buffer, internalBufferSize, status);
1047         if(len > 0) {
1048             j = 0;
1049             while(j < len) {
1050                 U16_NEXT(buffer, j, len, c);
1051                 if(j < len) {
1052                     uset_add(unsafe, c);
1053                 }
1054             }
1055         }
1056     }
1057
1058     uset_close(contractions);
1059
1060     return uset_size(unsafe);
1061 }
1062 #endif