icuSources/i18n/ucol_imp.h

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 1998-2004, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *
   9 * Private implementation header for C collation
  10 *   file name:  ucol_imp.h
  11 *   encoding:   US-ASCII
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2000dec11
  16 *   created by: Vladimir Weinstein
  17 *
  18 * Modification history
  19 * Date        Name      Comments
  20 * 02/16/2001  synwee    Added UCOL_GETPREVCE for the use in ucoleitr
  21 * 02/27/2001  synwee    Added getMaxExpansion data structure in UCollator
  22 * 03/02/2001  synwee    Added UCOL_IMPLICIT_CE
  23 * 03/12/2001  synwee    Added pointer start to collIterate.
  24 */
  25
  26 #ifndef UCOL_IMP_H
  27 #define UCOL_IMP_H
  28
  29 #include "unicode/utypes.h"
  30
  31 #define UCA_DATA_TYPE "icu"
  32 #define UCA_DATA_NAME "ucadata"
  33 #define INVC_DATA_TYPE "icu"
  34 #define INVC_DATA_NAME "invuca"
  35
  36 /**
  37  * Convenience string denoting the Collation data tree
  38  * @internal ICU 3.0
  39  */
  40 #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll"
  41
  42 #if !UCONFIG_NO_COLLATION
  43
  44 #include "unicode/ucol.h"
  45 #include "utrie.h"
  46 #include "unicode/ures.h"
  47 #include "unicode/udata.h"
  48 #include "unicode/uiter.h"
  49
  50 /* This is the internal header file which contains important declarations for
  51  * the collation framework.
  52  * Ready to use collators are stored as binary images. Both UCA and tailorings
  53  * share the same binary format. Individual files (currently only UCA) have a
  54  * udata header in front of the image and should be opened using udata_open.
  55  * Tailoring images are currently stored inside resource bundles and are intialized
  56  * through ucol_open API.
  57  *
  58  * The following describes the formats for collation binaries
  59  * (UCA & tailorings) and for the inverse UCA table.
  60  * Substructures are described in the collation design document at
  61  * http://oss.software.ibm.com/cvs/icu/~checkout~/icuhtml/design/collation/ICU_collation_design.htm
  62  *
  63  * -------------------------------------------------------------
  64  *
  65  * Here is the format of binary collation image.
  66  *
  67  * Physical order of structures:
  68  * - header (UCATableHeader)
  69  * - options (UColOptionSet)
  70  * - expansions (CE[])
  71  * - contractions (UChar[contractionSize] + CE[contractionSize])
  72  * - serialized UTrie with mappings of code points to CEs
  73  * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount])
  74  * - two bit sets for backward processing in strcoll (identical prefixes)
  75  *   and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE])
  76  * - UCA constants (UCAConstants)
  77  * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth])
  78  *
  79  * UCATableHeader fields:
  80  *
  81  * int32_t size; - image size in bytes
  82  *
  83  * Offsets to interesting data. All offsets are in bytes.
  84  * to get the address add to the header address and cast properly.
  85  * Some offsets are zero if the corresponding structures are empty.
  86  *
  87  * Tailoring binaries that only set options and contain no mappings etc.
  88  * will have all offsets 0 except for the options and expansion offsets,
  89  * which give the position and length of the options array.
  90  *
  91  * uint32_t options; - offset to default collator options (UColOptionSet *),
  92  *                     a set of 32-bit values. See declaration of UColOptionSet for more details
  93  *
  94  * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges
  95  *                       See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store
  96  *                       important constant values that are defined in the UCA and used for building and runtime.
  97  *
  98  * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth],
  99  *                                  containing contractions from the UCA. These are needed in the build process to copy UCA contractions
 100  *                                  in case the base contraction symbol is tailored.
 101  *
 102  * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3)
 103  *
 104  * uint32_t mappingPosition;  - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such.
 105  *                              Used as a primary lookup table for collation elements.
 106  *
 107  * uint32_t expansion;  - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0.
 108  *
 109  * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents
 110  *                              are aligned with the contents of contractionCEs table. 0 if no contractions.
 111  *
 112  * uint32_t contractionCEs;  - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the
 113  *                             in the contractionIndex table, the resulting index is used to look up corresponding CE in this table.
 114  *                             0 if no contractions.
 115  * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs).
 116  *
 117  * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion
 118  * and last CEs in expansions.
 119  * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *).
 120  *                            Never 0.
 121  * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *)
 122  * int32_t  endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION
 123  *                                 for the usage model
 124  *
 125  * These two offsets point to byte tables that are used in the backup heuristics.
 126  * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function.
 127  * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP.
 128  *
 129  * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos
 130  *                                     (formatVersion 2.3)
 131  * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition.
 132  * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3)
 133  * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3)
 134  * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3)
 135  *
 136  * Various version fields
 137  * UVersionInfo version; - version 4 uint8_t
 138  * UVersionInfo UCAVersion;  - version 4 uint8_t
 139  * UVersionInfo UCDVersion;  - version 4 uint8_t
 140  * UVersionInfo formatVersion; - version of the format of the collation binary
 141  *                               same formatVersion as in ucadata.icu's UDataInfo header
 142  *                               (formatVersion 2.3)
 143  *
 144  * uint8_t reserved[84];  - currently unused
 145  *
 146  * -------------------------------------------------------------
 147  *
 148  * Inverse UCA is used for constructing collators from rules. It is always an individual file
 149  * and always has a UDataInfo header.
 150  * here is the structure:
 151  *
 152  * uint32_t byteSize; - size of inverse UCA image in bytes
 153  * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows)
 154  * uint32_t contsSize; - size of continuation table (number of UChars in table)
 155  *
 156  * uint32_t table; - offset to inverse table (uint32_t *)
 157  *                   Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation
 158  *                   the third value is either a code unit (if there is only one code unit for element) or an index to continuation
 159  *                   (number of code units combined with an index).
 160  *                   table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final
 161  *                   code unit sequence for a CE is terminated by FFFE.
 162  * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE.
 163  *
 164  * UVersionInfo UCAVersion; -  version of the UCA, read from file 4 uint8_t
 165  * uint8_t padding[8]; - padding 8 uint8_t
 166  * Header is followed by the table and continuation table.
 167 */
 168
 169 /* let us know whether reserved fields are reset to zero or junked */
 170 #define UCOL_HEADER_MAGIC 0x20030618
 171
 172 /* UDataInfo for UCA mapping table */
 173 /* dataFormat="UCol"            */
 174 #define UCA_DATA_FORMAT_0 ((uint8_t)0x55)
 175 #define UCA_DATA_FORMAT_1 ((uint8_t)0x43)
 176 #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f)
 177 #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c)
 178
 179 #define UCA_FORMAT_VERSION_0 ((uint8_t)2)
 180 #define UCA_FORMAT_VERSION_1 ((uint8_t)3)
 181 #define UCA_FORMAT_VERSION_2 ((uint8_t)0)
 182 #define UCA_FORMAT_VERSION_3 ((uint8_t)0)
 183
 184 /* UDataInfo for inverse UCA table */
 185 /* dataFormat="InvC"            */
 186 #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49)
 187 #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E)
 188 #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76)
 189 #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43)
 190
 191 #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2)
 192 #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1)
 193 #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0)
 194 #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0)
 195
 196 /* This is the size of the stack allocated buffer for sortkey generation and similar operations */
 197 /* if it is too small, heap allocation will occur.*/
 198 /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */
 199 #define UCOL_MAX_BUFFER 128
 200 #define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER
 201 #define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER
 202 #define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER
 203 #define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4
 204 #define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER
 205
 206 #define UCOL_NORMALIZATION_GROWTH 2
 207 #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH
 208
 209 /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */
 210 /* Sometimes we already have a writable buffer (like in case of normalized strings). */
 211 /*
 212 you can change this value to any value >= 4 if you need memory -
 213 it will affect the performance, though, since we're going to malloc.
 214 Note 3 is the minimum value for Thai collation and 4 is the
 215 minimum number for special Jamo
 216 */
 217 #define UCOL_WRITABLE_BUFFER_SIZE 256
 218
 219 /* This is the size of the buffer for expansion CE's */
 220 /* In reality we should not have to deal with expm sequences longer then 16 */
 221 /* you can change this value if you need memory */
 222 /* WARNING THIS BUFFER DOES NOT HAVE MALLOC FALLBACK. If you make it too small, you'll get in trouble */
 223 /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */
 224 /* This is the longest expansion sequence we can handle without bombing out */
 225 #define UCOL_EXPAND_CE_BUFFER_SIZE 512 /* synwee :TODO revert back 64*/
 226
 227
 228 /* Unsafe UChar hash table table size.                                           */
 229 /*  size is 32 bytes for 1 bit for each latin 1 char + some power of two for     */
 230 /*  hashing the rest of the chars.   Size in bytes                               */
 231 #define UCOL_UNSAFECP_TABLE_SIZE 1056
 232                                     /* mask value down to "some power of two"-1  */
 233                                     /*  number of bits, not num of bytes.        */
 234 #define UCOL_UNSAFECP_TABLE_MASK 0x1fff
 235
 236
 237 /* flags bits for collIterate.flags       */
 238 /*                                        */
 239 /*  NORM - set for incremental normalize of source string */
 240 #define UCOL_ITER_NORM  1
 241
 242 #define UCOL_ITER_HASLEN 2
 243
 244                               /* UCOL_ITER_INNORMBUF - set if the "pos" is in          */
 245                               /*               the writable side buffer, handling      */
 246                               /*               incrementally normalized characters.    */
 247 #define UCOL_ITER_INNORMBUF 4
 248
 249                               /* UCOL_ITER_ALLOCATED - set if this iterator has        */
 250                               /*    malloced storage to expand a buffer.               */
 251 #define UCOL_ITER_ALLOCATED 8
 252                               /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana  */
 253 #define UCOL_HIRAGANA_Q     16
 254                               /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */
 255                               /* otherwise set to false                                  */
 256 #define UCOL_WAS_HIRAGANA   32
 257                               /* UCOL_USE_ITERATOR - set this if collIterate uses a */
 258                               /* character iterator instead of simply accessing string */
 259                               /* by index */
 260 #define UCOL_USE_ITERATOR   64
 261
 262 #define NFC_ZERO_CC_BLOCK_LIMIT_  0x300
 263
 264 struct collIterate {
 265   UChar *string; /* Original string */
 266   /* UChar *start;  Pointer to the start of the source string. Either points to string
 267                     or to writableBuffer */
 268   UChar *endp;   /* string end ptr.  Is undefined for null terminated strings */
 269   UChar *pos; /* This is position in the string.  Can be to original or writable buf */
 270
 271   uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */
 272   uint32_t *CEpos; /* This is the position to which we have stored processed CEs */
 273   UChar *writableBuffer;
 274   uint32_t writableBufSize;
 275   UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
 276   const UCollator *coll;
 277   uint8_t   flags;
 278   uint8_t   origFlags;
 279   uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */
 280   UChar stackWritableBuffer[UCOL_WRITABLE_BUFFER_SIZE]; /* A writable buffer. */
 281   UCharIterator *iterator;
 282   /*int32_t iteratorIndex;*/
 283 };
 284
 285 /*
 286 struct used internally in getSpecial*CE.
 287 data similar to collIterate.
 288 */
 289 struct collIterateState {
 290     UChar    *pos; /* This is position in the string.  Can be to original or writable buf */
 291     UChar    *fcdPosition; /* Position in the original string to continue FCD check from. */
 292     UChar    *bufferaddress; /* address of the normalization buffer */
 293     uint32_t  buffersize;
 294     uint8_t   flags;
 295     uint8_t   origFlags;
 296     uint32_t   iteratorIndex;
 297     int32_t    iteratorMove;
 298 };
 299
 300 U_CAPI void U_EXPORT2
 301 uprv_init_collIterate(const UCollator *collator, const UChar *sourceString, int32_t sourceLen, collIterate *s);
 302
 303
 304 struct UCollationElements
 305 {
 306   /**
 307   * Struct wrapper for source data
 308   */
 309         collIterate        iteratordata_;
 310   /**
 311   * Indicates if this data has been reset.
 312   */
 313         UBool              reset_;
 314   /**
 315   * Indicates if the data should be deleted.
 316   */
 317         UBool              isWritable;
 318 };
 319
 320
 321 #define UCOL_LEVELTERMINATOR 1
 322
 323 /* mask off anything but primary order */
 324 #define UCOL_PRIMARYORDERMASK 0xffff0000
 325 /* mask off anything but secondary order */
 326 #define UCOL_SECONDARYORDERMASK 0x0000ff00
 327 /* mask off anything but tertiary order */
 328 #define UCOL_TERTIARYORDERMASK 0x000000ff
 329 /* primary order shift */
 330 #define UCOL_PRIMARYORDERSHIFT 16
 331 /* secondary order shift */
 332 #define UCOL_SECONDARYORDERSHIFT 8
 333
 334 #define UCOL_BYTE_SIZE_MASK 0xFF
 335
 336 #define UCOL_CASE_BYTE_START 0x80
 337 #define UCOL_CASE_SHIFT_START 7
 338
 339 #define UCOL_IGNORABLE 0
 340
 341 /* get weights from a CE */
 342 #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT)
 343 #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT)
 344 #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK)
 345
 346 /**
 347  * Determine if a character is a Thai vowel (which sorts after
 348  * its base consonant).
 349  */
 350 #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \
 351                                  (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0)))
 352
 353 /**
 354  * Determine if a character is a Thai base consonant
 355  */
 356 #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01)
 357
 358 #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \
 359                          (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \
 360                          (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8)))
 361
 362
 363
 364 #if 0
 365 /* initializes collIterate structure */
 366 /* made as macro to speed up things */
 367 #define init_collIterate(collator, sourceString, sourceLen, s) { \
 368     (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \
 369     (s)->endp  = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \
 370     (s)->CEpos = (s)->toReturn = (s)->CEs; \
 371     (s)->isThai = TRUE; \
 372     (s)->writableBuffer = (s)->stackWritableBuffer; \
 373     (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \
 374     (s)->coll = (collator); \
 375     (s)->fcdPosition = 0;   \
 376     (s)->flags = 0; \
 377     if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \
 378 }
 379 #endif
 380
 381
 382
 383 /*
 384 * Macro to get the maximum size of an expansion ending with the argument ce.
 385 * Used in the Boyer Moore algorithm.
 386 * Note for tailoring, the UCA maxexpansion table has been merged.
 387 * Hence we only have to search the tailored collator only.
 388 * @param coll const UCollator pointer
 389 * @param order last collation element of the expansion sequence
 390 * @param result size of the longest expansion with argument collation element
 391 *        as the last element
 392 */
 393 #define UCOL_GETMAXEXPANSION(coll, order, result) {                          \
 394   const uint32_t *start;                                                     \
 395   const uint32_t *limit;                                                     \
 396   const uint32_t *mid;                                                       \
 397   start = (coll)->endExpansionCE;                                            \
 398   limit = (coll)->lastEndExpansionCE;                                        \
 399   while (start < limit - 1) {                                                \
 400     mid = start + ((limit - start) >> 1);                                    \
 401     if ((order) <= *mid) {                                                   \
 402       limit = mid;                                                           \
 403     }                                                                        \
 404     else {                                                                   \
 405       start = mid;                                                           \
 406     }                                                                        \
 407   }                                                                          \
 408   if (*start == order) {                                                     \
 409     result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE));  \
 410   }                                                                          \
 411   else if (*limit == order) {                                                \
 412          result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \
 413        }                                                                     \
 414        else if ((order & 0xFFFF) == 0x00C0) {                                \
 415               result = 2;                                                    \
 416             }                                                                \
 417             else {                                                           \
 418               result = 1;                                                    \
 419             }                                                                \
 420 }
 421
 422 U_CFUNC
 423 uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, collIterate *source, UErrorCode *status);
 424
 425 U_CFUNC
 426 uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
 427                           collIterate *source, UErrorCode *status);
 428 U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, collIterate *collationSource, UErrorCode *status);
 429 U_CAPI uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll,
 430                                          collIterate *collationSource,
 431                                          UErrorCode *status);
 432 /* function used by C++ getCollationKey to prevent restarting the calculation */
 433 U_CFUNC int32_t
 434 ucol_getSortKeyWithAllocation(const UCollator *coll,
 435                               const UChar *source, int32_t sourceLength,
 436                               uint8_t **pResult,
 437                               UErrorCode *pErrorCode);
 438
 439 /* get some memory */
 440 void *ucol_getABuffer(const UCollator *coll, uint32_t size);
 441
 442 /* worker function for generating sortkeys */
 443 U_CFUNC
 444 int32_t U_CALLCONV
 445 ucol_calcSortKey(const    UCollator    *coll,
 446         const    UChar        *source,
 447         int32_t        sourceLength,
 448         uint8_t        **result,
 449         uint32_t        resultLength,
 450         UBool allocatePrimary,
 451         UErrorCode *status);
 452
 453 U_CFUNC
 454 int32_t U_CALLCONV
 455 ucol_calcSortKeySimpleTertiary(const    UCollator    *coll,
 456         const    UChar        *source,
 457         int32_t        sourceLength,
 458         uint8_t        **result,
 459         uint32_t        resultLength,
 460         UBool allocatePrimary,
 461         UErrorCode *status);
 462
 463 U_CFUNC
 464 int32_t
 465 ucol_getSortKeySize(const UCollator *coll, collIterate *s,
 466                     int32_t currentSize, UColAttributeValue strength,
 467                     int32_t len);
 468 /**
 469  * Makes a copy of the Collator's rule data. The format is
 470  * that of .col files.
 471  *
 472  * @param length returns the length of the data, in bytes.
 473  * @param status the error status
 474  * @return memory, owned by the caller, of size 'length' bytes.
 475  * @internal INTERNAL USE ONLY
 476  */
 477 U_CAPI uint8_t* U_EXPORT2
 478 ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status);
 479
 480 /**
 481  * Used to set requested and valid locales on a collator returned by the collator
 482  * service.
 483  */
 484 U_CAPI void U_EXPORT2
 485 ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt);
 486
 487 #define UCOL_SPECIAL_FLAG 0xF0000000
 488 #define UCOL_TAG_SHIFT 24
 489 #define UCOL_TAG_MASK 0x0F000000
 490 #define INIT_EXP_TABLE_SIZE 1024
 491 #define UCOL_NOT_FOUND 0xF0000000
 492 #define UCOL_EXPANSION 0xF1000000
 493 #define UCOL_CONTRACTION 0xF2000000
 494 #define UCOL_THAI 0xF3000000
 495 #define UCOL_UNMARKED 0x03
 496 #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f
 497
 498 /* Bit mask for primary collation strength. */
 499 #define UCOL_PRIMARYMASK    0xFFFF0000
 500
 501 /* Bit mask for secondary collation strength. */
 502 #define UCOL_SECONDARYMASK  0x0000FF00
 503
 504 /* Bit mask for tertiary collation strength. */
 505 #define UCOL_TERTIARYMASK   0x000000FF
 506
 507 /**
 508  * Internal.
 509  * This indicates the last element in a UCollationElements has been consumed.
 510  * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs.
 511  */
 512 #define UCOL_NO_MORE_CES            0x00010101
 513 #define UCOL_NO_MORE_CES_PRIMARY    0x00010000
 514 #define UCOL_NO_MORE_CES_SECONDARY  0x00000100
 515 #define UCOL_NO_MORE_CES_TERTIARY   0x00000001
 516
 517 #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF)
 518
 519 #define UCOL_UPPER_CASE 0x80
 520 #define UCOL_MIXED_CASE 0x40
 521 #define UCOL_LOWER_CASE 0x00
 522
 523 #define UCOL_CONTINUATION_MARKER 0xC0
 524 #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F
 525
 526 #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER)
 527 #define isFlagged(CE) (((CE) & 0x80) == 0x80)
 528 #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0)
 529
 530 #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT)
 531 #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG))
 532 #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG))
 533 #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF)
 534 #define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF)
 535 #define getContractOffset(CE) ((CE)&0xFFFFFF)
 536 #define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4)
 537 #define getExpansionCount(CE) ((CE)&0xF)
 538 #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0)
 539
 540 /* StringSearch internal use */
 541 #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF)
 542 #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL)
 543 #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs)
 544 #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs)
 545 #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset)
 546 #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn)
 547 #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces)
 548
 549 /* This is an enum that lists magic special byte values from the fractional UCA */
 550 /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */
 551
 552 enum {
 553     UCOL_BYTE_ZERO = 0x00,
 554     UCOL_BYTE_LEVEL_SEPARATOR = 0x01,
 555     UCOL_BYTE_SORTKEY_GLUE = 0x02,
 556     UCOL_BYTE_SHIFT_PREFIX = 0x03,
 557     UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX,
 558     UCOL_BYTE_FIRST_TAILORED = 0x04,
 559     UCOL_BYTE_COMMON = 0x05,
 560     UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON,
 561     UCOL_CODAN_PLACEHOLDER = 0x24,
 562     UCOL_BYTE_LAST_LATIN_PRIMARY = 0x4C,
 563     UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x4D,
 564     UCOL_BYTE_UNSHIFTED_MAX = 0xFF
 565 };
 566
 567 #if 0
 568 #define UCOL_RESET_TOP_VALUE                0x9F000303
 569 #define UCOL_FIRST_PRIMARY_IGNORABLE        0x00008705
 570 #define UCOL_LAST_PRIMARY_IGNORABLE         0x0000DD05
 571 #define UCOL_LAST_PRIMARY_IGNORABLE_CONT    0x000051C0
 572 #define UCOL_FIRST_SECONDARY_IGNORABLE      0x00000000
 573 #define UCOL_LAST_SECONDARY_IGNORABLE       0x00000500
 574 #define UCOL_FIRST_TERTIARY_IGNORABLE       0x00000000
 575 #define UCOL_LAST_TERTIARY_IGNORABLE        0x00000000
 576 #define UCOL_FIRST_VARIABLE                 0x05070505
 577 #define UCOL_LAST_VARIABLE                  0x179B0505
 578 #define UCOL_FIRST_NON_VARIABLE             0x1A200505
 579 #define UCOL_LAST_NON_VARIABLE              0x7B41058F
 580
 581 #define UCOL_NEXT_TOP_VALUE                 0xE8960303
 582 #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE   0x00008905
 583 #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE    0x03000303
 584 #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705
 585 #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE  0x00000500
 586 #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE  0x00000000
 587 #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE   0x00000000
 588 #define UCOL_NEXT_FIRST_VARIABLE            0x05090505
 589 #define UCOL_NEXT_LAST_VARIABLE             0x1A200505
 590
 591 #define PRIMARY_IMPLICIT_MIN 0xE8000000
 592 #define PRIMARY_IMPLICIT_MAX 0xF0000000
 593 #endif
 594
 595 /* These constants can be changed - sortkey size is affected by them */
 596 #define UCOL_PROPORTION2 0.5
 597 #define UCOL_PROPORTION3 0.667
 598
 599 /* These values come from the UCA */
 600 #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON
 601 #define UCOL_COMMON_TOP2 0x86u
 602 #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1)
 603
 604 #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80
 605 #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40
 606 #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85
 607 #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45
 608 #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5
 609
 610 /* These values come from the UCA */
 611 #define UCOL_COMMON_BOT3 0x05
 612
 613 #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86;
 614 #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3;
 615
 616 #define UCOL_TOP_COUNT2  (UCOL_PROPORTION2*UCOL_TOTAL2)
 617 #define UCOL_BOT_COUNT2  (UCOL_TOTAL2-UCOL_TOP_COUNT2)
 618
 619
 620 #define UCOL_COMMON2 UCOL_COMMON_BOT2
 621 #define UCOL_COMMON3_UPPERFIRST 0xC5
 622 #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3
 623
 624 #define UCOL_COMMON4 0xFF
 625
 626 /* constants for case level/case first handling */
 627 /* used to instantiate UCollators fields in ucol_updateInternalState */
 628 #define UCOL_CASE_SWITCH      0xC0
 629 #define UCOL_NO_CASE_SWITCH   0x00
 630
 631 #define UCOL_REMOVE_CASE      0x3F
 632 #define UCOL_KEEP_CASE        0xFF
 633
 634 #define UCOL_CASE_BIT_MASK    0xC0
 635
 636 #define UCOL_TERT_CASE_MASK   0xFF
 637
 638 #define UCOL_ENDOFLATINONERANGE 0xFF
 639 #define UCOL_LATINONETABLELEN   (UCOL_ENDOFLATINONERANGE+50)
 640 #define UCOL_BAIL_OUT_CE      0xFF000000
 641
 642
 643 typedef enum {
 644     NOT_FOUND_TAG = 0,
 645     EXPANSION_TAG = 1,       /* This code point results in an expansion */
 646     CONTRACTION_TAG = 2,     /* Start of a contraction */
 647     THAI_TAG = 3,            /* Thai character - do the reordering */
 648     CHARSET_TAG = 4,         /* Charset processing, not yet implemented */
 649     SURROGATE_TAG = 5,       /* Lead surrogate that is tailored and doesn't start a contraction */
 650     HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/
 651     LEAD_SURROGATE_TAG = 7,  /* D800-DBFF*/
 652     TRAIL_SURROGATE_TAG = 8,     /* DC00-DFFF*/
 653     CJK_IMPLICIT_TAG = 9,    /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
 654     IMPLICIT_TAG = 10,
 655     SPEC_PROC_TAG = 11,
 656     /* ICU 2.1 */
 657     LONG_PRIMARY_TAG = 12,   /* This is a three byte primary with starting secondaries and tertiaries */
 658                              /* It fits in a single 32 bit CE and is used instead of expansion to save */
 659                              /* space without affecting the performance (hopefully) */
 660
 661     DIGIT_TAG = 13,          /* COllate Digits As Numbers (CODAN) implementation */
 662
 663     CE_TAGS_COUNT
 664 } UColCETags;
 665
 666 /*
 667  *****************************************************************************************
 668  * set to zero
 669  * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF
 670  ******************************************************************************************
 671  */
 672
 673 typedef struct {
 674       uint32_t variableTopValue;
 675       /*UColAttributeValue*/ int32_t frenchCollation;
 676       /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/
 677       /*UColAttributeValue*/ int32_t caseFirst;         /* who goes first, lower case or uppercase */
 678       /*UColAttributeValue*/ int32_t caseLevel;         /* do we have an extra case level */
 679       /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */
 680       /*UColAttributeValue*/ int32_t strength;          /* attribute for strength */
 681       /*UColAttributeValue*/ int32_t hiraganaQ;         /* attribute for special Hiragana */
 682       /*UColAttributeValue*/ int32_t numericCollation;  /* attribute for numeric collation */
 683       uint32_t reserved[15];                 /* for future use */
 684 } UColOptionSet;
 685
 686 typedef struct {
 687   uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2];       /*0x00000000*/
 688   uint32_t UCA_LAST_TERTIARY_IGNORABLE[2];        /*0x00000000*/
 689   uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2];        /*0x00008705*/
 690   uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2];      /*0x00000000*/
 691   uint32_t UCA_LAST_SECONDARY_IGNORABLE[2];       /*0x00000500*/
 692   uint32_t UCA_LAST_PRIMARY_IGNORABLE[2];         /*0x0000DD05*/
 693   uint32_t UCA_FIRST_VARIABLE[2];                 /*0x05070505*/
 694   uint32_t UCA_LAST_VARIABLE[2];                  /*0x13CF0505*/
 695   uint32_t UCA_FIRST_NON_VARIABLE[2];             /*0x16200505*/
 696   uint32_t UCA_LAST_NON_VARIABLE[2];              /*0x767C0505*/
 697   uint32_t UCA_RESET_TOP_VALUE[2];                /*0x9F000303*/
 698   uint32_t UCA_FIRST_IMPLICIT[2];
 699   uint32_t UCA_LAST_IMPLICIT[2];
 700   uint32_t UCA_FIRST_TRAILING[2];
 701   uint32_t UCA_LAST_TRAILING[2];
 702
 703 #if 0
 704   uint32_t UCA_NEXT_TOP_VALUE[2];                 /*0xE8960303*/
 705   uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE;   /*0x00008905*/
 706   uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE;    /*0x03000303*/
 707   uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/
 708   uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE;  /*0x00000500*/
 709   uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE;  /*0x00000000*/
 710   uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE;   /*0x00000000*/
 711   uint32_t UCA_NEXT_FIRST_VARIABLE;            /*0x05090505*/
 712   uint32_t UCA_NEXT_LAST_VARIABLE;             /*0x16200505*/
 713 #endif
 714
 715   uint32_t UCA_PRIMARY_TOP_MIN;
 716   uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/
 717   uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/
 718   uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/
 719   uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/
 720   uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/
 721   uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/
 722 } UCAConstants;
 723
 724 typedef struct {
 725       int32_t size;
 726       /* all the offsets are in bytes */
 727       /* to get the address add to the header address and cast properly */
 728       uint32_t options; /* these are the default options for the collator */
 729       uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */
 730       uint32_t contractionUCACombos;        /* this one is needed only for UCA, to copy the appropriate contractions */
 731       uint32_t magic;            /* magic number - lets us know whether reserved data is reset or junked */
 732       uint32_t mappingPosition;  /* const uint8_t *mappingPosition; */
 733       uint32_t expansion;        /* uint32_t *expansion;            */
 734       uint32_t contractionIndex; /* UChar *contractionIndex;        */
 735       uint32_t contractionCEs;   /* uint32_t *contractionCEs;       */
 736       uint32_t contractionSize;  /* needed for various closures */
 737       /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars      */
 738
 739       uint32_t endExpansionCE;      /* array of last collation element in
 740                                        expansion */
 741       uint32_t expansionCESize;     /* array of maximum expansion size
 742                                        corresponding to the expansion
 743                                        collation elements with last element
 744                                        in endExpansionCE*/
 745       int32_t  endExpansionCECount; /* size of endExpansionCE */
 746       uint32_t unsafeCP;            /* hash table of unsafe code points */
 747       uint32_t contrEndCP;          /* hash table of final code points  */
 748                                     /*   in contractions.               */
 749
 750       int32_t contractionUCACombosSize;     /* number of UCA contraction items. */
 751                                             /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */
 752       UBool jamoSpecial;                    /* is jamoSpecial */
 753       UBool isBigEndian;                    /* is this data big endian? from the UDataInfo header*/
 754       uint8_t charSetFamily;                /* what is the charset family of this data from the UDataInfo header*/
 755       uint8_t contractionUCACombosWidth;    /* width of UCA combos field */
 756       UVersionInfo version;
 757       UVersionInfo UCAVersion;              /* version of the UCA, read from file */
 758       UVersionInfo UCDVersion;              /* UCD version, obtained by u_getUnicodeVersion */
 759       UVersionInfo formatVersion;           /* format version from the UDataInfo header */
 760       uint8_t reserved[84];                 /* for future use */
 761 } UCATableHeader;
 762
 763 #define U_UNKNOWN_STATE 0
 764 #define U_COLLATOR_STATE 0x01
 765 #define U_STATE_LIMIT 0x02
 766
 767 /* This is the first structure in a state */
 768 /* it should be machine independent */
 769 typedef struct {
 770   /* this structure is supposed to be readable on all the platforms.*/
 771   /* first 2 fields hold the size of the structure in a platform independent way */
 772   uint8_t sizeLo;
 773   uint8_t sizeHi;
 774   /* identifying the writing platform */
 775   uint8_t isBigEndian;
 776   /* see U_CHARSET_FAMILY values in utypes.h */
 777   uint8_t charsetFamily;
 778   /* version of ICU this state structure comes from */
 779   uint8_t icuVersion[4];
 780   /* What is the data following this state */
 781   uint8_t type;
 782   /* more stuff to come, keep it on 16 byte boundary */
 783   uint8_t reserved[7];
 784 } UStateStruct;
 785
 786 /* This structure follows UStatusStruct */
 787 /* and contains data specific for the collators */
 788 /* Endianess needs to be decided before accessing this structure */
 789 /* However, it's size IS endianess independent */
 790 typedef struct {
 791   /* size of this structure */
 792   uint8_t sizeLo;
 793   uint8_t sizeHi;
 794   /* This state is followed by the frozen tailoring */
 795   uint8_t containsTailoring;
 796   /* This state is followed by the frozen UCA */
 797   uint8_t containsUCA;
 798   /* Version info - the same one */
 799   uint8_t versionInfo[4];
 800
 801   /* for charset CEs */
 802   uint8_t charsetName[32];
 803   /* this is the resolved locale name*/
 804   uint8_t locale[32];
 805
 806   /* Attributes. Open ended */
 807   /* all the following will be moved to uint32_t because of portability */
 808   /* variable top value */
 809   uint32_t variableTopValue;
 810   /* attribute for handling variable elements*/
 811   uint32_t /*UColAttributeValue*/ alternateHandling;
 812   /* how to handle secondary weights */
 813   uint32_t /*UColAttributeValue*/ frenchCollation;
 814   /* who goes first, lower case or uppercase */
 815   uint32_t /*UColAttributeValue*/ caseFirst;
 816   /* do we have an extra case level */
 817   uint32_t /*UColAttributeValue*/ caseLevel;
 818   /* attribute for normalization */
 819   uint32_t /*UColAttributeValue*/ normalizationMode;
 820   /* attribute for strength */
 821   uint32_t /*UColAttributeValue*/ strength;
 822   /* to be immediately 16 byte aligned */
 823   uint8_t reserved[12];
 824 } UColStateStruct;
 825
 826 #define UCOL_INV_SIZEMASK 0xFFF00000
 827 #define UCOL_INV_OFFSETMASK 0x000FFFFF
 828 #define UCOL_INV_SHIFTVALUE 20
 829
 830 U_CDECL_BEGIN
 831
 832 typedef struct {
 833   uint32_t byteSize;
 834   uint32_t tableSize;
 835   uint32_t contsSize;
 836   uint32_t table;
 837   uint32_t conts;
 838   UVersionInfo UCAVersion;              /* version of the UCA, read from file */
 839   uint8_t padding[8];
 840 } InverseUCATableHeader;
 841
 842 typedef int32_t U_CALLCONV
 843 SortKeyGenerator(const    UCollator    *coll,
 844         const    UChar        *source,
 845         int32_t        sourceLength,
 846         uint8_t        **result,
 847         uint32_t        resultLength,
 848         UBool allocatePrimary,
 849         UErrorCode *status);
 850
 851 struct UCollator {
 852     UColOptionSet  *options;
 853     SortKeyGenerator *sortKeyGen;
 854     uint32_t *latinOneCEs;
 855     char* validLocale;
 856     char* requestedLocale;
 857     UResourceBundle *rb;
 858     UResourceBundle *elements;
 859     const UCATableHeader *image;
 860     /*CompactEIntArray *mapping;*/
 861     UTrie *mapping;
 862     const uint32_t *latinOneMapping;
 863     const uint32_t *expansion;
 864     const UChar    *contractionIndex;
 865     const uint32_t *contractionCEs;
 866     const uint8_t  *scriptOrder;
 867
 868     const uint32_t *endExpansionCE;    /* array of last ces in an expansion ce.
 869                                           corresponds to expansionCESize */
 870     const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */
 871     const uint8_t  *expansionCESize;   /* array of the maximum size of a
 872                                          expansion ce with the last ce
 873                                          corresponding to endExpansionCE,
 874                                          terminated with a null */
 875     const uint8_t *unsafeCP;           /* unsafe code points hashtable */
 876     const uint8_t *contrEndCP;         /* Contraction ending chars hash table */
 877     UChar          minUnsafeCP;        /* Smallest unsafe Code Point. */
 878     UChar          minContrEndCP;      /* Smallest code point at end of a contraction */
 879
 880     const UChar *rules;
 881     int32_t rulesLength;
 882     int32_t latinOneTableLen;
 883
 884     /*UErrorCode errorCode;*/             /* internal error code */
 885
 886     uint32_t variableTopValue;
 887     UColAttributeValue frenchCollation;
 888     UColAttributeValue alternateHandling; /* attribute for handling variable elements*/
 889     UColAttributeValue caseFirst;         /* who goes first, lower case or uppercase */
 890     UColAttributeValue caseLevel;         /* do we have an extra case level */
 891     UColAttributeValue normalizationMode; /* attribute for normalization */
 892     UColAttributeValue strength;          /* attribute for strength */
 893     UColAttributeValue hiraganaQ;         /* attribute for Hiragana */
 894     UColAttributeValue numericCollation;
 895     UBool variableTopValueisDefault;
 896     UBool frenchCollationisDefault;
 897     UBool alternateHandlingisDefault; /* attribute for handling variable elements*/
 898     UBool caseFirstisDefault;         /* who goes first, lower case or uppercase */
 899     UBool caseLevelisDefault;         /* do we have an extra case level */
 900     UBool normalizationModeisDefault; /* attribute for normalization */
 901     UBool strengthisDefault;          /* attribute for strength */
 902     UBool hiraganaQisDefault;         /* attribute for Hiragana */
 903     UBool numericCollationisDefault;
 904     UBool hasRealData;                /* some collators have only options, like French, no rules */
 905                                       /* to speed up things, we use the UCA image, but we don't want it */
 906                                       /* to run around */
 907
 908     UBool freeOnClose;
 909     UBool freeOptionsOnClose;
 910     UBool freeRulesOnClose;
 911     UBool freeImageOnClose;
 912
 913     UBool latinOneUse;
 914     UBool latinOneRegenTable;
 915     UBool latinOneFailed;
 916
 917     int32_t tertiaryAddition; /* when switching case, we need to add or subtract different values */
 918     uint8_t caseSwitch;
 919     uint8_t tertiaryCommon;
 920     uint8_t tertiaryMask;
 921     uint8_t tertiaryTop; /* Upper range when compressing */
 922     uint8_t tertiaryBottom; /* Upper range when compressing */
 923     uint8_t tertiaryTopCount;
 924     uint8_t tertiaryBottomCount;
 925
 926     UDataInfo dataInfo;               /* Data info of UCA table */
 927     const UCollator *UCA;
 928
 929 };
 930
 931 U_CDECL_END
 932
 933 /* various internal functions */
 934
 935 /* do not close UCA returned by ucol_initUCA! */
 936 U_CFUNC
 937 UCollator* ucol_initUCA(UErrorCode *status);
 938
 939 U_CFUNC
 940 UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status);
 941
 942 U_CFUNC
 943 void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
 944
 945 U_CFUNC
 946 UCollator* ucol_open_internal(const char* loc, UErrorCode* status);
 947
 948 #if 0
 949 U_CFUNC
 950 void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
 951 #endif
 952
 953 U_CFUNC
 954 void ucol_updateInternalState(UCollator *coll, UErrorCode *status);
 955
 956 U_CAPI uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status);
 957 U_CAPI char* U_EXPORT2 ucol_sortKeyToString(const UCollator *coll, const uint8_t *sortkey, char *buffer, uint32_t *len);
 958 U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status);
 959
 960 U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status);
 961
 962 U_CAPI void U_EXPORT2
 963 uprv_uca_initImplicitConstants(int32_t minPrimary, int32_t maxPrimary, UErrorCode *status);
 964
 965 U_CAPI uint32_t U_EXPORT2
 966 uprv_uca_getImplicitFromRaw(UChar32 cp);
 967
 968 U_CAPI uint32_t U_EXPORT2
 969 uprv_uca_getImplicitPrimary(UChar32 cp);
 970
 971 U_CAPI UChar32 U_EXPORT2
 972 uprv_uca_getRawFromImplicit(uint32_t implicit);
 973
 974 U_CAPI UChar32 U_EXPORT2
 975 uprv_uca_getRawFromCodePoint(UChar32 i);
 976
 977 U_CAPI UChar32 U_EXPORT2
 978 uprv_uca_getCodePointFromRaw(UChar32 i);
 979
 980
 981
 982 #ifdef XP_CPLUSPLUS
 983 /*
 984  *  Test whether a character is potentially "unsafe" for use as a collation
 985  *  starting point.  Unsafe chars are those with combining class != 0 plus
 986  *  those that are the 2nd thru nth character in a contraction sequence.
 987  *
 988  *  Function is in header file because it's used in both collation and string search,
 989  *  and needs to be inline for performance.
 990  */
 991 static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) {
 992     int32_t  hash;
 993     uint8_t  htbyte;
 994
 995     if (c < coll->minUnsafeCP) {
 996         return FALSE;
 997     }
 998
 999     hash = c;
1000     if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
1001       if(UTF_IS_LEAD(c) || UTF_IS_TRAIL(c)) {
1002             /*  Trail surrogate                     */
1003             /*  These are always considered unsafe. */
1004             return TRUE;
1005         }
1006         hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
1007     }
1008     htbyte = coll->unsafeCP[hash>>3];
1009     return (((htbyte >> (hash & 7)) & 1) == 1);
1010 }
1011 #endif /* XP_CPLUSPLUS */
1012
1013
1014 #endif /* #if !UCONFIG_NO_COLLATION */
1015
1016 #endif