icuSources/common/unormimp.h

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2001-2007, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  unormimp.h
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2001may25
  14 *   created by: Markus W. Scherer
  15 */
  16
  17 #ifndef __UNORMIMP_H__
  18 #define __UNORMIMP_H__
  19
  20 #include "unicode/utypes.h"
  21
  22 #if !UCONFIG_NO_NORMALIZATION
  23
  24 #ifdef XP_CPLUSPLUS
  25 #include "unicode/uniset.h"
  26 #endif
  27
  28 #include "unicode/uiter.h"
  29 #include "unicode/unorm.h"
  30 #include "unicode/uset.h"
  31 #include "utrie.h"
  32 #include "ustr_imp.h"
  33 #include "udataswp.h"
  34
  35 /*
  36  * This new implementation of the normalization code loads its data from
  37  * unorm.icu, which is generated with the gennorm tool.
  38  * The format of that file is described at the end of this file.
  39  */
  40
  41 /* norm32 value constants */
  42 enum {
  43     /* quick check flags 0..3 set mean "no" for their forms */
  44     _NORM_QC_NFC=0x11,          /* no|maybe */
  45     _NORM_QC_NFKC=0x22,         /* no|maybe */
  46     _NORM_QC_NFD=4,             /* no */
  47     _NORM_QC_NFKD=8,            /* no */
  48
  49     _NORM_QC_ANY_NO=0xf,
  50
  51     /* quick check flags 4..5 mean "maybe" for their forms; test flags>=_NORM_QC_MAYBE */
  52     _NORM_QC_MAYBE=0x10,
  53     _NORM_QC_ANY_MAYBE=0x30,
  54
  55     _NORM_QC_MASK=0x3f,
  56
  57     _NORM_COMBINES_FWD=0x40,
  58     _NORM_COMBINES_BACK=0x80,
  59     _NORM_COMBINES_ANY=0xc0,
  60
  61     _NORM_CC_SHIFT=8,           /* UnicodeData.txt combining class in bits 15..8 */
  62     _NORM_CC_MASK=0xff00,
  63
  64     _NORM_EXTRA_SHIFT=16,               /* 16 bits for the index to UChars and other extra data */
  65     _NORM_EXTRA_INDEX_TOP=0xfc00,       /* start of surrogate specials after shift */
  66
  67     _NORM_EXTRA_SURROGATE_MASK=0x3ff,
  68     _NORM_EXTRA_SURROGATE_TOP=0x3f0,    /* hangul etc. */
  69
  70     _NORM_EXTRA_HANGUL=_NORM_EXTRA_SURROGATE_TOP,
  71     _NORM_EXTRA_JAMO_L,
  72     _NORM_EXTRA_JAMO_V,
  73     _NORM_EXTRA_JAMO_T
  74 };
  75
  76 /* norm32 value constants using >16 bits */
  77 #define _NORM_MIN_SPECIAL       0xfc000000
  78 #define _NORM_SURROGATES_TOP    0xfff00000
  79 #define _NORM_MIN_HANGUL        0xfff00000
  80 #define _NORM_MIN_JAMO_V        0xfff20000
  81 #define _NORM_JAMO_V_TOP        0xfff30000
  82
  83 /* value constants for auxTrie */
  84 enum {
  85     _NORM_AUX_COMP_EX_SHIFT=10,
  86     _NORM_AUX_UNSAFE_SHIFT=11,
  87     _NORM_AUX_NFC_SKIPPABLE_F_SHIFT=12
  88 };
  89
  90 #define _NORM_AUX_MAX_FNC           ((int32_t)1<<_NORM_AUX_COMP_EX_SHIFT)
  91
  92 #define _NORM_AUX_FNC_MASK          (uint32_t)(_NORM_AUX_MAX_FNC-1)
  93 #define _NORM_AUX_COMP_EX_MASK      ((uint32_t)1<<_NORM_AUX_COMP_EX_SHIFT)
  94 #define _NORM_AUX_UNSAFE_MASK       ((uint32_t)1<<_NORM_AUX_UNSAFE_SHIFT)
  95 #define _NORM_AUX_NFC_SKIP_F_MASK   ((uint32_t)1<<_NORM_AUX_NFC_SKIPPABLE_F_SHIFT)
  96
  97 /* canonStartSets[0..31] contains indexes for what is in the array */
  98 enum {
  99     _NORM_SET_INDEX_CANON_SETS_LENGTH,      /* number of uint16_t in canonical starter sets */
 100     _NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH, /* number of uint16_t in the BMP search table (contains pairs) */
 101     _NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH,/* number of uint16_t in the supplementary search table (contains triplets) */
 102
 103     /* from formatVersion 2.3: */
 104     _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET,   /* uint16_t offset from canonStartSets[0] to the
 105                                                exclusion set for CJK compatibility characters */
 106     _NORM_SET_INDEX_NX_UNICODE32_OFFSET,    /* uint16_t offset from canonStartSets[0] to the
 107                                                exclusion set for Unicode 3.2 characters */
 108     _NORM_SET_INDEX_NX_RESERVED_OFFSET,     /* uint16_t offset from canonStartSets[0] to the
 109                                                end of the previous exclusion set */
 110
 111     _NORM_SET_INDEX_TOP=32                  /* changing this requires a new formatVersion */
 112 };
 113
 114 /* more constants for canonical starter sets */
 115
 116 /* 14 bit indexes to canonical USerializedSets */
 117 #define _NORM_MAX_CANON_SETS            0x4000
 118
 119 /* single-code point BMP sets are encoded directly in the search table except if result=0x4000..0x7fff */
 120 #define _NORM_CANON_SET_BMP_MASK        0xc000
 121 #define _NORM_CANON_SET_BMP_IS_INDEX    0x4000
 122
 123 /* indexes[] value names */
 124 enum {
 125     _NORM_INDEX_TRIE_SIZE,              /* number of bytes in normalization trie */
 126     _NORM_INDEX_UCHAR_COUNT,            /* number of UChars in extra data */
 127
 128     _NORM_INDEX_COMBINE_DATA_COUNT,     /* number of uint16_t words for combining data */
 129     _NORM_INDEX_COMBINE_FWD_COUNT,      /* number of code points that combine forward */
 130     _NORM_INDEX_COMBINE_BOTH_COUNT,     /* number of code points that combine forward and backward */
 131     _NORM_INDEX_COMBINE_BACK_COUNT,     /* number of code points that combine backward */
 132
 133     _NORM_INDEX_MIN_NFC_NO_MAYBE,       /* first code point with quick check NFC NO/MAYBE */
 134     _NORM_INDEX_MIN_NFKC_NO_MAYBE,      /* first code point with quick check NFKC NO/MAYBE */
 135     _NORM_INDEX_MIN_NFD_NO_MAYBE,       /* first code point with quick check NFD NO/MAYBE */
 136     _NORM_INDEX_MIN_NFKD_NO_MAYBE,      /* first code point with quick check NFKD NO/MAYBE */
 137
 138     _NORM_INDEX_FCD_TRIE_SIZE,          /* number of bytes in FCD trie */
 139
 140     _NORM_INDEX_AUX_TRIE_SIZE,          /* number of bytes in the auxiliary trie */
 141     _NORM_INDEX_CANON_SET_COUNT,        /* number of uint16_t in the array of serialized USet */
 142
 143     _NORM_INDEX_TOP=32                  /* changing this requires a new formatVersion */
 144 };
 145
 146 enum {
 147     /* FCD check: everything below this code point is known to have a 0 lead combining class */
 148     _NORM_MIN_WITH_LEAD_CC=0x300
 149 };
 150
 151 enum {
 152     /**
 153      * Bit 7 of the length byte for a decomposition string in extra data is
 154      * a flag indicating whether the decomposition string is
 155      * preceded by a 16-bit word with the leading and trailing cc
 156      * of the decomposition (like for A-umlaut);
 157      * if not, then both cc's are zero (like for compatibility ideographs).
 158      */
 159     _NORM_DECOMP_FLAG_LENGTH_HAS_CC=0x80,
 160     /**
 161      * Bits 6..0 of the length byte contain the actual length.
 162      */
 163     _NORM_DECOMP_LENGTH_MASK=0x7f
 164 };
 165
 166 #endif /* #if !UCONFIG_NO_NORMALIZATION */
 167
 168 /* Korean Hangul and Jamo constants */
 169 enum {
 170     JAMO_L_BASE=0x1100,     /* "lead" jamo */
 171     JAMO_V_BASE=0x1161,     /* "vowel" jamo */
 172     JAMO_T_BASE=0x11a7,     /* "trail" jamo */
 173
 174     HANGUL_BASE=0xac00,
 175
 176     JAMO_L_COUNT=19,
 177     JAMO_V_COUNT=21,
 178     JAMO_T_COUNT=28,
 179
 180     HANGUL_COUNT=JAMO_L_COUNT*JAMO_V_COUNT*JAMO_T_COUNT
 181 };
 182
 183 #if !UCONFIG_NO_NORMALIZATION
 184
 185 /* Constants for options flags for normalization. @draft ICU 2.6 */
 186 enum {
 187     /** Options bit 0, do not decompose Hangul syllables. @draft ICU 2.6 */
 188     UNORM_NX_HANGUL=1,
 189     /** Options bit 1, do not decompose CJK compatibility characters. @draft ICU 2.6 */
 190     UNORM_NX_CJK_COMPAT=2,
 191     /**
 192      * Options bit 8, use buggy recomposition described in
 193      * Unicode Public Review Issue #29
 194      * at http://www.unicode.org/review/resolved-pri.html#pri29
 195      *
 196      * Used in IDNA implementation according to strict interpretation
 197      * of IDNA definition based on Unicode 3.2 which predates PRI #29.
 198      */
 199     UNORM_BEFORE_PRI_29=0x100
 200 };
 201
 202 /**
 203  * Is the normalizer data loaded?
 204  * This is used internally before other internal normalizer functions
 205  * are called.
 206  * It saves this check in each of many normalization calls that
 207  * are made for, e.g., collation.
 208  *
 209  * @param pErrorCode as usual
 210  * @return boolean value for whether the normalization data is loaded
 211  *
 212  * @internal
 213  */
 214 U_CAPI UBool U_EXPORT2
 215 unorm_haveData(UErrorCode *pErrorCode);
 216
 217 /**
 218  * Internal API for normalizing.
 219  * Does not check for bad input.
 220  * @internal
 221  */
 222 U_CAPI int32_t U_EXPORT2
 223 unorm_internalNormalize(UChar *dest, int32_t destCapacity,
 224                         const UChar *src, int32_t srcLength,
 225                         UNormalizationMode mode, int32_t options,
 226                         UErrorCode *pErrorCode);
 227
 228 #ifdef XP_CPLUSPLUS
 229
 230 /**
 231  * Internal API for normalizing.
 232  * Does not check for bad input.
 233  * Requires _haveData() to be true.
 234  * @internal
 235  */
 236 U_CFUNC int32_t
 237 unorm_internalNormalizeWithNX(UChar *dest, int32_t destCapacity,
 238                               const UChar *src, int32_t srcLength,
 239                               UNormalizationMode mode, int32_t options, const U_NAMESPACE_QUALIFIER UnicodeSet *nx,
 240                               UErrorCode *pErrorCode);
 241
 242 #endif
 243
 244 /**
 245  * internal API, used by normlzr.cpp
 246  * @internal
 247  */
 248 U_CAPI int32_t U_EXPORT2
 249 unorm_decompose(UChar *dest, int32_t destCapacity,
 250                 const UChar *src, int32_t srcLength,
 251                 UBool compat, int32_t options,
 252                 UErrorCode *pErrorCode);
 253
 254 /**
 255  * internal API, used by normlzr.cpp
 256  * @internal
 257  */
 258 U_CAPI int32_t U_EXPORT2
 259 unorm_compose(UChar *dest, int32_t destCapacity,
 260               const UChar *src, int32_t srcLength,
 261               UBool compat, int32_t options,
 262               UErrorCode *pErrorCode);
 263
 264 #ifdef XP_CPLUSPLUS
 265
 266 /**
 267  * internal API, used by unormcmp.cpp
 268  * @internal
 269  */
 270 U_CFUNC UNormalizationCheckResult
 271 unorm_internalQuickCheck(const UChar *src,
 272                          int32_t srcLength,
 273                          UNormalizationMode mode,
 274                          UBool allowMaybe,
 275                          const U_NAMESPACE_QUALIFIER UnicodeSet *nx,
 276                          UErrorCode *pErrorCode);
 277
 278 #endif
 279
 280 #endif /* #if !UCONFIG_NO_NORMALIZATION */
 281
 282 /**
 283  * Internal option for unorm_cmpEquivFold() for decomposing.
 284  * If not set, just do strcasecmp().
 285  * @internal
 286  */
 287 #define _COMPARE_EQUIV 0x80000
 288
 289 #ifndef U_COMPARE_IGNORE_CASE
 290 /* see also unorm.h */
 291 /**
 292  * Option bit for unorm_compare:
 293  * Perform case-insensitive comparison.
 294  * @draft ICU 2.2
 295  */
 296 #define U_COMPARE_IGNORE_CASE       0x10000
 297 #endif
 298
 299 /**
 300  * Internal option for unorm_cmpEquivFold() for strncmp style.
 301  * If set, checks for both string length and terminating NUL.
 302  * @internal
 303  */
 304 #define _STRNCMP_STYLE 0x1000
 305
 306 #if !UCONFIG_NO_NORMALIZATION
 307
 308 /**
 309  * Internal API to get the 16-bit FCD value (lccc + tccc) for c,
 310  * for u_getIntPropertyValue().
 311  * @internal
 312  */
 313 U_CFUNC uint16_t U_EXPORT2
 314 unorm_getFCD16FromCodePoint(UChar32 c);
 315
 316 /**
 317  * Internal API, used by collation code.
 318  * Get access to the internal FCD trie table to be able to perform
 319  * incremental, per-code unit, FCD checks in collation.
 320  * One pointer is sufficient because the trie index values are offset
 321  * by the index size, so that the same pointer is used to access the trie data.
 322  * @internal
 323  */
 324 U_CAPI const uint16_t * U_EXPORT2
 325 unorm_getFCDTrie(UErrorCode *pErrorCode);
 326
 327 #ifdef XP_CPLUSPLUS
 328
 329 /**
 330  * Internal API, used by collation code.
 331  * Get the FCD value for a code unit, with
 332  * bits 15..8   lead combining class
 333  * bits  7..0   trail combining class
 334  *
 335  * If c is a lead surrogate and the value is not 0,
 336  * then instead of combining classes the value
 337  * is used in unorm_getFCD16FromSurrogatePair() to get the real value
 338  * of the supplementary code point.
 339  *
 340  * @internal
 341  */
 342 static inline uint16_t
 343 unorm_getFCD16(const uint16_t *fcdTrieIndex, UChar c) {
 344     return
 345         fcdTrieIndex[
 346             (fcdTrieIndex[
 347                 c>>UTRIE_SHIFT
 348             ]<<UTRIE_INDEX_SHIFT)+
 349             (c&UTRIE_MASK)
 350         ];
 351 }
 352
 353 /**
 354  * Internal API, used by collation code.
 355  * Get the FCD value for a supplementary code point, with
 356  * bits 15..8   lead combining class
 357  * bits  7..0   trail combining class
 358  *
 359  * @param fcd16  The FCD value for the lead surrogate, not 0.
 360  * @param c2     The trail surrogate code unit.
 361  *
 362  * @internal
 363  */
 364 static inline uint16_t
 365 unorm_getFCD16FromSurrogatePair(const uint16_t *fcdTrieIndex, uint16_t fcd16, UChar c2) {
 366     return
 367         fcdTrieIndex[
 368             (fcdTrieIndex[
 369                 (int32_t)fcd16+((c2&0x3ff)>>UTRIE_SHIFT)
 370             ]<<UTRIE_INDEX_SHIFT)+
 371             (c2&UTRIE_MASK)
 372         ];
 373 }
 374
 375
 376 #endif
 377
 378 /**
 379  * internal API, used by StringPrep
 380  * @internal
 381  */
 382 U_CAPI void U_EXPORT2
 383 unorm_getUnicodeVersion(UVersionInfo *versionInfo, UErrorCode *pErrorCode);
 384
 385 /**
 386  * Get the canonical decomposition for one code point.
 387  * Requires unorm_haveData() and buffer!=NULL and pLength!=NULL.
 388  * @param c code point
 389  * @param buffer out-only buffer for algorithmic decompositions of Hangul
 390  * @param length out-only, takes the length of the decomposition, if any
 391  * @return pointer to decomposition, or 0 if none
 392  * @internal
 393  */
 394 U_CFUNC const UChar *
 395 unorm_getCanonicalDecomposition(UChar32 c, UChar buffer[4], int32_t *pLength);
 396
 397 /**
 398  * internal API, used by the canonical iterator
 399  * TODO Consider using signature similar to unorm_getCanonicalDecomposition()
 400  * for more efficiency
 401  * @internal
 402  */
 403 U_CAPI int32_t U_EXPORT2
 404 unorm_getDecomposition(UChar32 c, UBool compat,
 405                        UChar *dest, int32_t destCapacity);
 406
 407 /**
 408  * internal API, used by uprops.cpp
 409  * @internal
 410  */
 411 U_CFUNC UBool U_EXPORT2
 412 unorm_internalIsFullCompositionExclusion(UChar32 c);
 413
 414 /**
 415  * Internal API, used by enumeration of canonically equivalent strings
 416  * @internal
 417  */
 418 U_CFUNC UBool U_EXPORT2
 419 unorm_isCanonSafeStart(UChar32 c);
 420
 421 /**
 422  * Internal API, used by enumeration of canonically equivalent strings
 423  * @internal
 424  */
 425 U_CAPI UBool U_EXPORT2
 426 unorm_getCanonStartSet(UChar32 c, USerializedSet *fillSet);
 427
 428 /**
 429  * Is c an NF<mode>-skippable code point? See unormimp.h.
 430  * @internal
 431  */
 432 U_CAPI UBool U_EXPORT2
 433 unorm_isNFSkippable(UChar32 c, UNormalizationMode mode);
 434
 435 #ifdef XP_CPLUSPLUS
 436
 437 /**
 438  * Get normalization exclusion set for the options.
 439  * Requires unorm_haveData().
 440  * @internal
 441  */
 442 U_CFUNC const U_NAMESPACE_QUALIFIER UnicodeSet *
 443 unorm_getNX(int32_t options, UErrorCode *pErrorCode);
 444
 445 #endif
 446
 447 /**
 448  * Enumerate each normalization data trie and add the
 449  * start of each range of same properties to the set.
 450  * @internal
 451  */
 452 U_CAPI void U_EXPORT2
 453 unorm_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode);
 454
 455 /**
 456  * Swap unorm.icu. See udataswp.h.
 457  * @internal
 458  */
 459 U_CAPI int32_t U_EXPORT2
 460 unorm_swap(const UDataSwapper *ds,
 461            const void *inData, int32_t length, void *outData,
 462            UErrorCode *pErrorCode);
 463
 464 /**
 465  * Get the NF*_QC property for a code point, for u_getIntPropertyValue().
 466  * @internal
 467  */
 468 U_CFUNC UNormalizationCheckResult U_EXPORT2
 469 unorm_getQuickCheck(UChar32 c, UNormalizationMode mode);
 470
 471 /**
 472  * Description of the format of unorm.icu version 2.3.
 473  *
 474  * Main change from version 1 to version 2:
 475  * Use of new, common UTrie instead of normalization-specific tries.
 476  * Change to version 2.1: add third/auxiliary trie with associated data.
 477  * Change to version 2.2: add skippable (f) flag data (_NORM_AUX_NFC_SKIP_F_MASK).
 478  * Change to version 2.3: add serialized sets for normalization exclusions
 479  *                        stored inside canonStartSets[]
 480  *
 481  * For more details of how to use the data structures see the code
 482  * in unorm.cpp (runtime normalization code) and
 483  * in gennorm.c and gennorm/store.c (build-time data generation).
 484  *
 485  * For the serialized format of UTrie see utrie.c/UTrieHeader.
 486  *
 487  * - Overall partition
 488  *
 489  * unorm.dat customarily begins with a UDataInfo structure, see udata.h and .c.
 490  * After that there are the following structures:
 491  *
 492  * int32_t indexes[_NORM_INDEX_TOP];            -- _NORM_INDEX_TOP=32, see enum in this file
 493  *
 494  * UTrie normTrie;                              -- size in bytes=indexes[_NORM_INDEX_TRIE_SIZE]
 495  *
 496  * uint16_t extraData[extraDataTop];            -- extraDataTop=indexes[_NORM_INDEX_UCHAR_COUNT]
 497  *                                                 extraData[0] contains the number of units for
 498  *                                                 FC_NFKC_Closure (formatVersion>=2.1)
 499  *
 500  * uint16_t combiningTable[combiningTableTop];  -- combiningTableTop=indexes[_NORM_INDEX_COMBINE_DATA_COUNT]
 501  *                                                 combiningTableTop may include one 16-bit padding unit
 502  *                                                 to make sure that fcdTrie is 32-bit-aligned
 503  *
 504  * UTrie fcdTrie;                               -- size in bytes=indexes[_NORM_INDEX_FCD_TRIE_SIZE]
 505  *
 506  * UTrie auxTrie;                               -- size in bytes=indexes[_NORM_INDEX_AUX_TRIE_SIZE]
 507  *
 508  * uint16_t canonStartSets[canonStartSetsTop]   -- canonStartSetsTop=indexes[_NORM_INDEX_CANON_SET_COUNT]
 509  *                                                 serialized USets and binary search tables, see below
 510  *
 511  *
 512  * The indexes array contains lengths and sizes of the following arrays and structures
 513  * as well as the following values:
 514  *  indexes[_NORM_INDEX_COMBINE_FWD_COUNT]=combineFwdTop
 515  *      -- one more than the highest combining index computed for forward-only-combining characters
 516  *  indexes[_NORM_INDEX_COMBINE_BOTH_COUNT]=combineBothTop-combineFwdTop
 517  *      -- number of combining indexes computed for both-ways-combining characters
 518  *  indexes[_NORM_INDEX_COMBINE_BACK_COUNT]=combineBackTop-combineBothTop
 519  *      -- number of combining indexes computed for backward-only-combining characters
 520  *
 521  *  indexes[_NORM_INDEX_MIN_NF*_NO_MAYBE] (where *={ C, D, KC, KD })
 522  *      -- first code point with a quick check NF* value of NO/MAYBE
 523  *
 524  *
 525  * - Tries
 526  *
 527  * The main structures are two UTrie tables ("compact arrays"),
 528  * each with one index array and one data array.
 529  * See utrie.h and utrie.c.
 530  *
 531  *
 532  * - Tries in unorm.dat
 533  *
 534  * The first trie (normTrie above)
 535  * provides data for the NF* quick checks and normalization.
 536  * The second trie (fcdTrie above) provides data just for FCD checks.
 537  *
 538  *
 539  * - norm32 data words from the first trie
 540  *
 541  * The norm32Table contains one 32-bit word "norm32" per code point.
 542  * It contains the following bit fields:
 543  * 31..16   extra data index, _NORM_EXTRA_SHIFT is used to shift this field down
 544  *          if this index is <_NORM_EXTRA_INDEX_TOP then it is an index into
 545  *              extraData[] where variable-length normalization data for this
 546  *              code point is found
 547  *          if this index is <_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_SURROGATE_TOP
 548  *              then this is a norm32 for a leading surrogate, and the index
 549  *              value is used together with the following trailing surrogate
 550  *              code unit in the second trie access
 551  *          if this index is >=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_SURROGATE_TOP
 552  *              then this is a norm32 for a "special" character,
 553  *              i.e., the character is a Hangul syllable or a Jamo
 554  *              see _NORM_EXTRA_HANGUL etc.
 555  *          generally, instead of extracting this index from the norm32 and
 556  *              comparing it with the above constants,
 557  *              the normalization code compares the entire norm32 value
 558  *              with _NORM_MIN_SPECIAL, _NORM_SURROGATES_TOP, _NORM_MIN_HANGUL etc.
 559  *
 560  * 15..8    combining class (cc) according to UnicodeData.txt
 561  *
 562  *  7..6    _NORM_COMBINES_ANY flags, used in composition to see if a character
 563  *              combines with any following or preceding character(s)
 564  *              at all
 565  *     7    _NORM_COMBINES_BACK
 566  *     6    _NORM_COMBINES_FWD
 567  *
 568  *  5..0    quick check flags, set for "no" or "maybe", with separate flags for
 569  *              each normalization form
 570  *              the higher bits are "maybe" flags; for NF*D there are no such flags
 571  *              the lower bits are "no" flags for all forms, in the same order
 572  *              as the "maybe" flags,
 573  *              which is (MSB to LSB): NFKD NFD NFKC NFC
 574  *  5..4    _NORM_QC_ANY_MAYBE
 575  *  3..0    _NORM_QC_ANY_NO
 576  *              see further related constants
 577  *
 578  *
 579  * - Extra data per code point
 580  *
 581  * "Extra data" is referenced by the index in norm32.
 582  * It is variable-length data. It is only present, and only those parts
 583  * of it are, as needed for a given character.
 584  * The norm32 extra data index is added to the beginning of extraData[]
 585  * to get to a vector of 16-bit words with data at the following offsets:
 586  *
 587  * [-1]     Combining index for composition.
 588  *              Stored only if norm32&_NORM_COMBINES_ANY .
 589  * [0]      Lengths of the canonical and compatibility decomposition strings.
 590  *              Stored only if there are decompositions, i.e.,
 591  *              if norm32&(_NORM_QC_NFD|_NORM_QC_NFKD)
 592  *          High byte: length of NFKD, or 0 if none
 593  *          Low byte: length of NFD, or 0 if none
 594  *          Each length byte also has another flag:
 595  *              Bit 7 of a length byte is set if there are non-zero
 596  *              combining classes (cc's) associated with the respective
 597  *              decomposition. If this flag is set, then the decomposition
 598  *              is preceded by a 16-bit word that contains the
 599  *              leading and trailing cc's.
 600  *              Bits 6..0 of a length byte are the length of the
 601  *              decomposition string, not counting the cc word.
 602  * [1..n]   NFD
 603  * [n+1..]  NFKD
 604  *
 605  * Each of the two decompositions consists of up to two parts:
 606  * - The 16-bit words with the leading and trailing cc's.
 607  *   This is only stored if bit 7 of the corresponding length byte
 608  *   is set. In this case, at least one of the cc's is not zero.
 609  *   High byte: leading cc==cc of the first code point in the decomposition string
 610  *   Low byte: trailing cc==cc of the last code point in the decomposition string
 611  * - The decomposition string in UTF-16, with length code units.
 612  *
 613  *
 614  * - Combining indexes and combiningTable[]
 615  *
 616  * Combining indexes are stored at the [-1] offset of the extra data
 617  * if the character combines forward or backward with any other characters.
 618  * They are used for (re)composition in NF*C.
 619  * Values of combining indexes are arranged according to whether a character
 620  * combines forward, backward, or both ways:
 621  *    forward-only < both ways < backward-only
 622  *
 623  * The index values for forward-only and both-ways combining characters
 624  * are indexes into the combiningTable[].
 625  * The index values for backward-only combining characters are simply
 626  * incremented from the preceding index values to be unique.
 627  *
 628  * In the combiningTable[], a variable-length list
 629  * of variable-length (back-index, code point) pair entries is stored
 630  * for each forward-combining character.
 631  *
 632  * These back-indexes are the combining indexes of both-ways or backward-only
 633  * combining characters that the forward-combining character combines with.
 634  *
 635  * Each list is sorted in ascending order of back-indexes.
 636  * Each list is terminated with the last back-index having bit 15 set.
 637  *
 638  * Each pair (back-index, code point) takes up either 2 or 3
 639  * 16-bit words.
 640  * The first word of a list entry is the back-index, with its bit 15 set if
 641  * this is the last pair in the list.
 642  *
 643  * The second word contains flags in bits 15..13 that determine
 644  * if there is a third word and how the combined character is encoded:
 645  * 15   set if there is a third word in this list entry
 646  * 14   set if the result is a supplementary character
 647  * 13   set if the result itself combines forward
 648  *
 649  * According to these bits 15..14 of the second word,
 650  * the result character is encoded as follows:
 651  * 00 or 01 The result is <=0x1fff and stored in bits 12..0 of
 652  *          the second word.
 653  * 10       The result is 0x2000..0xffff and stored in the third word.
 654  *          Bits 12..0 of the second word are not used.
 655  * 11       The result is a supplementary character.
 656  *          Bits 9..0 of the leading surrogate are in bits 9..0 of
 657  *          the second word.
 658  *          Add 0xd800 to these bits to get the complete surrogate.
 659  *          Bits 12..10 of the second word are not used.
 660  *          The trailing surrogate is stored in the third word.
 661  *
 662  *
 663  * - FCD trie
 664  *
 665  * The FCD trie is very simple.
 666  * It is a folded trie with 16-bit data words.
 667  * In each word, the high byte contains the leading cc of the character,
 668  * and the low byte contains the trailing cc of the character.
 669  * These cc's are the cc's of the first and last code points in the
 670  * canonical decomposition of the character.
 671  *
 672  * Since all 16 bits are used for cc's, lead surrogates must be tested
 673  * by checking the code unit instead of the trie data.
 674  * This is done only if the 16-bit data word is not zero.
 675  * If the code unit is a leading surrogate and the data word is not zero,
 676  * then instead of cc's it contains the offset for the second trie lookup.
 677  *
 678  *
 679  * - Auxiliary trie and data
 680  *
 681  * The auxiliary 16-bit trie contains data for additional properties.
 682  * Bits
 683  * 15..13   reserved
 684  *     12   not NFC_Skippable (f) (formatVersion>=2.2)
 685  *     11   flag: not a safe starter for canonical closure
 686  *     10   composition exclusion
 687  *  9.. 0   index into extraData[] to FC_NFKC_Closure string
 688  *          (not for lead surrogate),
 689  *          or lead surrogate offset (for lead surrogate, if 9..0 not zero)
 690  *
 691  * - FC_NFKC_Closure strings in extraData[]
 692  *
 693  * Strings are either stored as a single code unit or as the length
 694  * followed by that many units.
 695  *   const UChar *s=extraData+(index from auxTrie data bits 9..0);
 696  *   int32_t length;
 697  *   if(*s<0xff00) {
 698  *     // s points to the single-unit string
 699  *     length=1;
 700  *   } else {
 701  *     length=*s&0xff;
 702  *     ++s;
 703  *   }
 704  *
 705  * Conditions for "NF* Skippable" from Mark Davis' com.ibm.text.UCD.NFSkippable:
 706  * (used in NormalizerTransliterator)
 707  *
 708  * A skippable character is
 709  * a) unassigned, or ALL of the following:
 710  * b) of combining class 0.
 711  * c) not decomposed by this normalization form.
 712  * AND if NFC or NFKC,
 713  * d) can never compose with a previous character.
 714  * e) can never compose with a following character.
 715  * f) can never change if another character is added.
 716  *    Example: a-breve might satisfy all but f, but if you
 717  *    add an ogonek it changes to a-ogonek + breve
 718  *
 719  * a)..e) must be tested from norm32.
 720  * Since f) is more complicated, the (not-)NFC_Skippable flag (f) is built
 721  * into the auxiliary trie.
 722  * The same bit is used for NFC and NFKC; (c) differs for them.
 723  * As usual, we build the "not skippable" flags so that unassigned
 724  * code points get a 0 bit.
 725  * This bit is only valid after (a)..(e) test FALSE; test NFD_NO before (f) as well.
 726  * Test Hangul LV syllables entirely in code.
 727  *
 728  *
 729  * - structure inside canonStartSets[]
 730  *
 731  * This array maps from code points c to sets of code points (USerializedSet).
 732  * The result sets are the code points whose canonical decompositions start
 733  * with c.
 734  *
 735  * canonStartSets[] contains the following sub-arrays:
 736  *
 737  * indexes[_NORM_SET_INDEX_TOP]
 738  *   - contains lengths of sub-arrays etc.
 739  *
 740  * startSets[indexes[_NORM_SET_INDEX_CANON_SETS_LENGTH]-_NORM_SET_INDEX_TOP]
 741  *   - contains serialized sets (USerializedSet) of canonical starters for
 742  *     enumerating canonically equivalent strings
 743  *     indexes[_NORM_SET_INDEX_CANON_SETS_LENGTH] includes _NORM_SET_INDEX_TOP
 744  *     for details about the structure see uset.c
 745  *
 746  * bmpTable[indexes[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]]
 747  *   - a sorted search table for BMP code points whose results are
 748  *     either indexes to USerializedSets or single code points for
 749  *     single-code point sets;
 750  *     each entry is a pair of { code point, result } with result=(binary) yy xxxxxx xxxxxxxx
 751  *     if yy==01 then there is a USerializedSet at canonStartSets+x
 752  *     else build a USerializedSet with result as the single code point
 753  *
 754  * suppTable[indexes[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]]
 755  *   - a sorted search table for supplementary code points whose results are
 756  *     either indexes to USerializedSets or single code points for
 757  *     single-code point sets;
 758  *     each entry is a triplet of { high16(cp), low16(cp), result }
 759  *     each code point's high-word may contain extra data in bits 15..5:
 760  *     if the high word has bit 15 set, then build a set with a single code point
 761  *     which is (((high16(cp)&0x1f00)<<8)|result;
 762  *     else there is a USerializedSet at canonStartSets+result
 763  *
 764  * FormatVersion 2.3 adds 2 serialized sets for normalization exclusions.
 765  * They are stored in the data file so that the runtime normalization code need
 766  * not depend on other properties and their data and implementation files.
 767  * The _NORM_SET_INDEX_NX_..._OFFSET offsets in the canonStartSets index table
 768  * give the location for each set.
 769  * There is no set stored for UNORM_NX_HANGUL because it's trivial to create
 770  * without using properties.
 771  *
 772  * Set contents:
 773  *
 774  * _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET (for UNORM_NX_CJK_COMPAT)
 775  *     [[:Ideographic:]&[:NFD_QC=No:]]
 776  *     =[CJK Ideographs]&[has canonical decomposition]
 777  *
 778  * _NORM_SET_INDEX_NX_UNICODE32_OFFSET (for UNORM_UNICODE_3_2)
 779  *     [:^Age=3.2:]
 780  *     =set with all code points that were not designated by the specified Unicode version
 781  *
 782  * _NORM_SET_INDEX_NX_RESERVED_OFFSET
 783  *     This is an offset that points to where the next, future set would start.
 784  *     Currently it indicates where the previous set ends, and thus its length.
 785  *     The name for this enum constant may in the future be applied to different
 786  *     index slots. In order to get the limit of a set, use its index slot and
 787  *     the immediately following one regardless of that one's enum name.
 788  */
 789
 790 #endif /* #if !UCONFIG_NO_NORMALIZATION */
 791
 792 #endif