icuSources/common/unicode/unorm.h

   1 /*
   2 *******************************************************************************
   3 * Copyright (c) 1996-2016, International Business Machines Corporation
   4 *               and others. All Rights Reserved.
   5 *******************************************************************************
   6 * File unorm.h
   7 *
   8 * Created by: Vladimir Weinstein 12052000
   9 *
  10 * Modification history :
  11 *
  12 * Date        Name        Description
  13 * 02/01/01    synwee      Added normalization quickcheck enum and method.
  14 */
  15 #ifndef UNORM_H
  16 #define UNORM_H
  17
  18 #include "unicode/utypes.h"
  19
  20 #if !UCONFIG_NO_NORMALIZATION
  21
  22 #include "unicode/uiter.h"
  23 #include "unicode/unorm2.h"
  24
  25 #ifndef U_HIDE_DEPRECATED_API
  26
  27 /**
  28  * \file
  29  * \brief C API: Unicode Normalization
  30  *
  31  * Old Unicode normalization API.
  32  *
  33  * This API has been replaced by the unorm2.h API and is only available
  34  * for backward compatibility. The functions here simply delegate to the
  35  * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize().
  36  * There is one exception: The new API does not provide a replacement for unorm_compare().
  37  * Its declaration has been moved to unorm2.h.
  38  *
  39  * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or
  40  * decomposed form, allowing for easier sorting and searching of text.
  41  * <code>unorm_normalize</code> supports the standard normalization forms described in
  42  * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">
  43  * Unicode Standard Annex #15: Unicode Normalization Forms</a>.
  44  *
  45  * Characters with accents or other adornments can be encoded in
  46  * several different ways in Unicode.  For example, take the character A-acute.
  47  * In Unicode, this can be encoded as a single character (the
  48  * "composed" form):
  49  *
  50  * \code
  51  *      00C1    LATIN CAPITAL LETTER A WITH ACUTE
  52  * \endcode
  53  *
  54  * or as two separate characters (the "decomposed" form):
  55  *
  56  * \code
  57  *      0041    LATIN CAPITAL LETTER A
  58  *      0301    COMBINING ACUTE ACCENT
  59  * \endcode
  60  *
  61  * To a user of your program, however, both of these sequences should be
  62  * treated as the same "user-level" character "A with acute accent".  When you are searching or
  63  * comparing text, you must ensure that these two sequences are treated
  64  * equivalently.  In addition, you must handle characters with more than one
  65  * accent.  Sometimes the order of a character's combining accents is
  66  * significant, while in other cases accent sequences in different orders are
  67  * really equivalent.
  68  *
  69  * Similarly, the string "ffi" can be encoded as three separate letters:
  70  *
  71  * \code
  72  *      0066    LATIN SMALL LETTER F
  73  *      0066    LATIN SMALL LETTER F
  74  *      0069    LATIN SMALL LETTER I
  75  * \endcode
  76  *
  77  * or as the single character
  78  *
  79  * \code
  80  *      FB03    LATIN SMALL LIGATURE FFI
  81  * \endcode
  82  *
  83  * The ffi ligature is not a distinct semantic character, and strictly speaking
  84  * it shouldn't be in Unicode at all, but it was included for compatibility
  85  * with existing character sets that already provided it.  The Unicode standard
  86  * identifies such characters by giving them "compatibility" decompositions
  87  * into the corresponding semantic characters.  When sorting and searching, you
  88  * will often want to use these mappings.
  89  *
  90  * <code>unorm_normalize</code> helps solve these problems by transforming text into the
  91  * canonical composed and decomposed forms as shown in the first example above.
  92  * In addition, you can have it perform compatibility decompositions so that
  93  * you can treat compatibility characters the same as their equivalents.
  94  * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical
  95  * order, so that you do not have to worry about accent rearrangement on your
  96  * own.
  97  *
  98  * Form FCD, "Fast C or D", is also designed for collation.
  99  * It allows to work on strings that are not necessarily normalized
 100  * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed
 101  * characters and their decomposed equivalents the same.
 102  *
 103  * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings
 104  * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical
 105  * themselves.
 106  *
 107  * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,
 108  * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long
 109  * as their decompositions do not need canonical reordering.
 110  *
 111  * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -
 112  * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will
 113  * return UNORM_YES for most strings in practice.
 114  *
 115  * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.
 116  *
 117  * For more details on FCD see the collation design document:
 118  * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
 119  *
 120  * ICU collation performs either NFD or FCD normalization automatically if normalization
 121  * is turned on for the collator object.
 122  * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,
 123  * transliteration/transcription, unique representations, etc.
 124  *
 125  * The W3C generally recommends to exchange texts in NFC.
 126  * Note also that most legacy character encodings use only precomposed forms and often do not
 127  * encode any combining marks by themselves. For conversion to such character encodings the
 128  * Unicode text needs to be normalized to NFC.
 129  * For more usage examples, see the Unicode Standard Annex.
 130  */
 131
 132 /**
 133  * Constants for normalization modes.
 134  * @deprecated ICU 56 Use unorm2.h instead.
 135  */
 136 typedef enum {
 137   /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */
 138   UNORM_NONE = 1,
 139   /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */
 140   UNORM_NFD = 2,
 141   /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */
 142   UNORM_NFKD = 3,
 143   /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
 144   UNORM_NFC = 4,
 145   /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */
 146   UNORM_DEFAULT = UNORM_NFC,
 147   /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
 148   UNORM_NFKC =5,
 149   /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */
 150   UNORM_FCD = 6,
 151
 152   /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */
 153   UNORM_MODE_COUNT
 154 } UNormalizationMode;
 155
 156 /**
 157  * Constants for options flags for normalization.
 158  * Use 0 for default options,
 159  * including normalization according to the Unicode version
 160  * that is currently supported by ICU (see u_getUnicodeVersion).
 161  * @deprecated ICU 56 Use unorm2.h instead.
 162  */
 163 enum {
 164     /**
 165      * Options bit set value to select Unicode 3.2 normalization
 166      * (except NormalizationCorrections).
 167      * At most one Unicode version can be selected at a time.
 168      * @deprecated ICU 56 Use unorm2.h instead.
 169      */
 170     UNORM_UNICODE_3_2=0x20
 171 };
 172
 173 /**
 174  * Lowest-order bit number of unorm_compare() options bits corresponding to
 175  * normalization options bits.
 176  *
 177  * The options parameter for unorm_compare() uses most bits for
 178  * itself and for various comparison and folding flags.
 179  * The most significant bits, however, are shifted down and passed on
 180  * to the normalization implementation.
 181  * (That is, from unorm_compare(..., options, ...),
 182  * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
 183  * internal normalization functions.)
 184  *
 185  * @see unorm_compare
 186  * @deprecated ICU 56 Use unorm2.h instead.
 187  */
 188 #define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20
 189
 190 /**
 191  * Normalize a string.
 192  * The string will be normalized according the specified normalization mode
 193  * and options.
 194  * The source and result buffers must not be the same, nor overlap.
 195  *
 196  * @param source The string to normalize.
 197  * @param sourceLength The length of source, or -1 if NUL-terminated.
 198  * @param mode The normalization mode; one of UNORM_NONE,
 199  *             UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.
 200  * @param options The normalization options, ORed together (0 for no options).
 201  * @param result A pointer to a buffer to receive the result string.
 202  *               The result string is NUL-terminated if possible.
 203  * @param resultLength The maximum size of result.
 204  * @param status A pointer to a UErrorCode to receive any errors.
 205  * @return The total buffer size needed; if greater than resultLength,
 206  *         the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.
 207  * @deprecated ICU 56 Use unorm2.h instead.
 208  */
 209 U_STABLE int32_t U_EXPORT2
 210 unorm_normalize(const UChar *source, int32_t sourceLength,
 211                 UNormalizationMode mode, int32_t options,
 212                 UChar *result, int32_t resultLength,
 213                 UErrorCode *status);
 214
 215 /**
 216  * Performing quick check on a string, to quickly determine if the string is
 217  * in a particular normalization format.
 218  * Three types of result can be returned UNORM_YES, UNORM_NO or
 219  * UNORM_MAYBE. Result UNORM_YES indicates that the argument
 220  * string is in the desired normalized format, UNORM_NO determines that
 221  * argument string is not in the desired normalized format. A
 222  * UNORM_MAYBE result indicates that a more thorough check is required,
 223  * the user may have to put the string in its normalized form and compare the
 224  * results.
 225  *
 226  * @param source       string for determining if it is in a normalized format
 227  * @param sourcelength length of source to test, or -1 if NUL-terminated
 228  * @param mode         which normalization form to test for
 229  * @param status       a pointer to a UErrorCode to receive any errors
 230  * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
 231  *
 232  * @see unorm_isNormalized
 233  * @deprecated ICU 56 Use unorm2.h instead.
 234  */
 235 U_STABLE UNormalizationCheckResult U_EXPORT2
 236 unorm_quickCheck(const UChar *source, int32_t sourcelength,
 237                  UNormalizationMode mode,
 238                  UErrorCode *status);
 239
 240 /**
 241  * Performing quick check on a string; same as unorm_quickCheck but
 242  * takes an extra options parameter like most normalization functions.
 243  *
 244  * @param src        String that is to be tested if it is in a normalization format.
 245  * @param srcLength  Length of source to test, or -1 if NUL-terminated.
 246  * @param mode       Which normalization form to test for.
 247  * @param options    The normalization options, ORed together (0 for no options).
 248  * @param pErrorCode ICU error code in/out parameter.
 249  *                   Must fulfill U_SUCCESS before the function call.
 250  * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
 251  *
 252  * @see unorm_quickCheck
 253  * @see unorm_isNormalized
 254  * @deprecated ICU 56 Use unorm2.h instead.
 255  */
 256 U_STABLE UNormalizationCheckResult U_EXPORT2
 257 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
 258                             UNormalizationMode mode, int32_t options,
 259                             UErrorCode *pErrorCode);
 260
 261 /**
 262  * Test if a string is in a given normalization form.
 263  * This is semantically equivalent to source.equals(normalize(source, mode)) .
 264  *
 265  * Unlike unorm_quickCheck(), this function returns a definitive result,
 266  * never a "maybe".
 267  * For NFD, NFKD, and FCD, both functions work exactly the same.
 268  * For NFC and NFKC where quickCheck may return "maybe", this function will
 269  * perform further tests to arrive at a TRUE/FALSE result.
 270  *
 271  * @param src        String that is to be tested if it is in a normalization format.
 272  * @param srcLength  Length of source to test, or -1 if NUL-terminated.
 273  * @param mode       Which normalization form to test for.
 274  * @param pErrorCode ICU error code in/out parameter.
 275  *                   Must fulfill U_SUCCESS before the function call.
 276  * @return Boolean value indicating whether the source string is in the
 277  *         "mode" normalization form.
 278  *
 279  * @see unorm_quickCheck
 280  * @deprecated ICU 56 Use unorm2.h instead.
 281  */
 282 U_STABLE UBool U_EXPORT2
 283 unorm_isNormalized(const UChar *src, int32_t srcLength,
 284                    UNormalizationMode mode,
 285                    UErrorCode *pErrorCode);
 286
 287 /**
 288  * Test if a string is in a given normalization form; same as unorm_isNormalized but
 289  * takes an extra options parameter like most normalization functions.
 290  *
 291  * @param src        String that is to be tested if it is in a normalization format.
 292  * @param srcLength  Length of source to test, or -1 if NUL-terminated.
 293  * @param mode       Which normalization form to test for.
 294  * @param options    The normalization options, ORed together (0 for no options).
 295  * @param pErrorCode ICU error code in/out parameter.
 296  *                   Must fulfill U_SUCCESS before the function call.
 297  * @return Boolean value indicating whether the source string is in the
 298  *         "mode/options" normalization form.
 299  *
 300  * @see unorm_quickCheck
 301  * @see unorm_isNormalized
 302  * @deprecated ICU 56 Use unorm2.h instead.
 303  */
 304 U_STABLE UBool U_EXPORT2
 305 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
 306                               UNormalizationMode mode, int32_t options,
 307                               UErrorCode *pErrorCode);
 308
 309 /**
 310  * Iterative normalization forward.
 311  * This function (together with unorm_previous) is somewhat
 312  * similar to the C++ Normalizer class (see its non-static functions).
 313  *
 314  * Iterative normalization is useful when only a small portion of a longer
 315  * string/text needs to be processed.
 316  *
 317  * For example, the likelihood may be high that processing the first 10% of some
 318  * text will be sufficient to find certain data.
 319  * Another example: When one wants to concatenate two normalized strings and get a
 320  * normalized result, it is much more efficient to normalize just a small part of
 321  * the result around the concatenation place instead of re-normalizing everything.
 322  *
 323  * The input text is an instance of the C character iteration API UCharIterator.
 324  * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any
 325  * other kind of text object.
 326  *
 327  * If a buffer overflow occurs, then the caller needs to reset the iterator to the
 328  * old index and call the function again with a larger buffer - if the caller cares
 329  * for the actual output.
 330  * Regardless of the output buffer, the iterator will always be moved to the next
 331  * normalization boundary.
 332  *
 333  * This function (like unorm_previous) serves two purposes:
 334  *
 335  * 1) To find the next boundary so that the normalization of the part of the text
 336  * from the current position to that boundary does not affect and is not affected
 337  * by the part of the text beyond that boundary.
 338  *
 339  * 2) To normalize the text up to the boundary.
 340  *
 341  * The second step is optional, per the doNormalize parameter.
 342  * It is omitted for operations like string concatenation, where the two adjacent
 343  * string ends need to be normalized together.
 344  * In such a case, the output buffer will just contain a copy of the text up to the
 345  * boundary.
 346  *
 347  * pNeededToNormalize is an output-only parameter. Its output value is only defined
 348  * if normalization was requested (doNormalize) and successful (especially, no
 349  * buffer overflow).
 350  * It is useful for operations like a normalizing transliterator, where one would
 351  * not want to replace a piece of text if it is not modified.
 352  *
 353  * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE
 354  * if the normalization was necessary.
 355  *
 356  * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE.
 357  *
 358  * If the buffer overflows, then *pNeededToNormalize will be undefined;
 359  * essentially, whenever U_FAILURE is true (like in buffer overflows), this result
 360  * will be undefined.
 361  *
 362  * @param src The input text in the form of a C character iterator.
 363  * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
 364  * @param destCapacity The number of UChars that fit into dest.
 365  * @param mode The normalization mode.
 366  * @param options The normalization options, ORed together (0 for no options).
 367  * @param doNormalize Indicates if the source text up to the next boundary
 368  *                    is to be normalized (TRUE) or just copied (FALSE).
 369  * @param pNeededToNormalize Output flag indicating if the normalization resulted in
 370  *                           different text from the input.
 371  *                           Not defined if an error occurs including buffer overflow.
 372  *                           Always FALSE if !doNormalize.
 373  * @param pErrorCode ICU error code in/out parameter.
 374  *                   Must fulfill U_SUCCESS before the function call.
 375  * @return Length of output (number of UChars) when successful or buffer overflow.
 376  *
 377  * @see unorm_previous
 378  * @see unorm_normalize
 379  *
 380  * @deprecated ICU 56 Use unorm2.h instead.
 381  */
 382 U_STABLE int32_t U_EXPORT2
 383 unorm_next(UCharIterator *src,
 384            UChar *dest, int32_t destCapacity,
 385            UNormalizationMode mode, int32_t options,
 386            UBool doNormalize, UBool *pNeededToNormalize,
 387            UErrorCode *pErrorCode);
 388
 389 /**
 390  * Iterative normalization backward.
 391  * This function (together with unorm_next) is somewhat
 392  * similar to the C++ Normalizer class (see its non-static functions).
 393  * For all details see unorm_next.
 394  *
 395  * @param src The input text in the form of a C character iterator.
 396  * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
 397  * @param destCapacity The number of UChars that fit into dest.
 398  * @param mode The normalization mode.
 399  * @param options The normalization options, ORed together (0 for no options).
 400  * @param doNormalize Indicates if the source text up to the next boundary
 401  *                    is to be normalized (TRUE) or just copied (FALSE).
 402  * @param pNeededToNormalize Output flag indicating if the normalization resulted in
 403  *                           different text from the input.
 404  *                           Not defined if an error occurs including buffer overflow.
 405  *                           Always FALSE if !doNormalize.
 406  * @param pErrorCode ICU error code in/out parameter.
 407  *                   Must fulfill U_SUCCESS before the function call.
 408  * @return Length of output (number of UChars) when successful or buffer overflow.
 409  *
 410  * @see unorm_next
 411  * @see unorm_normalize
 412  *
 413  * @deprecated ICU 56 Use unorm2.h instead.
 414  */
 415 U_STABLE int32_t U_EXPORT2
 416 unorm_previous(UCharIterator *src,
 417                UChar *dest, int32_t destCapacity,
 418                UNormalizationMode mode, int32_t options,
 419                UBool doNormalize, UBool *pNeededToNormalize,
 420                UErrorCode *pErrorCode);
 421
 422 /**
 423  * Concatenate normalized strings, making sure that the result is normalized as well.
 424  *
 425  * If both the left and the right strings are in
 426  * the normalization form according to "mode/options",
 427  * then the result will be
 428  *
 429  * \code
 430  *     dest=normalize(left+right, mode, options)
 431  * \endcode
 432  *
 433  * With the input strings already being normalized,
 434  * this function will use unorm_next() and unorm_previous()
 435  * to find the adjacent end pieces of the input strings.
 436  * Only the concatenation of these end pieces will be normalized and
 437  * then concatenated with the remaining parts of the input strings.
 438  *
 439  * It is allowed to have dest==left to avoid copying the entire left string.
 440  *
 441  * @param left Left source string, may be same as dest.
 442  * @param leftLength Length of left source string, or -1 if NUL-terminated.
 443  * @param right Right source string. Must not be the same as dest, nor overlap.
 444  * @param rightLength Length of right source string, or -1 if NUL-terminated.
 445  * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
 446  * @param destCapacity The number of UChars that fit into dest.
 447  * @param mode The normalization mode.
 448  * @param options The normalization options, ORed together (0 for no options).
 449  * @param pErrorCode ICU error code in/out parameter.
 450  *                   Must fulfill U_SUCCESS before the function call.
 451  * @return Length of output (number of UChars) when successful or buffer overflow.
 452  *
 453  * @see unorm_normalize
 454  * @see unorm_next
 455  * @see unorm_previous
 456  *
 457  * @deprecated ICU 56 Use unorm2.h instead.
 458  */
 459 U_STABLE int32_t U_EXPORT2
 460 unorm_concatenate(const UChar *left, int32_t leftLength,
 461                   const UChar *right, int32_t rightLength,
 462                   UChar *dest, int32_t destCapacity,
 463                   UNormalizationMode mode, int32_t options,
 464                   UErrorCode *pErrorCode);
 465
 466 #endif  /* U_HIDE_DEPRECATED_API */
 467 #endif /* #if !UCONFIG_NO_NORMALIZATION */
 468 #endif