icuSources/common/unicode/unorm2.h

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2009-2012, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  unorm2.h
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2009dec15
  14 *   created by: Markus W. Scherer
  15 */
  16
  17 #ifndef __UNORM2_H__
  18 #define __UNORM2_H__
  19
  20 /**
  21  * \file
  22  * \brief C API: New API for Unicode Normalization.
  23  *
  24  * Unicode normalization functionality for standard Unicode normalization or
  25  * for using custom mapping tables.
  26  * All instances of UNormalizer2 are unmodifiable/immutable.
  27  * Instances returned by unorm2_getInstance() are singletons that must not be deleted by the caller.
  28  * For more details see the Normalizer2 C++ class.
  29  */
  30
  31 #include "unicode/utypes.h"
  32 #include "unicode/localpointer.h"
  33 #include "unicode/uset.h"
  34
  35 /**
  36  * Constants for normalization modes.
  37  * For details about standard Unicode normalization forms
  38  * and about the algorithms which are also used with custom mapping tables
  39  * see http://www.unicode.org/unicode/reports/tr15/
  40  * @stable ICU 4.4
  41  */
  42 typedef enum {
  43     /**
  44      * Decomposition followed by composition.
  45      * Same as standard NFC when using an "nfc" instance.
  46      * Same as standard NFKC when using an "nfkc" instance.
  47      * For details about standard Unicode normalization forms
  48      * see http://www.unicode.org/unicode/reports/tr15/
  49      * @stable ICU 4.4
  50      */
  51     UNORM2_COMPOSE,
  52     /**
  53      * Map, and reorder canonically.
  54      * Same as standard NFD when using an "nfc" instance.
  55      * Same as standard NFKD when using an "nfkc" instance.
  56      * For details about standard Unicode normalization forms
  57      * see http://www.unicode.org/unicode/reports/tr15/
  58      * @stable ICU 4.4
  59      */
  60     UNORM2_DECOMPOSE,
  61     /**
  62      * "Fast C or D" form.
  63      * If a string is in this form, then further decomposition <i>without reordering</i>
  64      * would yield the same form as DECOMPOSE.
  65      * Text in "Fast C or D" form can be processed efficiently with data tables
  66      * that are "canonically closed", that is, that provide equivalent data for
  67      * equivalent text, without having to be fully normalized.
  68      * Not a standard Unicode normalization form.
  69      * Not a unique form: Different FCD strings can be canonically equivalent.
  70      * For details see http://www.unicode.org/notes/tn5/#FCD
  71      * @stable ICU 4.4
  72      */
  73     UNORM2_FCD,
  74     /**
  75      * Compose only contiguously.
  76      * Also known as "FCC" or "Fast C Contiguous".
  77      * The result will often but not always be in NFC.
  78      * The result will conform to FCD which is useful for processing.
  79      * Not a standard Unicode normalization form.
  80      * For details see http://www.unicode.org/notes/tn5/#FCC
  81      * @stable ICU 4.4
  82      */
  83     UNORM2_COMPOSE_CONTIGUOUS
  84 } UNormalization2Mode;
  85
  86 /**
  87  * Result values for normalization quick check functions.
  88  * For details see http://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
  89  * @stable ICU 2.0
  90  */
  91 typedef enum UNormalizationCheckResult {
  92   /**
  93    * The input string is not in the normalization form.
  94    * @stable ICU 2.0
  95    */
  96   UNORM_NO,
  97   /**
  98    * The input string is in the normalization form.
  99    * @stable ICU 2.0
 100    */
 101   UNORM_YES,
 102   /**
 103    * The input string may or may not be in the normalization form.
 104    * This value is only returned for composition forms like NFC and FCC,
 105    * when a backward-combining character is found for which the surrounding text
 106    * would have to be analyzed further.
 107    * @stable ICU 2.0
 108    */
 109   UNORM_MAYBE
 110 } UNormalizationCheckResult;
 111
 112 /**
 113  * Opaque C service object type for the new normalization API.
 114  * @stable ICU 4.4
 115  */
 116 struct UNormalizer2;
 117 typedef struct UNormalizer2 UNormalizer2;  /**< C typedef for struct UNormalizer2. @stable ICU 4.4 */
 118
 119 #if !UCONFIG_NO_NORMALIZATION
 120
 121 #ifndef U_HIDE_DRAFT_API
 122 /**
 123  * Returns a UNormalizer2 instance for Unicode NFC normalization.
 124  * Same as unorm2_getInstance(NULL, "nfc", UNORM2_COMPOSE, pErrorCode).
 125  * Returns an unmodifiable singleton instance. Do not delete it.
 126  * @param errorCode Standard ICU error code. Its input value must
 127  *                  pass the U_SUCCESS() test, or else the function returns
 128  *                  immediately. Check for U_FAILURE() on output or use with
 129  *                  function chaining. (See User Guide for details.)
 130  * @return the requested Normalizer2, if successful
 131  * @draft ICU 49
 132  */
 133 U_DRAFT const UNormalizer2 * U_EXPORT2
 134 unorm2_getNFCInstance(UErrorCode *pErrorCode);
 135
 136 /**
 137  * Returns a UNormalizer2 instance for Unicode NFD normalization.
 138  * Same as unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, pErrorCode).
 139  * Returns an unmodifiable singleton instance. Do not delete it.
 140  * @param errorCode Standard ICU error code. Its input value must
 141  *                  pass the U_SUCCESS() test, or else the function returns
 142  *                  immediately. Check for U_FAILURE() on output or use with
 143  *                  function chaining. (See User Guide for details.)
 144  * @return the requested Normalizer2, if successful
 145  * @draft ICU 49
 146  */
 147 U_DRAFT const UNormalizer2 * U_EXPORT2
 148 unorm2_getNFDInstance(UErrorCode *pErrorCode);
 149
 150 /**
 151  * Returns a UNormalizer2 instance for Unicode NFKC normalization.
 152  * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_COMPOSE, pErrorCode).
 153  * Returns an unmodifiable singleton instance. Do not delete it.
 154  * @param errorCode Standard ICU error code. Its input value must
 155  *                  pass the U_SUCCESS() test, or else the function returns
 156  *                  immediately. Check for U_FAILURE() on output or use with
 157  *                  function chaining. (See User Guide for details.)
 158  * @return the requested Normalizer2, if successful
 159  * @draft ICU 49
 160  */
 161 U_DRAFT const UNormalizer2 * U_EXPORT2
 162 unorm2_getNFKCInstance(UErrorCode *pErrorCode);
 163
 164 /**
 165  * Returns a UNormalizer2 instance for Unicode NFKD normalization.
 166  * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, pErrorCode).
 167  * Returns an unmodifiable singleton instance. Do not delete it.
 168  * @param errorCode Standard ICU error code. Its input value must
 169  *                  pass the U_SUCCESS() test, or else the function returns
 170  *                  immediately. Check for U_FAILURE() on output or use with
 171  *                  function chaining. (See User Guide for details.)
 172  * @return the requested Normalizer2, if successful
 173  * @draft ICU 49
 174  */
 175 U_DRAFT const UNormalizer2 * U_EXPORT2
 176 unorm2_getNFKDInstance(UErrorCode *pErrorCode);
 177
 178 /**
 179  * Returns a UNormalizer2 instance for Unicode NFKC_Casefold normalization.
 180  * Same as unorm2_getInstance(NULL, "nfkc_cf", UNORM2_COMPOSE, pErrorCode).
 181  * Returns an unmodifiable singleton instance. Do not delete it.
 182  * @param errorCode Standard ICU error code. Its input value must
 183  *                  pass the U_SUCCESS() test, or else the function returns
 184  *                  immediately. Check for U_FAILURE() on output or use with
 185  *                  function chaining. (See User Guide for details.)
 186  * @return the requested Normalizer2, if successful
 187  * @draft ICU 49
 188  */
 189 U_DRAFT const UNormalizer2 * U_EXPORT2
 190 unorm2_getNFKCCasefoldInstance(UErrorCode *pErrorCode);
 191 #endif  /* U_HIDE_DRAFT_API */
 192
 193 /**
 194  * Returns a UNormalizer2 instance which uses the specified data file
 195  * (packageName/name similar to ucnv_openPackage() and ures_open()/ResourceBundle)
 196  * and which composes or decomposes text according to the specified mode.
 197  * Returns an unmodifiable singleton instance. Do not delete it.
 198  *
 199  * Use packageName=NULL for data files that are part of ICU's own data.
 200  * Use name="nfc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFC/NFD.
 201  * Use name="nfkc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFKC/NFKD.
 202  * Use name="nfkc_cf" and UNORM2_COMPOSE for Unicode standard NFKC_CF=NFKC_Casefold.
 203  *
 204  * @param packageName NULL for ICU built-in data, otherwise application data package name
 205  * @param name "nfc" or "nfkc" or "nfkc_cf" or name of custom data file
 206  * @param mode normalization mode (compose or decompose etc.)
 207  * @param pErrorCode Standard ICU error code. Its input value must
 208  *                  pass the U_SUCCESS() test, or else the function returns
 209  *                  immediately. Check for U_FAILURE() on output or use with
 210  *                  function chaining. (See User Guide for details.)
 211  * @return the requested UNormalizer2, if successful
 212  * @stable ICU 4.4
 213  */
 214 U_STABLE const UNormalizer2 * U_EXPORT2
 215 unorm2_getInstance(const char *packageName,
 216                    const char *name,
 217                    UNormalization2Mode mode,
 218                    UErrorCode *pErrorCode);
 219
 220 /**
 221  * Constructs a filtered normalizer wrapping any UNormalizer2 instance
 222  * and a filter set.
 223  * Both are aliased and must not be modified or deleted while this object
 224  * is used.
 225  * The filter set should be frozen; otherwise the performance will suffer greatly.
 226  * @param norm2 wrapped UNormalizer2 instance
 227  * @param filterSet USet which determines the characters to be normalized
 228  * @param pErrorCode Standard ICU error code. Its input value must
 229  *                   pass the U_SUCCESS() test, or else the function returns
 230  *                   immediately. Check for U_FAILURE() on output or use with
 231  *                   function chaining. (See User Guide for details.)
 232  * @return the requested UNormalizer2, if successful
 233  * @stable ICU 4.4
 234  */
 235 U_STABLE UNormalizer2 * U_EXPORT2
 236 unorm2_openFiltered(const UNormalizer2 *norm2, const USet *filterSet, UErrorCode *pErrorCode);
 237
 238 /**
 239  * Closes a UNormalizer2 instance from unorm2_openFiltered().
 240  * Do not close instances from unorm2_getInstance()!
 241  * @param norm2 UNormalizer2 instance to be closed
 242  * @stable ICU 4.4
 243  */
 244 U_STABLE void U_EXPORT2
 245 unorm2_close(UNormalizer2 *norm2);
 246
 247 #if U_SHOW_CPLUSPLUS_API
 248
 249 U_NAMESPACE_BEGIN
 250
 251 /**
 252  * \class LocalUNormalizer2Pointer
 253  * "Smart pointer" class, closes a UNormalizer2 via unorm2_close().
 254  * For most methods see the LocalPointerBase base class.
 255  *
 256  * @see LocalPointerBase
 257  * @see LocalPointer
 258  * @stable ICU 4.4
 259  */
 260 U_DEFINE_LOCAL_OPEN_POINTER(LocalUNormalizer2Pointer, UNormalizer2, unorm2_close);
 261
 262 U_NAMESPACE_END
 263
 264 #endif
 265
 266 /**
 267  * Writes the normalized form of the source string to the destination string
 268  * (replacing its contents) and returns the length of the destination string.
 269  * The source and destination strings must be different buffers.
 270  * @param norm2 UNormalizer2 instance
 271  * @param src source string
 272  * @param length length of the source string, or -1 if NUL-terminated
 273  * @param dest destination string; its contents is replaced with normalized src
 274  * @param capacity number of UChars that can be written to dest
 275  * @param pErrorCode Standard ICU error code. Its input value must
 276  *                   pass the U_SUCCESS() test, or else the function returns
 277  *                   immediately. Check for U_FAILURE() on output or use with
 278  *                   function chaining. (See User Guide for details.)
 279  * @return dest
 280  * @stable ICU 4.4
 281  */
 282 U_STABLE int32_t U_EXPORT2
 283 unorm2_normalize(const UNormalizer2 *norm2,
 284                  const UChar *src, int32_t length,
 285                  UChar *dest, int32_t capacity,
 286                  UErrorCode *pErrorCode);
 287 /**
 288  * Appends the normalized form of the second string to the first string
 289  * (merging them at the boundary) and returns the length of the first string.
 290  * The result is normalized if the first string was normalized.
 291  * The first and second strings must be different buffers.
 292  * @param norm2 UNormalizer2 instance
 293  * @param first string, should be normalized
 294  * @param firstLength length of the first string, or -1 if NUL-terminated
 295  * @param firstCapacity number of UChars that can be written to first
 296  * @param second string, will be normalized
 297  * @param secondLength length of the source string, or -1 if NUL-terminated
 298  * @param pErrorCode Standard ICU error code. Its input value must
 299  *                   pass the U_SUCCESS() test, or else the function returns
 300  *                   immediately. Check for U_FAILURE() on output or use with
 301  *                   function chaining. (See User Guide for details.)
 302  * @return first
 303  * @stable ICU 4.4
 304  */
 305 U_STABLE int32_t U_EXPORT2
 306 unorm2_normalizeSecondAndAppend(const UNormalizer2 *norm2,
 307                                 UChar *first, int32_t firstLength, int32_t firstCapacity,
 308                                 const UChar *second, int32_t secondLength,
 309                                 UErrorCode *pErrorCode);
 310 /**
 311  * Appends the second string to the first string
 312  * (merging them at the boundary) and returns the length of the first string.
 313  * The result is normalized if both the strings were normalized.
 314  * The first and second strings must be different buffers.
 315  * @param norm2 UNormalizer2 instance
 316  * @param first string, should be normalized
 317  * @param firstLength length of the first string, or -1 if NUL-terminated
 318  * @param firstCapacity number of UChars that can be written to first
 319  * @param second string, should be normalized
 320  * @param secondLength length of the source string, or -1 if NUL-terminated
 321  * @param pErrorCode Standard ICU error code. Its input value must
 322  *                   pass the U_SUCCESS() test, or else the function returns
 323  *                   immediately. Check for U_FAILURE() on output or use with
 324  *                   function chaining. (See User Guide for details.)
 325  * @return first
 326  * @stable ICU 4.4
 327  */
 328 U_STABLE int32_t U_EXPORT2
 329 unorm2_append(const UNormalizer2 *norm2,
 330               UChar *first, int32_t firstLength, int32_t firstCapacity,
 331               const UChar *second, int32_t secondLength,
 332               UErrorCode *pErrorCode);
 333
 334 /**
 335  * Gets the decomposition mapping of c.
 336  * Roughly equivalent to normalizing the String form of c
 337  * on a UNORM2_DECOMPOSE UNormalizer2 instance, but much faster, and except that this function
 338  * returns a negative value and does not write a string
 339  * if c does not have a decomposition mapping in this instance's data.
 340  * This function is independent of the mode of the UNormalizer2.
 341  * @param norm2 UNormalizer2 instance
 342  * @param c code point
 343  * @param decomposition String buffer which will be set to c's
 344  *                      decomposition mapping, if there is one.
 345  * @param capacity number of UChars that can be written to decomposition
 346  * @param pErrorCode Standard ICU error code. Its input value must
 347  *                   pass the U_SUCCESS() test, or else the function returns
 348  *                   immediately. Check for U_FAILURE() on output or use with
 349  *                   function chaining. (See User Guide for details.)
 350  * @return the non-negative length of c's decomposition, if there is one; otherwise a negative value
 351  * @stable ICU 4.6
 352  */
 353 U_DRAFT int32_t U_EXPORT2
 354 unorm2_getDecomposition(const UNormalizer2 *norm2,
 355                         UChar32 c, UChar *decomposition, int32_t capacity,
 356                         UErrorCode *pErrorCode);
 357
 358 #ifndef U_HIDE_DRAFT_API
 359 /**
 360  * Gets the raw decomposition mapping of c.
 361  *
 362  * This is similar to the unorm2_getDecomposition() function but returns the
 363  * raw decomposition mapping as specified in UnicodeData.txt or
 364  * (for custom data) in the mapping files processed by the gennorm2 tool.
 365  * By contrast, unorm2_getDecomposition() returns the processed,
 366  * recursively-decomposed version of this mapping.
 367  *
 368  * When used on a standard NFKC Normalizer2 instance,
 369  * unorm2_getRawDecomposition() returns the Unicode Decomposition_Mapping (dm) property.
 370  *
 371  * When used on a standard NFC Normalizer2 instance,
 372  * it returns the Decomposition_Mapping only if the Decomposition_Type (dt) is Canonical (Can);
 373  * in this case, the result contains either one or two code points (=1..4 UChars).
 374  *
 375  * This function is independent of the mode of the UNormalizer2.
 376  * @param norm2 UNormalizer2 instance
 377  * @param c code point
 378  * @param decomposition String buffer which will be set to c's
 379  *                      raw decomposition mapping, if there is one.
 380  * @param capacity number of UChars that can be written to decomposition
 381  * @param pErrorCode Standard ICU error code. Its input value must
 382  *                   pass the U_SUCCESS() test, or else the function returns
 383  *                   immediately. Check for U_FAILURE() on output or use with
 384  *                   function chaining. (See User Guide for details.)
 385  * @return the non-negative length of c's raw decomposition, if there is one; otherwise a negative value
 386  * @draft ICU 49
 387  */
 388 U_DRAFT int32_t U_EXPORT2
 389 unorm2_getRawDecomposition(const UNormalizer2 *norm2,
 390                            UChar32 c, UChar *decomposition, int32_t capacity,
 391                            UErrorCode *pErrorCode);
 392
 393 /**
 394  * Performs pairwise composition of a & b and returns the composite if there is one.
 395  *
 396  * Returns a composite code point c only if c has a two-way mapping to a+b.
 397  * In standard Unicode normalization, this means that
 398  * c has a canonical decomposition to a+b
 399  * and c does not have the Full_Composition_Exclusion property.
 400  *
 401  * This function is independent of the mode of the UNormalizer2.
 402  * @param norm2 UNormalizer2 instance
 403  * @param a A (normalization starter) code point.
 404  * @param b Another code point.
 405  * @return The non-negative composite code point if there is one; otherwise a negative value.
 406  * @draft ICU 49
 407  */
 408 U_DRAFT UChar32 U_EXPORT2
 409 unorm2_composePair(const UNormalizer2 *norm2, UChar32 a, UChar32 b);
 410
 411 /**
 412  * Gets the combining class of c.
 413  * The default implementation returns 0
 414  * but all standard implementations return the Unicode Canonical_Combining_Class value.
 415  * @param norm2 UNormalizer2 instance
 416  * @param c code point
 417  * @return c's combining class
 418  * @draft ICU 49
 419  */
 420 U_DRAFT uint8_t U_EXPORT2
 421 unorm2_getCombiningClass(const UNormalizer2 *norm2, UChar32 c);
 422 #endif  /* U_HIDE_DRAFT_API */
 423
 424 /**
 425  * Tests if the string is normalized.
 426  * Internally, in cases where the quickCheck() method would return "maybe"
 427  * (which is only possible for the two COMPOSE modes) this method
 428  * resolves to "yes" or "no" to provide a definitive result,
 429  * at the cost of doing more work in those cases.
 430  * @param norm2 UNormalizer2 instance
 431  * @param s input string
 432  * @param length length of the string, or -1 if NUL-terminated
 433  * @param pErrorCode Standard ICU error code. Its input value must
 434  *                   pass the U_SUCCESS() test, or else the function returns
 435  *                   immediately. Check for U_FAILURE() on output or use with
 436  *                   function chaining. (See User Guide for details.)
 437  * @return TRUE if s is normalized
 438  * @stable ICU 4.4
 439  */
 440 U_STABLE UBool U_EXPORT2
 441 unorm2_isNormalized(const UNormalizer2 *norm2,
 442                     const UChar *s, int32_t length,
 443                     UErrorCode *pErrorCode);
 444
 445 /**
 446  * Tests if the string is normalized.
 447  * For the two COMPOSE modes, the result could be "maybe" in cases that
 448  * would take a little more work to resolve definitively.
 449  * Use spanQuickCheckYes() and normalizeSecondAndAppend() for a faster
 450  * combination of quick check + normalization, to avoid
 451  * re-checking the "yes" prefix.
 452  * @param norm2 UNormalizer2 instance
 453  * @param s input string
 454  * @param length length of the string, or -1 if NUL-terminated
 455  * @param pErrorCode Standard ICU error code. Its input value must
 456  *                   pass the U_SUCCESS() test, or else the function returns
 457  *                   immediately. Check for U_FAILURE() on output or use with
 458  *                   function chaining. (See User Guide for details.)
 459  * @return UNormalizationCheckResult
 460  * @stable ICU 4.4
 461  */
 462 U_STABLE UNormalizationCheckResult U_EXPORT2
 463 unorm2_quickCheck(const UNormalizer2 *norm2,
 464                   const UChar *s, int32_t length,
 465                   UErrorCode *pErrorCode);
 466
 467 /**
 468  * Returns the end of the normalized substring of the input string.
 469  * In other words, with <code>end=spanQuickCheckYes(s, ec);</code>
 470  * the substring <code>UnicodeString(s, 0, end)</code>
 471  * will pass the quick check with a "yes" result.
 472  *
 473  * The returned end index is usually one or more characters before the
 474  * "no" or "maybe" character: The end index is at a normalization boundary.
 475  * (See the class documentation for more about normalization boundaries.)
 476  *
 477  * When the goal is a normalized string and most input strings are expected
 478  * to be normalized already, then call this method,
 479  * and if it returns a prefix shorter than the input string,
 480  * copy that prefix and use normalizeSecondAndAppend() for the remainder.
 481  * @param norm2 UNormalizer2 instance
 482  * @param s input string
 483  * @param length length of the string, or -1 if NUL-terminated
 484  * @param pErrorCode Standard ICU error code. Its input value must
 485  *                   pass the U_SUCCESS() test, or else the function returns
 486  *                   immediately. Check for U_FAILURE() on output or use with
 487  *                   function chaining. (See User Guide for details.)
 488  * @return "yes" span end index
 489  * @stable ICU 4.4
 490  */
 491 U_STABLE int32_t U_EXPORT2
 492 unorm2_spanQuickCheckYes(const UNormalizer2 *norm2,
 493                          const UChar *s, int32_t length,
 494                          UErrorCode *pErrorCode);
 495
 496 /**
 497  * Tests if the character always has a normalization boundary before it,
 498  * regardless of context.
 499  * For details see the Normalizer2 base class documentation.
 500  * @param norm2 UNormalizer2 instance
 501  * @param c character to test
 502  * @return TRUE if c has a normalization boundary before it
 503  * @stable ICU 4.4
 504  */
 505 U_STABLE UBool U_EXPORT2
 506 unorm2_hasBoundaryBefore(const UNormalizer2 *norm2, UChar32 c);
 507
 508 /**
 509  * Tests if the character always has a normalization boundary after it,
 510  * regardless of context.
 511  * For details see the Normalizer2 base class documentation.
 512  * @param norm2 UNormalizer2 instance
 513  * @param c character to test
 514  * @return TRUE if c has a normalization boundary after it
 515  * @stable ICU 4.4
 516  */
 517 U_STABLE UBool U_EXPORT2
 518 unorm2_hasBoundaryAfter(const UNormalizer2 *norm2, UChar32 c);
 519
 520 /**
 521  * Tests if the character is normalization-inert.
 522  * For details see the Normalizer2 base class documentation.
 523  * @param norm2 UNormalizer2 instance
 524  * @param c character to test
 525  * @return TRUE if c is normalization-inert
 526  * @stable ICU 4.4
 527  */
 528 U_STABLE UBool U_EXPORT2
 529 unorm2_isInert(const UNormalizer2 *norm2, UChar32 c);
 530
 531 #endif  /* !UCONFIG_NO_NORMALIZATION */
 532 #endif  /* __UNORM2_H__ */