2 *******************************************************************************
4 * Copyright (C) 2009-2015, International Business Machines
5 * Corporation and others. All Rights Reserved.
7 *******************************************************************************
10 * tab size: 8 (not used)
13 * created on: 2009dec15
14 * created by: Markus W. Scherer
22 * \brief C API: New API for Unicode Normalization.
24 * Unicode normalization functionality for standard Unicode normalization or
25 * for using custom mapping tables.
26 * All instances of UNormalizer2 are unmodifiable/immutable.
27 * Instances returned by unorm2_getInstance() are singletons that must not be deleted by the caller.
28 * For more details see the Normalizer2 C++ class.
31 #include "unicode/utypes.h"
32 #include "unicode/localpointer.h"
33 #include "unicode/uset.h"
36 * Constants for normalization modes.
37 * For details about standard Unicode normalization forms
38 * and about the algorithms which are also used with custom mapping tables
39 * see http://www.unicode.org/unicode/reports/tr15/
44 * Decomposition followed by composition.
45 * Same as standard NFC when using an "nfc" instance.
46 * Same as standard NFKC when using an "nfkc" instance.
47 * For details about standard Unicode normalization forms
48 * see http://www.unicode.org/unicode/reports/tr15/
53 * Map, and reorder canonically.
54 * Same as standard NFD when using an "nfc" instance.
55 * Same as standard NFKD when using an "nfkc" instance.
56 * For details about standard Unicode normalization forms
57 * see http://www.unicode.org/unicode/reports/tr15/
63 * If a string is in this form, then further decomposition <i>without reordering</i>
64 * would yield the same form as DECOMPOSE.
65 * Text in "Fast C or D" form can be processed efficiently with data tables
66 * that are "canonically closed", that is, that provide equivalent data for
67 * equivalent text, without having to be fully normalized.
68 * Not a standard Unicode normalization form.
69 * Not a unique form: Different FCD strings can be canonically equivalent.
70 * For details see http://www.unicode.org/notes/tn5/#FCD
75 * Compose only contiguously.
76 * Also known as "FCC" or "Fast C Contiguous".
77 * The result will often but not always be in NFC.
78 * The result will conform to FCD which is useful for processing.
79 * Not a standard Unicode normalization form.
80 * For details see http://www.unicode.org/notes/tn5/#FCC
83 UNORM2_COMPOSE_CONTIGUOUS
84 } UNormalization2Mode
;
87 * Result values for normalization quick check functions.
88 * For details see http://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
91 typedef enum UNormalizationCheckResult
{
93 * The input string is not in the normalization form.
98 * The input string is in the normalization form.
103 * The input string may or may not be in the normalization form.
104 * This value is only returned for composition forms like NFC and FCC,
105 * when a backward-combining character is found for which the surrounding text
106 * would have to be analyzed further.
110 } UNormalizationCheckResult
;
113 * Opaque C service object type for the new normalization API.
117 typedef struct UNormalizer2 UNormalizer2
; /**< C typedef for struct UNormalizer2. @stable ICU 4.4 */
119 #if !UCONFIG_NO_NORMALIZATION
122 * Returns a UNormalizer2 instance for Unicode NFC normalization.
123 * Same as unorm2_getInstance(NULL, "nfc", UNORM2_COMPOSE, pErrorCode).
124 * Returns an unmodifiable singleton instance. Do not delete it.
125 * @param pErrorCode Standard ICU error code. Its input value must
126 * pass the U_SUCCESS() test, or else the function returns
127 * immediately. Check for U_FAILURE() on output or use with
128 * function chaining. (See User Guide for details.)
129 * @return the requested Normalizer2, if successful
132 U_STABLE
const UNormalizer2
* U_EXPORT2
133 unorm2_getNFCInstance(UErrorCode
*pErrorCode
);
136 * Returns a UNormalizer2 instance for Unicode NFD normalization.
137 * Same as unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, pErrorCode).
138 * Returns an unmodifiable singleton instance. Do not delete it.
139 * @param pErrorCode Standard ICU error code. Its input value must
140 * pass the U_SUCCESS() test, or else the function returns
141 * immediately. Check for U_FAILURE() on output or use with
142 * function chaining. (See User Guide for details.)
143 * @return the requested Normalizer2, if successful
146 U_STABLE
const UNormalizer2
* U_EXPORT2
147 unorm2_getNFDInstance(UErrorCode
*pErrorCode
);
150 * Returns a UNormalizer2 instance for Unicode NFKC normalization.
151 * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_COMPOSE, pErrorCode).
152 * Returns an unmodifiable singleton instance. Do not delete it.
153 * @param pErrorCode Standard ICU error code. Its input value must
154 * pass the U_SUCCESS() test, or else the function returns
155 * immediately. Check for U_FAILURE() on output or use with
156 * function chaining. (See User Guide for details.)
157 * @return the requested Normalizer2, if successful
160 U_STABLE
const UNormalizer2
* U_EXPORT2
161 unorm2_getNFKCInstance(UErrorCode
*pErrorCode
);
164 * Returns a UNormalizer2 instance for Unicode NFKD normalization.
165 * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, pErrorCode).
166 * Returns an unmodifiable singleton instance. Do not delete it.
167 * @param pErrorCode Standard ICU error code. Its input value must
168 * pass the U_SUCCESS() test, or else the function returns
169 * immediately. Check for U_FAILURE() on output or use with
170 * function chaining. (See User Guide for details.)
171 * @return the requested Normalizer2, if successful
174 U_STABLE
const UNormalizer2
* U_EXPORT2
175 unorm2_getNFKDInstance(UErrorCode
*pErrorCode
);
178 * Returns a UNormalizer2 instance for Unicode NFKC_Casefold normalization.
179 * Same as unorm2_getInstance(NULL, "nfkc_cf", UNORM2_COMPOSE, pErrorCode).
180 * Returns an unmodifiable singleton instance. Do not delete it.
181 * @param pErrorCode Standard ICU error code. Its input value must
182 * pass the U_SUCCESS() test, or else the function returns
183 * immediately. Check for U_FAILURE() on output or use with
184 * function chaining. (See User Guide for details.)
185 * @return the requested Normalizer2, if successful
188 U_STABLE
const UNormalizer2
* U_EXPORT2
189 unorm2_getNFKCCasefoldInstance(UErrorCode
*pErrorCode
);
192 * Returns a UNormalizer2 instance which uses the specified data file
193 * (packageName/name similar to ucnv_openPackage() and ures_open()/ResourceBundle)
194 * and which composes or decomposes text according to the specified mode.
195 * Returns an unmodifiable singleton instance. Do not delete it.
197 * Use packageName=NULL for data files that are part of ICU's own data.
198 * Use name="nfc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFC/NFD.
199 * Use name="nfkc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFKC/NFKD.
200 * Use name="nfkc_cf" and UNORM2_COMPOSE for Unicode standard NFKC_CF=NFKC_Casefold.
202 * @param packageName NULL for ICU built-in data, otherwise application data package name
203 * @param name "nfc" or "nfkc" or "nfkc_cf" or name of custom data file
204 * @param mode normalization mode (compose or decompose etc.)
205 * @param pErrorCode Standard ICU error code. Its input value must
206 * pass the U_SUCCESS() test, or else the function returns
207 * immediately. Check for U_FAILURE() on output or use with
208 * function chaining. (See User Guide for details.)
209 * @return the requested UNormalizer2, if successful
212 U_STABLE
const UNormalizer2
* U_EXPORT2
213 unorm2_getInstance(const char *packageName
,
215 UNormalization2Mode mode
,
216 UErrorCode
*pErrorCode
);
219 * Constructs a filtered normalizer wrapping any UNormalizer2 instance
221 * Both are aliased and must not be modified or deleted while this object
223 * The filter set should be frozen; otherwise the performance will suffer greatly.
224 * @param norm2 wrapped UNormalizer2 instance
225 * @param filterSet USet which determines the characters to be normalized
226 * @param pErrorCode Standard ICU error code. Its input value must
227 * pass the U_SUCCESS() test, or else the function returns
228 * immediately. Check for U_FAILURE() on output or use with
229 * function chaining. (See User Guide for details.)
230 * @return the requested UNormalizer2, if successful
233 U_STABLE UNormalizer2
* U_EXPORT2
234 unorm2_openFiltered(const UNormalizer2
*norm2
, const USet
*filterSet
, UErrorCode
*pErrorCode
);
237 * Closes a UNormalizer2 instance from unorm2_openFiltered().
238 * Do not close instances from unorm2_getInstance()!
239 * @param norm2 UNormalizer2 instance to be closed
242 U_STABLE
void U_EXPORT2
243 unorm2_close(UNormalizer2
*norm2
);
245 #if U_SHOW_CPLUSPLUS_API
250 * \class LocalUNormalizer2Pointer
251 * "Smart pointer" class, closes a UNormalizer2 via unorm2_close().
252 * For most methods see the LocalPointerBase base class.
254 * @see LocalPointerBase
258 U_DEFINE_LOCAL_OPEN_POINTER(LocalUNormalizer2Pointer
, UNormalizer2
, unorm2_close
);
265 * Writes the normalized form of the source string to the destination string
266 * (replacing its contents) and returns the length of the destination string.
267 * The source and destination strings must be different buffers.
268 * @param norm2 UNormalizer2 instance
269 * @param src source string
270 * @param length length of the source string, or -1 if NUL-terminated
271 * @param dest destination string; its contents is replaced with normalized src
272 * @param capacity number of UChars that can be written to dest
273 * @param pErrorCode Standard ICU error code. Its input value must
274 * pass the U_SUCCESS() test, or else the function returns
275 * immediately. Check for U_FAILURE() on output or use with
276 * function chaining. (See User Guide for details.)
280 U_STABLE
int32_t U_EXPORT2
281 unorm2_normalize(const UNormalizer2
*norm2
,
282 const UChar
*src
, int32_t length
,
283 UChar
*dest
, int32_t capacity
,
284 UErrorCode
*pErrorCode
);
286 * Appends the normalized form of the second string to the first string
287 * (merging them at the boundary) and returns the length of the first string.
288 * The result is normalized if the first string was normalized.
289 * The first and second strings must be different buffers.
290 * @param norm2 UNormalizer2 instance
291 * @param first string, should be normalized
292 * @param firstLength length of the first string, or -1 if NUL-terminated
293 * @param firstCapacity number of UChars that can be written to first
294 * @param second string, will be normalized
295 * @param secondLength length of the source string, or -1 if NUL-terminated
296 * @param pErrorCode Standard ICU error code. Its input value must
297 * pass the U_SUCCESS() test, or else the function returns
298 * immediately. Check for U_FAILURE() on output or use with
299 * function chaining. (See User Guide for details.)
303 U_STABLE
int32_t U_EXPORT2
304 unorm2_normalizeSecondAndAppend(const UNormalizer2
*norm2
,
305 UChar
*first
, int32_t firstLength
, int32_t firstCapacity
,
306 const UChar
*second
, int32_t secondLength
,
307 UErrorCode
*pErrorCode
);
309 * Appends the second string to the first string
310 * (merging them at the boundary) and returns the length of the first string.
311 * The result is normalized if both the strings were normalized.
312 * The first and second strings must be different buffers.
313 * @param norm2 UNormalizer2 instance
314 * @param first string, should be normalized
315 * @param firstLength length of the first string, or -1 if NUL-terminated
316 * @param firstCapacity number of UChars that can be written to first
317 * @param second string, should be normalized
318 * @param secondLength length of the source string, or -1 if NUL-terminated
319 * @param pErrorCode Standard ICU error code. Its input value must
320 * pass the U_SUCCESS() test, or else the function returns
321 * immediately. Check for U_FAILURE() on output or use with
322 * function chaining. (See User Guide for details.)
326 U_STABLE
int32_t U_EXPORT2
327 unorm2_append(const UNormalizer2
*norm2
,
328 UChar
*first
, int32_t firstLength
, int32_t firstCapacity
,
329 const UChar
*second
, int32_t secondLength
,
330 UErrorCode
*pErrorCode
);
333 * Gets the decomposition mapping of c.
334 * Roughly equivalent to normalizing the String form of c
335 * on a UNORM2_DECOMPOSE UNormalizer2 instance, but much faster, and except that this function
336 * returns a negative value and does not write a string
337 * if c does not have a decomposition mapping in this instance's data.
338 * This function is independent of the mode of the UNormalizer2.
339 * @param norm2 UNormalizer2 instance
340 * @param c code point
341 * @param decomposition String buffer which will be set to c's
342 * decomposition mapping, if there is one.
343 * @param capacity number of UChars that can be written to decomposition
344 * @param pErrorCode Standard ICU error code. Its input value must
345 * pass the U_SUCCESS() test, or else the function returns
346 * immediately. Check for U_FAILURE() on output or use with
347 * function chaining. (See User Guide for details.)
348 * @return the non-negative length of c's decomposition, if there is one; otherwise a negative value
351 U_STABLE
int32_t U_EXPORT2
352 unorm2_getDecomposition(const UNormalizer2
*norm2
,
353 UChar32 c
, UChar
*decomposition
, int32_t capacity
,
354 UErrorCode
*pErrorCode
);
357 * Gets the raw decomposition mapping of c.
359 * This is similar to the unorm2_getDecomposition() function but returns the
360 * raw decomposition mapping as specified in UnicodeData.txt or
361 * (for custom data) in the mapping files processed by the gennorm2 tool.
362 * By contrast, unorm2_getDecomposition() returns the processed,
363 * recursively-decomposed version of this mapping.
365 * When used on a standard NFKC Normalizer2 instance,
366 * unorm2_getRawDecomposition() returns the Unicode Decomposition_Mapping (dm) property.
368 * When used on a standard NFC Normalizer2 instance,
369 * it returns the Decomposition_Mapping only if the Decomposition_Type (dt) is Canonical (Can);
370 * in this case, the result contains either one or two code points (=1..4 UChars).
372 * This function is independent of the mode of the UNormalizer2.
373 * @param norm2 UNormalizer2 instance
374 * @param c code point
375 * @param decomposition String buffer which will be set to c's
376 * raw decomposition mapping, if there is one.
377 * @param capacity number of UChars that can be written to decomposition
378 * @param pErrorCode Standard ICU error code. Its input value must
379 * pass the U_SUCCESS() test, or else the function returns
380 * immediately. Check for U_FAILURE() on output or use with
381 * function chaining. (See User Guide for details.)
382 * @return the non-negative length of c's raw decomposition, if there is one; otherwise a negative value
385 U_STABLE
int32_t U_EXPORT2
386 unorm2_getRawDecomposition(const UNormalizer2
*norm2
,
387 UChar32 c
, UChar
*decomposition
, int32_t capacity
,
388 UErrorCode
*pErrorCode
);
391 * Performs pairwise composition of a & b and returns the composite if there is one.
393 * Returns a composite code point c only if c has a two-way mapping to a+b.
394 * In standard Unicode normalization, this means that
395 * c has a canonical decomposition to a+b
396 * and c does not have the Full_Composition_Exclusion property.
398 * This function is independent of the mode of the UNormalizer2.
399 * @param norm2 UNormalizer2 instance
400 * @param a A (normalization starter) code point.
401 * @param b Another code point.
402 * @return The non-negative composite code point if there is one; otherwise a negative value.
405 U_STABLE UChar32 U_EXPORT2
406 unorm2_composePair(const UNormalizer2
*norm2
, UChar32 a
, UChar32 b
);
409 * Gets the combining class of c.
410 * The default implementation returns 0
411 * but all standard implementations return the Unicode Canonical_Combining_Class value.
412 * @param norm2 UNormalizer2 instance
413 * @param c code point
414 * @return c's combining class
417 U_STABLE
uint8_t U_EXPORT2
418 unorm2_getCombiningClass(const UNormalizer2
*norm2
, UChar32 c
);
421 * Tests if the string is normalized.
422 * Internally, in cases where the quickCheck() method would return "maybe"
423 * (which is only possible for the two COMPOSE modes) this method
424 * resolves to "yes" or "no" to provide a definitive result,
425 * at the cost of doing more work in those cases.
426 * @param norm2 UNormalizer2 instance
427 * @param s input string
428 * @param length length of the string, or -1 if NUL-terminated
429 * @param pErrorCode Standard ICU error code. Its input value must
430 * pass the U_SUCCESS() test, or else the function returns
431 * immediately. Check for U_FAILURE() on output or use with
432 * function chaining. (See User Guide for details.)
433 * @return TRUE if s is normalized
436 U_STABLE UBool U_EXPORT2
437 unorm2_isNormalized(const UNormalizer2
*norm2
,
438 const UChar
*s
, int32_t length
,
439 UErrorCode
*pErrorCode
);
442 * Tests if the string is normalized.
443 * For the two COMPOSE modes, the result could be "maybe" in cases that
444 * would take a little more work to resolve definitively.
445 * Use spanQuickCheckYes() and normalizeSecondAndAppend() for a faster
446 * combination of quick check + normalization, to avoid
447 * re-checking the "yes" prefix.
448 * @param norm2 UNormalizer2 instance
449 * @param s input string
450 * @param length length of the string, or -1 if NUL-terminated
451 * @param pErrorCode Standard ICU error code. Its input value must
452 * pass the U_SUCCESS() test, or else the function returns
453 * immediately. Check for U_FAILURE() on output or use with
454 * function chaining. (See User Guide for details.)
455 * @return UNormalizationCheckResult
458 U_STABLE UNormalizationCheckResult U_EXPORT2
459 unorm2_quickCheck(const UNormalizer2
*norm2
,
460 const UChar
*s
, int32_t length
,
461 UErrorCode
*pErrorCode
);
464 * Returns the end of the normalized substring of the input string.
465 * In other words, with <code>end=spanQuickCheckYes(s, ec);</code>
466 * the substring <code>UnicodeString(s, 0, end)</code>
467 * will pass the quick check with a "yes" result.
469 * The returned end index is usually one or more characters before the
470 * "no" or "maybe" character: The end index is at a normalization boundary.
471 * (See the class documentation for more about normalization boundaries.)
473 * When the goal is a normalized string and most input strings are expected
474 * to be normalized already, then call this method,
475 * and if it returns a prefix shorter than the input string,
476 * copy that prefix and use normalizeSecondAndAppend() for the remainder.
477 * @param norm2 UNormalizer2 instance
478 * @param s input string
479 * @param length length of the string, or -1 if NUL-terminated
480 * @param pErrorCode Standard ICU error code. Its input value must
481 * pass the U_SUCCESS() test, or else the function returns
482 * immediately. Check for U_FAILURE() on output or use with
483 * function chaining. (See User Guide for details.)
484 * @return "yes" span end index
487 U_STABLE
int32_t U_EXPORT2
488 unorm2_spanQuickCheckYes(const UNormalizer2
*norm2
,
489 const UChar
*s
, int32_t length
,
490 UErrorCode
*pErrorCode
);
493 * Tests if the character always has a normalization boundary before it,
494 * regardless of context.
495 * For details see the Normalizer2 base class documentation.
496 * @param norm2 UNormalizer2 instance
497 * @param c character to test
498 * @return TRUE if c has a normalization boundary before it
501 U_STABLE UBool U_EXPORT2
502 unorm2_hasBoundaryBefore(const UNormalizer2
*norm2
, UChar32 c
);
505 * Tests if the character always has a normalization boundary after it,
506 * regardless of context.
507 * For details see the Normalizer2 base class documentation.
508 * @param norm2 UNormalizer2 instance
509 * @param c character to test
510 * @return TRUE if c has a normalization boundary after it
513 U_STABLE UBool U_EXPORT2
514 unorm2_hasBoundaryAfter(const UNormalizer2
*norm2
, UChar32 c
);
517 * Tests if the character is normalization-inert.
518 * For details see the Normalizer2 base class documentation.
519 * @param norm2 UNormalizer2 instance
520 * @param c character to test
521 * @return TRUE if c is normalization-inert
524 U_STABLE UBool U_EXPORT2
525 unorm2_isInert(const UNormalizer2
*norm2
, UChar32 c
);
528 * Option bit for unorm_compare:
529 * Both input strings are assumed to fulfill FCD conditions.
532 #define UNORM_INPUT_IS_FCD 0x20000
535 * Option bit for unorm_compare:
536 * Perform case-insensitive comparison.
539 #define U_COMPARE_IGNORE_CASE 0x10000
541 #ifndef U_COMPARE_CODE_POINT_ORDER
542 /* see also unistr.h and ustring.h */
544 * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc:
545 * Compare strings in code point order instead of code unit order.
548 #define U_COMPARE_CODE_POINT_ORDER 0x8000
552 * Compares two strings for canonical equivalence.
553 * Further options include case-insensitive comparison and
554 * code point order (as opposed to code unit order).
556 * Canonical equivalence between two strings is defined as their normalized
557 * forms (NFD or NFC) being identical.
558 * This function compares strings incrementally instead of normalizing
559 * (and optionally case-folding) both strings entirely,
560 * improving performance significantly.
562 * Bulk normalization is only necessary if the strings do not fulfill the FCD
563 * conditions. Only in this case, and only if the strings are relatively long,
564 * is memory allocated temporarily.
565 * For FCD strings and short non-FCD strings there is no memory allocation.
567 * Semantically, this is equivalent to
568 * strcmp[CodePointOrder](NFD(foldCase(NFD(s1))), NFD(foldCase(NFD(s2))))
569 * where code point order and foldCase are all optional.
571 * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match
572 * the case folding must be performed first, then the normalization.
574 * @param s1 First source string.
575 * @param length1 Length of first source string, or -1 if NUL-terminated.
577 * @param s2 Second source string.
578 * @param length2 Length of second source string, or -1 if NUL-terminated.
580 * @param options A bit set of options:
581 * - U_FOLD_CASE_DEFAULT or 0 is used for default options:
582 * Case-sensitive comparison in code unit order, and the input strings
583 * are quick-checked for FCD.
585 * - UNORM_INPUT_IS_FCD
586 * Set if the caller knows that both s1 and s2 fulfill the FCD conditions.
587 * If not set, the function will quickCheck for FCD
588 * and normalize if necessary.
590 * - U_COMPARE_CODE_POINT_ORDER
591 * Set to choose code point order instead of code unit order
592 * (see u_strCompare for details).
594 * - U_COMPARE_IGNORE_CASE
595 * Set to compare strings case-insensitively using case folding,
596 * instead of case-sensitively.
597 * If set, then the following case folding options are used.
599 * - Options as used with case-insensitive comparisons, currently:
601 * - U_FOLD_CASE_EXCLUDE_SPECIAL_I
602 * (see u_strCaseCompare for details)
604 * - regular normalization options shifted left by UNORM_COMPARE_NORM_OPTIONS_SHIFT
606 * @param pErrorCode ICU error code in/out parameter.
607 * Must fulfill U_SUCCESS before the function call.
608 * @return <0 or 0 or >0 as usual for string comparisons
610 * @see unorm_normalize
613 * @see u_strCaseCompare
617 U_STABLE
int32_t U_EXPORT2
618 unorm_compare(const UChar
*s1
, int32_t length1
,
619 const UChar
*s2
, int32_t length2
,
621 UErrorCode
*pErrorCode
);
623 #endif /* !UCONFIG_NO_NORMALIZATION */
624 #endif /* __UNORM2_H__ */