1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
4 ******************************************************************************
5 * Copyright (C) 1996-2015, International Business Machines Corporation and others.
7 ******************************************************************************
13 #include "unicode/utypes.h"
14 #include "unicode/uloc.h"
15 #include "unicode/utext.h"
16 #include "unicode/localpointer.h"
19 * A text-break iterator.
20 * For usage in C programs.
22 #ifndef UBRK_TYPEDEF_UBREAK_ITERATOR
23 # define UBRK_TYPEDEF_UBREAK_ITERATOR
25 * Opaque type representing an ICU Break iterator object.
28 typedef struct UBreakIterator UBreakIterator
;
31 #if !UCONFIG_NO_BREAK_ITERATION
33 #include "unicode/parseerr.h"
37 * \brief C API: BreakIterator
39 * <h2> BreakIterator C API </h2>
41 * The BreakIterator C API defines methods for finding the location
42 * of boundaries in text. Pointer to a UBreakIterator maintain a
43 * current position and scan over text returning the index of characters
44 * where boundaries occur.
46 * Line boundary analysis determines where a text string can be broken
47 * when line-wrapping. The mechanism correctly handles punctuation and
50 * Note: The locale keyword "lb" can be used to modify line break
51 * behavior according to the CSS level 3 line-break options, see
52 * <http://dev.w3.org/csswg/css-text/#line-breaking>. For example:
53 * "ja@lb=strict", "zh@lb=loose".
55 * Sentence boundary analysis allows selection with correct
56 * interpretation of periods within numbers and abbreviations, and
57 * trailing punctuation marks such as quotation marks and parentheses.
59 * Note: The locale keyword "ss" can be used to enable use of
60 * segmentation suppression data (preventing breaks in English after
61 * abbreviations such as "Mr." or "Est.", for example), as follows:
64 * Word boundary analysis is used by search and replace functions, as
65 * well as within text editing applications that allow the user to
66 * select words with a double click. Word selection provides correct
67 * interpretation of punctuation marks within and following
68 * words. Characters that are not part of a word, such as symbols or
69 * punctuation marks, have word-breaks on both sides.
71 * Character boundary analysis identifies the boundaries of
72 * "Extended Grapheme Clusters", which are groupings of codepoints
73 * that should be treated as character-like units for many text operations.
74 * Please see Unicode Standard Annex #29, Unicode Text Segmentation,
75 * http://www.unicode.org/reports/tr29/ for additional information
76 * on grapheme clusters and guidelines on their use.
78 * Title boundary analysis locates all positions,
79 * typically starts of words, that should be set to Title Case
80 * when title casing the text.
82 * The text boundary positions are found according to the rules
83 * described in Unicode Standard Annex #29, Text Boundaries, and
84 * Unicode Standard Annex #14, Line Breaking Properties. These
85 * are available at http://www.unicode.org/reports/tr14/ and
86 * http://www.unicode.org/reports/tr29/.
88 * In addition to the plain C API defined in this header file, an
89 * object oriented C++ API with equivalent functionality is defined in the
92 * Code snippets illustrating the use of the Break Iterator APIs
93 * are available in the ICU User Guide,
94 * http://icu-project.org/userguide/boundaryAnalysis.html
95 * and in the sample program icu/source/samples/break/break.cpp
98 /** The possible types of text boundaries. @stable ICU 2.0 */
99 typedef enum UBreakIteratorType
{
100 /** Character breaks @stable ICU 2.0 */
102 /** Word breaks @stable ICU 2.0 */
104 /** Line breaks @stable ICU 2.0 */
106 /** Sentence breaks @stable ICU 2.0 */
109 #ifndef U_HIDE_DEPRECATED_API
112 * The iterator created using this type locates title boundaries as described for
113 * Unicode 3.2 only. For Unicode 4.0 and above title boundary iteration,
114 * please use Word Boundary iterator.
116 * @deprecated ICU 2.8 Use the word break iterator for titlecasing for Unicode 4 and later.
120 * One more than the highest normal UBreakIteratorType value.
121 * @deprecated ICU 58 The numeric value may change over time, see ICU ticket #12420.
124 #endif // U_HIDE_DEPRECATED_API
125 } UBreakIteratorType
;
127 /** Value indicating all text boundaries have been returned.
130 #define UBRK_DONE ((int32_t) -1)
134 * Enum constants for the word break tags returned by
135 * getRuleStatus(). A range of values is defined for each category of
136 * word, to allow for further subdivisions of a category in future releases.
137 * Applications should check for tag values falling within the range, rather
138 * than for single individual values.
140 * The numeric values of all of these constants are stable (will not change).
144 typedef enum UWordBreak
{
145 /** Tag value for "words" that do not fit into any of other categories.
146 * Includes spaces and most punctuation. */
148 /** Upper bound for tags for uncategorized words. */
149 UBRK_WORD_NONE_LIMIT
= 100,
150 /** Tag value for words that appear to be numbers, lower limit. */
151 UBRK_WORD_NUMBER
= 100,
152 /** Tag value for words that appear to be numbers, upper limit. */
153 UBRK_WORD_NUMBER_LIMIT
= 200,
154 /** Tag value for words that contain letters, excluding
155 * hiragana, katakana or ideographic characters, lower limit. */
156 UBRK_WORD_LETTER
= 200,
157 /** Tag value for words containing letters, upper limit */
158 UBRK_WORD_LETTER_LIMIT
= 300,
159 /** Tag value for words containing kana characters, lower limit */
160 UBRK_WORD_KANA
= 300,
161 /** Tag value for words containing kana characters, upper limit */
162 UBRK_WORD_KANA_LIMIT
= 400,
163 /** Tag value for words containing ideographic characters, lower limit */
164 UBRK_WORD_IDEO
= 400,
165 /** Tag value for words containing ideographic characters, upper limit */
166 UBRK_WORD_IDEO_LIMIT
= 500
170 * Enum constants for the line break tags returned by getRuleStatus().
171 * A range of values is defined for each category of
172 * word, to allow for further subdivisions of a category in future releases.
173 * Applications should check for tag values falling within the range, rather
174 * than for single individual values.
176 * The numeric values of all of these constants are stable (will not change).
180 typedef enum ULineBreakTag
{
181 /** Tag value for soft line breaks, positions at which a line break
182 * is acceptable but not required */
184 /** Upper bound for soft line breaks. */
185 UBRK_LINE_SOFT_LIMIT
= 100,
186 /** Tag value for a hard, or mandatory line break */
187 UBRK_LINE_HARD
= 100,
188 /** Upper bound for hard line breaks. */
189 UBRK_LINE_HARD_LIMIT
= 200
195 * Enum constants for the sentence break tags returned by getRuleStatus().
196 * A range of values is defined for each category of
197 * sentence, to allow for further subdivisions of a category in future releases.
198 * Applications should check for tag values falling within the range, rather
199 * than for single individual values.
201 * The numeric values of all of these constants are stable (will not change).
205 typedef enum USentenceBreakTag
{
206 /** Tag value for for sentences ending with a sentence terminator
207 * ('.', '?', '!', etc.) character, possibly followed by a
208 * hard separator (CR, LF, PS, etc.)
210 UBRK_SENTENCE_TERM
= 0,
211 /** Upper bound for tags for sentences ended by sentence terminators. */
212 UBRK_SENTENCE_TERM_LIMIT
= 100,
213 /** Tag value for for sentences that do not contain an ending
214 * sentence terminator ('.', '?', '!', etc.) character, but
215 * are ended only by a hard separator (CR, LF, PS, etc.) or end of input.
217 UBRK_SENTENCE_SEP
= 100,
218 /** Upper bound for tags for sentences ended by a separator. */
219 UBRK_SENTENCE_SEP_LIMIT
= 200
220 /** Tag value for a hard, or mandatory line break */
225 * Masks to control line break word options (per the CSS word-break property).
226 * NORMAL allows breaks between CJK characters in the middle of words. Other masks
227 * prohibit breaks between characters of specific scripts (or all scripts) except as
228 * determined by a dictionary, or by spaces or other mechanisms (Western-style breaking).
230 * @internal Apple only
232 typedef enum ULineWordOptions
{
233 /** Allow breaks between characters of all CJK scripts */
234 UBRK_LINEWORD_NORMAL
= 0,
235 /** Prevent breaks between Hangul characters, except as determined by a dictionary. */
236 UBRK_LINEWORD_KEEP_HANGUL
= 1,
237 /** Prevent breaks between characters of any script, except as determined by a dictionary. */
238 UBRK_LINEWORD_KEEP_ALL
= 0x7F
244 * Open a new UBreakIterator for locating text boundaries for a specified locale.
245 * A UBreakIterator may be used for detecting character, line, word,
246 * and sentence breaks in text.
247 * @param type The type of UBreakIterator to open: one of UBRK_CHARACTER, UBRK_WORD,
248 * UBRK_LINE, UBRK_SENTENCE
249 * @param locale The locale specifying the text-breaking conventions. Note that
250 * locale keys such as "lb" and "ss" may be used to modify text break behavior,
251 * see general discussion of BreakIterator C API.
252 * @param text The text to be iterated over. May be null, in which case ubrk_setText() is
253 * used to specify the text to be iterated.
254 * @param textLength The number of characters in text, or -1 if null-terminated.
255 * @param status A UErrorCode to receive any errors.
256 * @return A UBreakIterator for the specified locale.
257 * @see ubrk_openRules
260 U_STABLE UBreakIterator
* U_EXPORT2
261 ubrk_open(UBreakIteratorType type
,
268 * Open a new UBreakIterator for locating text boundaries using specified breaking rules.
269 * The rule syntax is ... (TBD)
270 * @param rules A set of rules specifying the text breaking conventions.
271 * @param rulesLength The number of characters in rules, or -1 if null-terminated.
272 * @param text The text to be iterated over. May be null, in which case ubrk_setText() is
273 * used to specify the text to be iterated.
274 * @param textLength The number of characters in text, or -1 if null-terminated.
275 * @param parseErr Receives position and context information for any syntax errors
276 * detected while parsing the rules.
277 * @param status A UErrorCode to receive any errors.
278 * @return A UBreakIterator for the specified rules.
282 U_STABLE UBreakIterator
* U_EXPORT2
283 ubrk_openRules(const UChar
*rules
,
287 UParseError
*parseErr
,
291 * Open a new UBreakIterator for locating text boundaries using precompiled binary rules.
292 * Opening a UBreakIterator this way is substantially faster than using ubrk_openRules.
293 * Binary rules may be obtained using ubrk_getBinaryRules. The compiled rules are not
294 * compatible across different major versions of ICU, nor across platforms of different
295 * endianness or different base character set family (ASCII vs EBCDIC).
296 * @param binaryRules A set of compiled binary rules specifying the text breaking
297 * conventions. Ownership of the storage containing the compiled
298 * rules remains with the caller of this function. The compiled
299 * rules must not be modified or deleted during the life of the
301 * @param rulesLength The length of binaryRules in bytes; must be >= 0.
302 * @param text The text to be iterated over. May be null, in which case
303 * ubrk_setText() is used to specify the text to be iterated.
304 * @param textLength The number of characters in text, or -1 if null-terminated.
305 * @param status Pointer to UErrorCode to receive any errors.
306 * @return UBreakIterator for the specified rules.
307 * @see ubrk_getBinaryRules
310 U_STABLE UBreakIterator
* U_EXPORT2
311 ubrk_openBinaryRules(const uint8_t *binaryRules
, int32_t rulesLength
,
312 const UChar
* text
, int32_t textLength
,
313 UErrorCode
* status
);
316 * Thread safe cloning operation
317 * @param bi iterator to be cloned
318 * @param stackBuffer <em>Deprecated functionality as of ICU 52, use NULL.</em><br>
319 * user allocated space for the new clone. If NULL new memory will be allocated.
320 * If buffer is not large enough, new memory will be allocated.
321 * Clients can use the U_BRK_SAFECLONE_BUFFERSIZE.
322 * @param pBufferSize <em>Deprecated functionality as of ICU 52, use NULL or 1.</em><br>
323 * pointer to size of allocated space.
324 * If *pBufferSize == 0, a sufficient size for use in cloning will
325 * be returned ('pre-flighting')
326 * If *pBufferSize is not enough for a stack-based safe clone,
327 * new memory will be allocated.
328 * @param status to indicate whether the operation went on smoothly or there were errors
329 * An informational status value, U_SAFECLONE_ALLOCATED_ERROR, is used if any allocations were necessary.
330 * @return pointer to the new clone
333 U_STABLE UBreakIterator
* U_EXPORT2
335 const UBreakIterator
*bi
,
337 int32_t *pBufferSize
,
340 #ifndef U_HIDE_DEPRECATED_API
343 * A recommended size (in bytes) for the memory buffer to be passed to ubrk_saveClone().
344 * @deprecated ICU 52. Do not rely on ubrk_safeClone() cloning into any provided buffer.
346 #define U_BRK_SAFECLONE_BUFFERSIZE 1
348 #endif /* U_HIDE_DEPRECATED_API */
351 * Close a UBreakIterator.
352 * Once closed, a UBreakIterator may no longer be used.
353 * @param bi The break iterator to close.
356 U_STABLE
void U_EXPORT2
357 ubrk_close(UBreakIterator
*bi
);
359 #if U_SHOW_CPLUSPLUS_API
364 * \class LocalUBreakIteratorPointer
365 * "Smart pointer" class, closes a UBreakIterator via ubrk_close().
366 * For most methods see the LocalPointerBase base class.
368 * @see LocalPointerBase
372 U_DEFINE_LOCAL_OPEN_POINTER(LocalUBreakIteratorPointer
, UBreakIterator
, ubrk_close
);
376 #endif // U_SHOW_CPLUSPLUS_API
378 #ifndef U_HIDE_INTERNAL_API
380 * Set the ULineWordOptions for the specified break iterator.
382 * @param bi The iterator to use
383 * @param lineWordOpts The ULineWordOptions to set.
384 * @internal Apple only
386 U_INTERNAL
void U_EXPORT2
387 ubrk_setLineWordOpts(UBreakIterator
* bi
,
388 ULineWordOptions lineWordOpts
);
390 #endif /* U_HIDE_INTERNAL_API */
393 * Sets an existing iterator to point to a new piece of text.
394 * The break iterator retains a pointer to the supplied text.
395 * The caller must not modify or delete the text while the BreakIterator
396 * retains the reference.
398 * @param bi The iterator to use
399 * @param text The text to be set
400 * @param textLength The length of the text
401 * @param status The error code
404 U_STABLE
void U_EXPORT2
405 ubrk_setText(UBreakIterator
* bi
,
412 * Sets an existing iterator to point to a new piece of text.
414 * All index positions returned by break iterator functions are
415 * native indices from the UText. For example, when breaking UTF-8
416 * encoded text, the break positions returned by \ref ubrk_next, \ref ubrk_previous, etc.
417 * will be UTF-8 string indices, not UTF-16 positions.
419 * @param bi The iterator to use
420 * @param text The text to be set.
421 * This function makes a shallow clone of the supplied UText. This means
422 * that the caller is free to immediately close or otherwise reuse the
423 * UText that was passed as a parameter, but that the underlying text itself
424 * must not be altered while being referenced by the break iterator.
425 * @param status The error code
428 U_STABLE
void U_EXPORT2
429 ubrk_setUText(UBreakIterator
* bi
,
436 * Determine the most recently-returned text boundary.
438 * @param bi The break iterator to use.
439 * @return The character index most recently returned by \ref ubrk_next, \ref ubrk_previous,
440 * \ref ubrk_first, or \ref ubrk_last.
443 U_STABLE
int32_t U_EXPORT2
444 ubrk_current(const UBreakIterator
*bi
);
447 * Advance the iterator to the boundary following the current boundary.
449 * @param bi The break iterator to use.
450 * @return The character index of the next text boundary, or UBRK_DONE
451 * if all text boundaries have been returned.
455 U_STABLE
int32_t U_EXPORT2
456 ubrk_next(UBreakIterator
*bi
);
459 * Set the iterator position to the boundary preceding the current boundary.
461 * @param bi The break iterator to use.
462 * @return The character index of the preceding text boundary, or UBRK_DONE
463 * if all text boundaries have been returned.
467 U_STABLE
int32_t U_EXPORT2
468 ubrk_previous(UBreakIterator
*bi
);
471 * Set the iterator position to zero, the start of the text being scanned.
472 * @param bi The break iterator to use.
473 * @return The new iterator position (zero).
477 U_STABLE
int32_t U_EXPORT2
478 ubrk_first(UBreakIterator
*bi
);
481 * Set the iterator position to the index immediately <EM>beyond</EM> the last character in the text being scanned.
482 * This is not the same as the last character.
483 * @param bi The break iterator to use.
484 * @return The character offset immediately <EM>beyond</EM> the last character in the
485 * text being scanned.
489 U_STABLE
int32_t U_EXPORT2
490 ubrk_last(UBreakIterator
*bi
);
493 * Set the iterator position to the first boundary preceding the specified offset.
494 * The new position is always smaller than offset, or UBRK_DONE.
495 * @param bi The break iterator to use.
496 * @param offset The offset to begin scanning.
497 * @return The text boundary preceding offset, or UBRK_DONE.
498 * @see ubrk_following
501 U_STABLE
int32_t U_EXPORT2
502 ubrk_preceding(UBreakIterator
*bi
,
506 * Advance the iterator to the first boundary following the specified offset.
507 * The value returned is always greater than offset, or UBRK_DONE.
508 * @param bi The break iterator to use.
509 * @param offset The offset to begin scanning.
510 * @return The text boundary following offset, or UBRK_DONE.
511 * @see ubrk_preceding
514 U_STABLE
int32_t U_EXPORT2
515 ubrk_following(UBreakIterator
*bi
,
519 * Get a locale for which text breaking information is available.
520 * A UBreakIterator in a locale returned by this function will perform the correct
521 * text breaking for the locale.
522 * @param index The index of the desired locale.
523 * @return A locale for which number text breaking information is available, or 0 if none.
524 * @see ubrk_countAvailable
527 U_STABLE
const char* U_EXPORT2
528 ubrk_getAvailable(int32_t index
);
531 * Determine how many locales have text breaking information available.
532 * This function is most useful as determining the loop ending condition for
533 * calls to \ref ubrk_getAvailable.
534 * @return The number of locales for which text breaking information is available.
535 * @see ubrk_getAvailable
538 U_STABLE
int32_t U_EXPORT2
539 ubrk_countAvailable(void);
543 * Returns true if the specified position is a boundary position. As a side
544 * effect, leaves the iterator pointing to the first boundary position at
546 * @param bi The break iterator to use.
547 * @param offset the offset to check.
548 * @return True if "offset" is a boundary position.
551 U_STABLE UBool U_EXPORT2
552 ubrk_isBoundary(UBreakIterator
*bi
, int32_t offset
);
555 * Return the status from the break rule that determined the most recently
556 * returned break position. The values appear in the rule source
557 * within brackets, {123}, for example. For rules that do not specify a
558 * status, a default value of 0 is returned.
560 * For word break iterators, the possible values are defined in enum UWordBreak.
563 U_STABLE
int32_t U_EXPORT2
564 ubrk_getRuleStatus(UBreakIterator
*bi
);
567 * Get the statuses from the break rules that determined the most recently
568 * returned break position. The values appear in the rule source
569 * within brackets, {123}, for example. The default status value for rules
570 * that do not explicitly provide one is zero.
572 * For word break iterators, the possible values are defined in enum UWordBreak.
573 * @param bi The break iterator to use
574 * @param fillInVec an array to be filled in with the status values.
575 * @param capacity the length of the supplied vector. A length of zero causes
576 * the function to return the number of status values, in the
577 * normal way, without attempting to store any values.
578 * @param status receives error codes.
579 * @return The number of rule status values from rules that determined
580 * the most recent boundary returned by the break iterator.
583 U_STABLE
int32_t U_EXPORT2
584 ubrk_getRuleStatusVec(UBreakIterator
*bi
, int32_t *fillInVec
, int32_t capacity
, UErrorCode
*status
);
587 * Return the locale of the break iterator. You can choose between the valid and
589 * @param bi break iterator
590 * @param type locale type (valid or actual)
591 * @param status error code
592 * @return locale string
595 U_STABLE
const char* U_EXPORT2
596 ubrk_getLocaleByType(const UBreakIterator
*bi
, ULocDataLocaleType type
, UErrorCode
* status
);
599 * Set the subject text string upon which the break iterator is operating
600 * without changing any other aspect of the state.
601 * The new and previous text strings must have the same content.
603 * This function is intended for use in environments where ICU is operating on
604 * strings that may move around in memory. It provides a mechanism for notifying
605 * ICU that the string has been relocated, and providing a new UText to access the
606 * string in its new position.
608 * Note that the break iterator never copies the underlying text
609 * of a string being processed, but always operates directly on the original text
610 * provided by the user. Refreshing simply drops the references to the old text
611 * and replaces them with references to the new.
613 * Caution: this function is normally used only by very specialized
614 * system-level code. One example use case is with garbage collection
615 * that moves the text in memory.
617 * @param bi The break iterator.
618 * @param text The new (moved) text string.
619 * @param status Receives errors detected by this function.
623 U_STABLE
void U_EXPORT2
624 ubrk_refreshUText(UBreakIterator
*bi
,
630 * Get a compiled binary version of the rules specifying the behavior of a UBreakIterator.
631 * The binary rules may be used with ubrk_openBinaryRules to open a new UBreakIterator
632 * more quickly than using ubrk_openRules. The compiled rules are not compatible across
633 * different major versions of ICU, nor across platforms of different endianness or
634 * different base character set family (ASCII vs EBCDIC). Supports preflighting (with
635 * binaryRules=NULL and rulesCapacity=0) to get the rules length without copying them to
636 * the binaryRules buffer. However, whether preflighting or not, if the actual length
637 * is greater than INT32_MAX, then the function returns 0 and sets *status to
638 * U_INDEX_OUTOFBOUNDS_ERROR.
640 * @param bi The break iterator to use.
641 * @param binaryRules Buffer to receive the compiled binary rules; set to NULL for
643 * @param rulesCapacity Capacity (in bytes) of the binaryRules buffer; set to 0 for
644 * preflighting. Must be >= 0.
645 * @param status Pointer to UErrorCode to receive any errors, such as
646 * U_BUFFER_OVERFLOW_ERROR, U_INDEX_OUTOFBOUNDS_ERROR, or
647 * U_ILLEGAL_ARGUMENT_ERROR.
648 * @return The actual byte length of the binary rules, if <= INT32_MAX;
649 * otherwise 0. If not preflighting and this is larger than
650 * rulesCapacity, *status will be set to an error.
651 * @see ubrk_openBinaryRules
654 U_STABLE
int32_t U_EXPORT2
655 ubrk_getBinaryRules(UBreakIterator
*bi
,
656 uint8_t * binaryRules
, int32_t rulesCapacity
,
657 UErrorCode
* status
);
659 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */