1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
4 ******************************************************************************
5 * Copyright (C) 1996-2015, International Business Machines Corporation and others.
7 ******************************************************************************
13 #include "unicode/utypes.h"
14 #include "unicode/uloc.h"
15 #include "unicode/utext.h"
16 #include "unicode/localpointer.h"
19 * A text-break iterator.
20 * For usage in C programs.
22 #ifndef UBRK_TYPEDEF_UBREAK_ITERATOR
23 # define UBRK_TYPEDEF_UBREAK_ITERATOR
25 * Opaque type representing an ICU Break iterator object.
28 typedef struct UBreakIterator UBreakIterator
;
31 #if !UCONFIG_NO_BREAK_ITERATION
33 #include "unicode/parseerr.h"
37 * \brief C API: BreakIterator
39 * <h2> BreakIterator C API </h2>
41 * The BreakIterator C API defines methods for finding the location
42 * of boundaries in text. Pointer to a UBreakIterator maintain a
43 * current position and scan over text returning the index of characters
44 * where boundaries occur.
46 * Line boundary analysis determines where a text string can be broken
47 * when line-wrapping. The mechanism correctly handles punctuation and
50 * Note: The locale keyword "lb" can be used to modify line break
51 * behavior according to the CSS level 3 line-break options, see
52 * <http://dev.w3.org/csswg/css-text/#line-breaking>. For example:
53 * "ja@lb=strict", "zh@lb=loose".
55 * Sentence boundary analysis allows selection with correct
56 * interpretation of periods within numbers and abbreviations, and
57 * trailing punctuation marks such as quotation marks and parentheses.
59 * Note: The locale keyword "ss" can be used to enable use of
60 * segmentation suppression data (preventing breaks in English after
61 * abbreviations such as "Mr." or "Est.", for example), as follows:
64 * Word boundary analysis is used by search and replace functions, as
65 * well as within text editing applications that allow the user to
66 * select words with a double click. Word selection provides correct
67 * interpretation of punctuation marks within and following
68 * words. Characters that are not part of a word, such as symbols or
69 * punctuation marks, have word-breaks on both sides.
71 * Character boundary analysis identifies the boundaries of
72 * "Extended Grapheme Clusters", which are groupings of codepoints
73 * that should be treated as character-like units for many text operations.
74 * Please see Unicode Standard Annex #29, Unicode Text Segmentation,
75 * http://www.unicode.org/reports/tr29/ for additional information
76 * on grapheme clusters and guidelines on their use.
78 * Title boundary analysis locates all positions,
79 * typically starts of words, that should be set to Title Case
80 * when title casing the text.
82 * The text boundary positions are found according to the rules
83 * described in Unicode Standard Annex #29, Text Boundaries, and
84 * Unicode Standard Annex #14, Line Breaking Properties. These
85 * are available at http://www.unicode.org/reports/tr14/ and
86 * http://www.unicode.org/reports/tr29/.
88 * In addition to the plain C API defined in this header file, an
89 * object oriented C++ API with equivalent functionality is defined in the
92 * Code snippets illustrating the use of the Break Iterator APIs
93 * are available in the ICU User Guide,
94 * http://icu-project.org/userguide/boundaryAnalysis.html
95 * and in the sample program icu/source/samples/break/break.cpp
98 /** The possible types of text boundaries. @stable ICU 2.0 */
99 typedef enum UBreakIteratorType
{
100 /** Character breaks @stable ICU 2.0 */
102 /** Word breaks @stable ICU 2.0 */
104 /** Line breaks @stable ICU 2.0 */
106 /** Sentence breaks @stable ICU 2.0 */
109 #ifndef U_HIDE_DEPRECATED_API
112 * The iterator created using this type locates title boundaries as described for
113 * Unicode 3.2 only. For Unicode 4.0 and above title boundary iteration,
114 * please use Word Boundary iterator.
116 * @deprecated ICU 2.8 Use the word break iterator for titlecasing for Unicode 4 and later.
120 * One more than the highest normal UBreakIteratorType value.
121 * @deprecated ICU 58 The numeric value may change over time, see ICU ticket #12420.
124 #endif // U_HIDE_DEPRECATED_API
125 } UBreakIteratorType
;
127 /** Value indicating all text boundaries have been returned.
130 #define UBRK_DONE ((int32_t) -1)
134 * Enum constants for the word break tags returned by
135 * getRuleStatus(). A range of values is defined for each category of
136 * word, to allow for further subdivisions of a category in future releases.
137 * Applications should check for tag values falling within the range, rather
138 * than for single individual values.
140 * The numeric values of all of these constants are stable (will not change).
144 typedef enum UWordBreak
{
145 /** Tag value for "words" that do not fit into any of other categories.
146 * Includes spaces and most punctuation. */
148 /** Upper bound for tags for uncategorized words. */
149 UBRK_WORD_NONE_LIMIT
= 100,
150 /** Tag value for words that appear to be numbers, lower limit. */
151 UBRK_WORD_NUMBER
= 100,
152 /** Tag value for words that appear to be numbers, upper limit. */
153 UBRK_WORD_NUMBER_LIMIT
= 200,
154 /** Tag value for words that contain letters, excluding
155 * hiragana, katakana or ideographic characters, lower limit. */
156 UBRK_WORD_LETTER
= 200,
157 /** Tag value for words containing letters, upper limit */
158 UBRK_WORD_LETTER_LIMIT
= 300,
159 /** Tag value for words containing kana characters, lower limit */
160 UBRK_WORD_KANA
= 300,
161 /** Tag value for words containing kana characters, upper limit */
162 UBRK_WORD_KANA_LIMIT
= 400,
163 /** Tag value for words containing ideographic characters, lower limit */
164 UBRK_WORD_IDEO
= 400,
165 /** Tag value for words containing ideographic characters, upper limit */
166 UBRK_WORD_IDEO_LIMIT
= 500
170 * Enum constants for the line break tags returned by getRuleStatus().
171 * A range of values is defined for each category of
172 * word, to allow for further subdivisions of a category in future releases.
173 * Applications should check for tag values falling within the range, rather
174 * than for single individual values.
176 * The numeric values of all of these constants are stable (will not change).
180 typedef enum ULineBreakTag
{
181 /** Tag value for soft line breaks, positions at which a line break
182 * is acceptable but not required */
184 /** Upper bound for soft line breaks. */
185 UBRK_LINE_SOFT_LIMIT
= 100,
186 /** Tag value for a hard, or mandatory line break */
187 UBRK_LINE_HARD
= 100,
188 /** Upper bound for hard line breaks. */
189 UBRK_LINE_HARD_LIMIT
= 200
195 * Enum constants for the sentence break tags returned by getRuleStatus().
196 * A range of values is defined for each category of
197 * sentence, to allow for further subdivisions of a category in future releases.
198 * Applications should check for tag values falling within the range, rather
199 * than for single individual values.
201 * The numeric values of all of these constants are stable (will not change).
205 typedef enum USentenceBreakTag
{
206 /** Tag value for for sentences ending with a sentence terminator
207 * ('.', '?', '!', etc.) character, possibly followed by a
208 * hard separator (CR, LF, PS, etc.)
210 UBRK_SENTENCE_TERM
= 0,
211 /** Upper bound for tags for sentences ended by sentence terminators. */
212 UBRK_SENTENCE_TERM_LIMIT
= 100,
213 /** Tag value for for sentences that do not contain an ending
214 * sentence terminator ('.', '?', '!', etc.) character, but
215 * are ended only by a hard separator (CR, LF, PS, etc.) or end of input.
217 UBRK_SENTENCE_SEP
= 100,
218 /** Upper bound for tags for sentences ended by a separator. */
219 UBRK_SENTENCE_SEP_LIMIT
= 200
220 /** Tag value for a hard, or mandatory line break */
225 * Open a new UBreakIterator for locating text boundaries for a specified locale.
226 * A UBreakIterator may be used for detecting character, line, word,
227 * and sentence breaks in text.
228 * @param type The type of UBreakIterator to open: one of UBRK_CHARACTER, UBRK_WORD,
229 * UBRK_LINE, UBRK_SENTENCE
230 * @param locale The locale specifying the text-breaking conventions. Note that
231 * locale keys such as "lb" and "ss" may be used to modify text break behavior,
232 * see general discussion of BreakIterator C API.
233 * @param text The text to be iterated over.
234 * @param textLength The number of characters in text, or -1 if null-terminated.
235 * @param status A UErrorCode to receive any errors.
236 * @return A UBreakIterator for the specified locale.
237 * @see ubrk_openRules
240 U_STABLE UBreakIterator
* U_EXPORT2
241 ubrk_open(UBreakIteratorType type
,
248 * Open a new UBreakIterator for locating text boundaries using specified breaking rules.
249 * The rule syntax is ... (TBD)
250 * @param rules A set of rules specifying the text breaking conventions.
251 * @param rulesLength The number of characters in rules, or -1 if null-terminated.
252 * @param text The text to be iterated over. May be null, in which case ubrk_setText() is
253 * used to specify the text to be iterated.
254 * @param textLength The number of characters in text, or -1 if null-terminated.
255 * @param parseErr Receives position and context information for any syntax errors
256 * detected while parsing the rules.
257 * @param status A UErrorCode to receive any errors.
258 * @return A UBreakIterator for the specified rules.
262 U_STABLE UBreakIterator
* U_EXPORT2
263 ubrk_openRules(const UChar
*rules
,
267 UParseError
*parseErr
,
270 #ifndef U_HIDE_DRAFT_API
272 * Open a new UBreakIterator for locating text boundaries using precompiled binary rules.
273 * Opening a UBreakIterator this way is substantially faster than using ubrk_openRules.
274 * Binary rules may be obtained using ubrk_getBinaryRules. The compiled rules are not
275 * compatible across different major versions of ICU, nor across platforms of different
276 * endianness or different base character set family (ASCII vs EBCDIC).
277 * @param binaryRules A set of compiled binary rules specifying the text breaking
278 * conventions. Ownership of the storage containing the compiled
279 * rules remains with the caller of this function. The compiled
280 * rules must not be modified or deleted during the life of the
282 * @param rulesLength The length of binaryRules in bytes; must be >= 0.
283 * @param text The text to be iterated over. May be null, in which case
284 * ubrk_setText() is used to specify the text to be iterated.
285 * @param textLength The number of characters in text, or -1 if null-terminated.
286 * @param status Pointer to UErrorCode to receive any errors.
287 * @return UBreakIterator for the specified rules.
288 * @see ubrk_getBinaryRules
291 U_DRAFT UBreakIterator
* U_EXPORT2
292 ubrk_openBinaryRules(const uint8_t *binaryRules
, int32_t rulesLength
,
293 const UChar
* text
, int32_t textLength
,
294 UErrorCode
* status
);
296 #endif /* U_HIDE_DRAFT_API */
299 * Thread safe cloning operation
300 * @param bi iterator to be cloned
301 * @param stackBuffer <em>Deprecated functionality as of ICU 52, use NULL.</em><br>
302 * user allocated space for the new clone. If NULL new memory will be allocated.
303 * If buffer is not large enough, new memory will be allocated.
304 * Clients can use the U_BRK_SAFECLONE_BUFFERSIZE.
305 * @param pBufferSize <em>Deprecated functionality as of ICU 52, use NULL or 1.</em><br>
306 * pointer to size of allocated space.
307 * If *pBufferSize == 0, a sufficient size for use in cloning will
308 * be returned ('pre-flighting')
309 * If *pBufferSize is not enough for a stack-based safe clone,
310 * new memory will be allocated.
311 * @param status to indicate whether the operation went on smoothly or there were errors
312 * An informational status value, U_SAFECLONE_ALLOCATED_ERROR, is used if any allocations were necessary.
313 * @return pointer to the new clone
316 U_STABLE UBreakIterator
* U_EXPORT2
318 const UBreakIterator
*bi
,
320 int32_t *pBufferSize
,
323 #ifndef U_HIDE_DEPRECATED_API
326 * A recommended size (in bytes) for the memory buffer to be passed to ubrk_saveClone().
327 * @deprecated ICU 52. Do not rely on ubrk_safeClone() cloning into any provided buffer.
329 #define U_BRK_SAFECLONE_BUFFERSIZE 1
331 #endif /* U_HIDE_DEPRECATED_API */
334 * Close a UBreakIterator.
335 * Once closed, a UBreakIterator may no longer be used.
336 * @param bi The break iterator to close.
339 U_STABLE
void U_EXPORT2
340 ubrk_close(UBreakIterator
*bi
);
342 #if U_SHOW_CPLUSPLUS_API
347 * \class LocalUBreakIteratorPointer
348 * "Smart pointer" class, closes a UBreakIterator via ubrk_close().
349 * For most methods see the LocalPointerBase base class.
351 * @see LocalPointerBase
355 U_DEFINE_LOCAL_OPEN_POINTER(LocalUBreakIteratorPointer
, UBreakIterator
, ubrk_close
);
359 #endif // U_SHOW_CPLUSPLUS_API
362 * Sets an existing iterator to point to a new piece of text.
363 * The break iterator retains a pointer to the supplied text.
364 * The caller must not modify or delete the text while the BreakIterator
365 * retains the reference.
367 * @param bi The iterator to use
368 * @param text The text to be set
369 * @param textLength The length of the text
370 * @param status The error code
373 U_STABLE
void U_EXPORT2
374 ubrk_setText(UBreakIterator
* bi
,
381 * Sets an existing iterator to point to a new piece of text.
383 * All index positions returned by break iterator functions are
384 * native indices from the UText. For example, when breaking UTF-8
385 * encoded text, the break positions returned by \ref ubrk_next, \ref ubrk_previous, etc.
386 * will be UTF-8 string indices, not UTF-16 positions.
388 * @param bi The iterator to use
389 * @param text The text to be set.
390 * This function makes a shallow clone of the supplied UText. This means
391 * that the caller is free to immediately close or otherwise reuse the
392 * UText that was passed as a parameter, but that the underlying text itself
393 * must not be altered while being referenced by the break iterator.
394 * @param status The error code
397 U_STABLE
void U_EXPORT2
398 ubrk_setUText(UBreakIterator
* bi
,
405 * Determine the most recently-returned text boundary.
407 * @param bi The break iterator to use.
408 * @return The character index most recently returned by \ref ubrk_next, \ref ubrk_previous,
409 * \ref ubrk_first, or \ref ubrk_last.
412 U_STABLE
int32_t U_EXPORT2
413 ubrk_current(const UBreakIterator
*bi
);
416 * Advance the iterator to the boundary following the current boundary.
418 * @param bi The break iterator to use.
419 * @return The character index of the next text boundary, or UBRK_DONE
420 * if all text boundaries have been returned.
424 U_STABLE
int32_t U_EXPORT2
425 ubrk_next(UBreakIterator
*bi
);
428 * Set the iterator position to the boundary preceding the current boundary.
430 * @param bi The break iterator to use.
431 * @return The character index of the preceding text boundary, or UBRK_DONE
432 * if all text boundaries have been returned.
436 U_STABLE
int32_t U_EXPORT2
437 ubrk_previous(UBreakIterator
*bi
);
440 * Set the iterator position to zero, the start of the text being scanned.
441 * @param bi The break iterator to use.
442 * @return The new iterator position (zero).
446 U_STABLE
int32_t U_EXPORT2
447 ubrk_first(UBreakIterator
*bi
);
450 * Set the iterator position to the index immediately <EM>beyond</EM> the last character in the text being scanned.
451 * This is not the same as the last character.
452 * @param bi The break iterator to use.
453 * @return The character offset immediately <EM>beyond</EM> the last character in the
454 * text being scanned.
458 U_STABLE
int32_t U_EXPORT2
459 ubrk_last(UBreakIterator
*bi
);
462 * Set the iterator position to the first boundary preceding the specified offset.
463 * The new position is always smaller than offset, or UBRK_DONE.
464 * @param bi The break iterator to use.
465 * @param offset The offset to begin scanning.
466 * @return The text boundary preceding offset, or UBRK_DONE.
467 * @see ubrk_following
470 U_STABLE
int32_t U_EXPORT2
471 ubrk_preceding(UBreakIterator
*bi
,
475 * Advance the iterator to the first boundary following the specified offset.
476 * The value returned is always greater than offset, or UBRK_DONE.
477 * @param bi The break iterator to use.
478 * @param offset The offset to begin scanning.
479 * @return The text boundary following offset, or UBRK_DONE.
480 * @see ubrk_preceding
483 U_STABLE
int32_t U_EXPORT2
484 ubrk_following(UBreakIterator
*bi
,
488 * Get a locale for which text breaking information is available.
489 * A UBreakIterator in a locale returned by this function will perform the correct
490 * text breaking for the locale.
491 * @param index The index of the desired locale.
492 * @return A locale for which number text breaking information is available, or 0 if none.
493 * @see ubrk_countAvailable
496 U_STABLE
const char* U_EXPORT2
497 ubrk_getAvailable(int32_t index
);
500 * Determine how many locales have text breaking information available.
501 * This function is most useful as determining the loop ending condition for
502 * calls to \ref ubrk_getAvailable.
503 * @return The number of locales for which text breaking information is available.
504 * @see ubrk_getAvailable
507 U_STABLE
int32_t U_EXPORT2
508 ubrk_countAvailable(void);
512 * Returns true if the specfied position is a boundary position. As a side
513 * effect, leaves the iterator pointing to the first boundary position at
515 * @param bi The break iterator to use.
516 * @param offset the offset to check.
517 * @return True if "offset" is a boundary position.
520 U_STABLE UBool U_EXPORT2
521 ubrk_isBoundary(UBreakIterator
*bi
, int32_t offset
);
524 * Return the status from the break rule that determined the most recently
525 * returned break position. The values appear in the rule source
526 * within brackets, {123}, for example. For rules that do not specify a
527 * status, a default value of 0 is returned.
529 * For word break iterators, the possible values are defined in enum UWordBreak.
532 U_STABLE
int32_t U_EXPORT2
533 ubrk_getRuleStatus(UBreakIterator
*bi
);
536 * Get the statuses from the break rules that determined the most recently
537 * returned break position. The values appear in the rule source
538 * within brackets, {123}, for example. The default status value for rules
539 * that do not explicitly provide one is zero.
541 * For word break iterators, the possible values are defined in enum UWordBreak.
542 * @param bi The break iterator to use
543 * @param fillInVec an array to be filled in with the status values.
544 * @param capacity the length of the supplied vector. A length of zero causes
545 * the function to return the number of status values, in the
546 * normal way, without attemtping to store any values.
547 * @param status receives error codes.
548 * @return The number of rule status values from rules that determined
549 * the most recent boundary returned by the break iterator.
552 U_STABLE
int32_t U_EXPORT2
553 ubrk_getRuleStatusVec(UBreakIterator
*bi
, int32_t *fillInVec
, int32_t capacity
, UErrorCode
*status
);
556 * Return the locale of the break iterator. You can choose between the valid and
558 * @param bi break iterator
559 * @param type locale type (valid or actual)
560 * @param status error code
561 * @return locale string
564 U_STABLE
const char* U_EXPORT2
565 ubrk_getLocaleByType(const UBreakIterator
*bi
, ULocDataLocaleType type
, UErrorCode
* status
);
568 * Set the subject text string upon which the break iterator is operating
569 * without changing any other aspect of the state.
570 * The new and previous text strings must have the same content.
572 * This function is intended for use in environments where ICU is operating on
573 * strings that may move around in memory. It provides a mechanism for notifying
574 * ICU that the string has been relocated, and providing a new UText to access the
575 * string in its new position.
577 * Note that the break iterator never copies the underlying text
578 * of a string being processed, but always operates directly on the original text
579 * provided by the user. Refreshing simply drops the references to the old text
580 * and replaces them with references to the new.
582 * Caution: this function is normally used only by very specialized
583 * system-level code. One example use case is with garbage collection
584 * that moves the text in memory.
586 * @param bi The break iterator.
587 * @param text The new (moved) text string.
588 * @param status Receives errors detected by this function.
592 U_STABLE
void U_EXPORT2
593 ubrk_refreshUText(UBreakIterator
*bi
,
598 #ifndef U_HIDE_DRAFT_API
600 * Get a compiled binary version of the rules specifying the behavior of a UBreakIterator.
601 * The binary rules may be used with ubrk_openBinaryRules to open a new UBreakIterator
602 * more quickly than using ubrk_openRules. The compiled rules are not compatible across
603 * different major versions of ICU, nor across platforms of different endianness or
604 * different base character set family (ASCII vs EBCDIC). Supports preflighting (with
605 * binaryRules=NULL and rulesCapacity=0) to get the rules length without copying them to
606 * the binaryRules buffer. However, whether preflighting or not, if the actual length
607 * is greater than INT32_MAX, then the function returns 0 and sets *status to
608 * U_INDEX_OUTOFBOUNDS_ERROR.
610 * @param bi The break iterator to use.
611 * @param binaryRules Buffer to receive the compiled binary rules; set to NULL for
613 * @param rulesCapacity Capacity (in bytes) of the binaryRules buffer; set to 0 for
614 * preflighting. Must be >= 0.
615 * @param status Pointer to UErrorCode to receive any errors, such as
616 * U_BUFFER_OVERFLOW_ERROR, U_INDEX_OUTOFBOUNDS_ERROR, or
617 * U_ILLEGAL_ARGUMENT_ERROR.
618 * @return The actual byte length of the binary rules, if <= INT32_MAX;
619 * otherwise 0. If not preflighting and this is larger than
620 * rulesCapacity, *status will be set to an error.
621 * @see ubrk_openBinaryRules
624 U_DRAFT
int32_t U_EXPORT2
625 ubrk_getBinaryRules(UBreakIterator
*bi
,
626 uint8_t * binaryRules
, int32_t rulesCapacity
,
627 UErrorCode
* status
);
629 #endif /* U_HIDE_DRAFT_API */
631 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */