icuSources/common/unicode/ubrk.h

   1 /*
   2 ******************************************************************************
   3 * Copyright (C) 1996-2015, International Business Machines Corporation and others.
   4 * All Rights Reserved.
   5 ******************************************************************************
   6 */
   7
   8 #ifndef UBRK_H
   9 #define UBRK_H
  10
  11 #include "unicode/utypes.h"
  12 #include "unicode/uloc.h"
  13 #include "unicode/utext.h"
  14 #include "unicode/localpointer.h"
  15
  16 /**
  17  * A text-break iterator.
  18  *  For usage in C programs.
  19  */
  20 #ifndef UBRK_TYPEDEF_UBREAK_ITERATOR
  21 #   define UBRK_TYPEDEF_UBREAK_ITERATOR
  22     /**
  23      *  Opaque type representing an ICU Break iterator object.
  24      *  @stable ICU 2.0
  25      */
  26     typedef struct UBreakIterator UBreakIterator;
  27 #endif
  28
  29 #if !UCONFIG_NO_BREAK_ITERATION
  30
  31 #include "unicode/parseerr.h"
  32
  33 /**
  34  * \file
  35  * \brief C API: BreakIterator
  36  *
  37  * <h2> BreakIterator C API </h2>
  38  *
  39  * The BreakIterator C API defines  methods for finding the location
  40  * of boundaries in text. Pointer to a UBreakIterator maintain a
  41  * current position and scan over text returning the index of characters
  42  * where boundaries occur.
  43  * <p>
  44  * Line boundary analysis determines where a text string can be broken
  45  * when line-wrapping. The mechanism correctly handles punctuation and
  46  * hyphenated words.
  47  * <p>
  48  * Note: The locale keyword "lb" can be used to modify line break
  49  * behavior according to the CSS level 3 line-break options, see
  50  * <http://dev.w3.org/csswg/css-text/#line-breaking>. For example:
  51  * "ja@lb=strict", "zh@lb=loose".
  52  * <p>
  53  * Sentence boundary analysis allows selection with correct
  54  * interpretation of periods within numbers and abbreviations, and
  55  * trailing punctuation marks such as quotation marks and parentheses.
  56  * <p>
  57  * Note: The locale keyword "ss" can be used to enable use of
  58  * segmentation suppression data (preventing breaks in English after
  59  * abbreviations such as "Mr." or "Est.", for example), as follows:
  60  * "en@ss=standard". However, use of the suppression data is
  61  * currently supported only for ubrk_next; ubrk_previous, ubrk_following,
  62  * and ubrk_preceding will ignore the suppression data.
  63  * <p>
  64  * Word boundary analysis is used by search and replace functions, as
  65  * well as within text editing applications that allow the user to
  66  * select words with a double click. Word selection provides correct
  67  * interpretation of punctuation marks within and following
  68  * words. Characters that are not part of a word, such as symbols or
  69  * punctuation marks, have word-breaks on both sides.
  70  * <p>
  71  * Character boundary analysis identifies the boundaries of
  72  * "Extended Grapheme Clusters", which are groupings of codepoints
  73  * that should be treated as character-like units for many text operations.
  74  * Please see Unicode Standard Annex #29, Unicode Text Segmentation,
  75  * http://www.unicode.org/reports/tr29/ for additional information
  76  * on grapheme clusters and guidelines on their use.
  77  * <p>
  78  * Title boundary analysis locates all positions,
  79  * typically starts of words, that should be set to Title Case
  80  * when title casing the text.
  81  * <p>
  82  * The text boundary positions are found according to the rules
  83  * described in Unicode Standard Annex #29, Text Boundaries, and
  84  * Unicode Standard Annex #14, Line Breaking Properties.  These
  85  * are available at http://www.unicode.org/reports/tr14/ and
  86  * http://www.unicode.org/reports/tr29/.
  87  * <p>
  88  * In addition to the plain C API defined in this header file, an
  89  * object oriented C++ API with equivalent functionality is defined in the
  90  * file brkiter.h.
  91  * <p>
  92  * Code snippets illustrating the use of the Break Iterator APIs
  93  * are available in the ICU User Guide,
  94  * http://icu-project.org/userguide/boundaryAnalysis.html
  95  * and in the sample program icu/source/samples/break/break.cpp
  96  */
  97
  98 /** The possible types of text boundaries.  @stable ICU 2.0 */
  99 typedef enum UBreakIteratorType {
 100   /** Character breaks  @stable ICU 2.0 */
 101   UBRK_CHARACTER = 0,
 102   /** Word breaks @stable ICU 2.0 */
 103   UBRK_WORD = 1,
 104   /** Line breaks @stable ICU 2.0 */
 105   UBRK_LINE = 2,
 106   /** Sentence breaks @stable ICU 2.0 */
 107   UBRK_SENTENCE = 3,
 108
 109 #ifndef U_HIDE_DEPRECATED_API
 110   /**
 111    * Title Case breaks
 112    * The iterator created using this type locates title boundaries as described for
 113    * Unicode 3.2 only. For Unicode 4.0 and above title boundary iteration,
 114    * please use Word Boundary iterator.
 115    *
 116    * @deprecated ICU 2.8 Use the word break iterator for titlecasing for Unicode 4 and later.
 117    */
 118   UBRK_TITLE = 4,
 119 #endif /* U_HIDE_DEPRECATED_API */
 120   UBRK_COUNT = 5
 121 } UBreakIteratorType;
 122
 123 /** Value indicating all text boundaries have been returned.
 124  *  @stable ICU 2.0
 125  */
 126 #define UBRK_DONE ((int32_t) -1)
 127
 128
 129 /**
 130  *  Enum constants for the word break tags returned by
 131  *  getRuleStatus().  A range of values is defined for each category of
 132  *  word, to allow for further subdivisions of a category in future releases.
 133  *  Applications should check for tag values falling within the range, rather
 134  *  than for single individual values.
 135  *  @stable ICU 2.2
 136 */
 137 typedef enum UWordBreak {
 138     /** Tag value for "words" that do not fit into any of other categories.
 139      *  Includes spaces and most punctuation. */
 140     UBRK_WORD_NONE           = 0,
 141     /** Upper bound for tags for uncategorized words. */
 142     UBRK_WORD_NONE_LIMIT     = 100,
 143     /** Tag value for words that appear to be numbers, lower limit.    */
 144     UBRK_WORD_NUMBER         = 100,
 145     /** Tag value for words that appear to be numbers, upper limit.    */
 146     UBRK_WORD_NUMBER_LIMIT   = 200,
 147     /** Tag value for words that contain letters, excluding
 148      *  hiragana, katakana or ideographic characters, lower limit.    */
 149     UBRK_WORD_LETTER         = 200,
 150     /** Tag value for words containing letters, upper limit  */
 151     UBRK_WORD_LETTER_LIMIT   = 300,
 152     /** Tag value for words containing kana characters, lower limit */
 153     UBRK_WORD_KANA           = 300,
 154     /** Tag value for words containing kana characters, upper limit */
 155     UBRK_WORD_KANA_LIMIT     = 400,
 156     /** Tag value for words containing ideographic characters, lower limit */
 157     UBRK_WORD_IDEO           = 400,
 158     /** Tag value for words containing ideographic characters, upper limit */
 159     UBRK_WORD_IDEO_LIMIT     = 500
 160 } UWordBreak;
 161
 162 /**
 163  *  Enum constants for the line break tags returned by getRuleStatus().
 164  *  A range of values is defined for each category of
 165  *  word, to allow for further subdivisions of a category in future releases.
 166  *  Applications should check for tag values falling within the range, rather
 167  *  than for single individual values.
 168  *  @stable ICU 2.8
 169 */
 170 typedef enum ULineBreakTag {
 171     /** Tag value for soft line breaks, positions at which a line break
 172       *  is acceptable but not required                */
 173     UBRK_LINE_SOFT            = 0,
 174     /** Upper bound for soft line breaks.              */
 175     UBRK_LINE_SOFT_LIMIT      = 100,
 176     /** Tag value for a hard, or mandatory line break  */
 177     UBRK_LINE_HARD            = 100,
 178     /** Upper bound for hard line breaks.              */
 179     UBRK_LINE_HARD_LIMIT      = 200
 180 } ULineBreakTag;
 181
 182
 183
 184 /**
 185  *  Enum constants for the sentence break tags returned by getRuleStatus().
 186  *  A range of values is defined for each category of
 187  *  sentence, to allow for further subdivisions of a category in future releases.
 188  *  Applications should check for tag values falling within the range, rather
 189  *  than for single individual values.
 190  *  @stable ICU 2.8
 191 */
 192 typedef enum USentenceBreakTag {
 193     /** Tag value for for sentences  ending with a sentence terminator
 194       * ('.', '?', '!', etc.) character, possibly followed by a
 195       * hard separator (CR, LF, PS, etc.)
 196       */
 197     UBRK_SENTENCE_TERM       = 0,
 198     /** Upper bound for tags for sentences ended by sentence terminators.    */
 199     UBRK_SENTENCE_TERM_LIMIT = 100,
 200     /** Tag value for for sentences that do not contain an ending
 201       * sentence terminator ('.', '?', '!', etc.) character, but
 202       * are ended only by a hard separator (CR, LF, PS, etc.) or end of input.
 203       */
 204     UBRK_SENTENCE_SEP        = 100,
 205     /** Upper bound for tags for sentences ended by a separator.              */
 206     UBRK_SENTENCE_SEP_LIMIT  = 200
 207     /** Tag value for a hard, or mandatory line break  */
 208 } USentenceBreakTag;
 209
 210
 211 /**
 212  * Open a new UBreakIterator for locating text boundaries for a specified locale.
 213  * A UBreakIterator may be used for detecting character, line, word,
 214  * and sentence breaks in text.
 215  * @param type The type of UBreakIterator to open: one of UBRK_CHARACTER, UBRK_WORD,
 216  * UBRK_LINE, UBRK_SENTENCE
 217  * @param locale The locale specifying the text-breaking conventions. Note that
 218  * locale keys such as "lb" and "ss" may be used to modify text break behavior,
 219  * see general discussion of BreakIterator C API.
 220  * @param text The text to be iterated over.
 221  * @param textLength The number of characters in text, or -1 if null-terminated.
 222  * @param status A UErrorCode to receive any errors.
 223  * @return A UBreakIterator for the specified locale.
 224  * @see ubrk_openRules
 225  * @stable ICU 2.0
 226  */
 227 U_STABLE UBreakIterator* U_EXPORT2
 228 ubrk_open(UBreakIteratorType type,
 229       const char *locale,
 230       const UChar *text,
 231       int32_t textLength,
 232       UErrorCode *status);
 233
 234 /**
 235  * Open a new UBreakIterator for locating text boundaries using specified breaking rules.
 236  * The rule syntax is ... (TBD)
 237  * @param rules A set of rules specifying the text breaking conventions.
 238  * @param rulesLength The number of characters in rules, or -1 if null-terminated.
 239  * @param text The text to be iterated over.  May be null, in which case ubrk_setText() is
 240  *        used to specify the text to be iterated.
 241  * @param textLength The number of characters in text, or -1 if null-terminated.
 242  * @param parseErr   Receives position and context information for any syntax errors
 243  *                   detected while parsing the rules.
 244  * @param status A UErrorCode to receive any errors.
 245  * @return A UBreakIterator for the specified rules.
 246  * @see ubrk_open
 247  * @stable ICU 2.2
 248  */
 249 U_STABLE UBreakIterator* U_EXPORT2
 250 ubrk_openRules(const UChar     *rules,
 251                int32_t         rulesLength,
 252                const UChar     *text,
 253                int32_t          textLength,
 254                UParseError     *parseErr,
 255                UErrorCode      *status);
 256
 257 /**
 258  * Thread safe cloning operation
 259  * @param bi iterator to be cloned
 260  * @param stackBuffer <em>Deprecated functionality as of ICU 52, use NULL.</em><br>
 261  *  user allocated space for the new clone. If NULL new memory will be allocated.
 262  *  If buffer is not large enough, new memory will be allocated.
 263  *  Clients can use the U_BRK_SAFECLONE_BUFFERSIZE.
 264  * @param pBufferSize <em>Deprecated functionality as of ICU 52, use NULL or 1.</em><br>
 265  *  pointer to size of allocated space.
 266  *  If *pBufferSize == 0, a sufficient size for use in cloning will
 267  *  be returned ('pre-flighting')
 268  *  If *pBufferSize is not enough for a stack-based safe clone,
 269  *  new memory will be allocated.
 270  * @param status to indicate whether the operation went on smoothly or there were errors
 271  *  An informational status value, U_SAFECLONE_ALLOCATED_ERROR, is used if any allocations were necessary.
 272  * @return pointer to the new clone
 273  * @stable ICU 2.0
 274  */
 275 U_STABLE UBreakIterator * U_EXPORT2
 276 ubrk_safeClone(
 277           const UBreakIterator *bi,
 278           void *stackBuffer,
 279           int32_t *pBufferSize,
 280           UErrorCode *status);
 281
 282 #ifndef U_HIDE_DEPRECATED_API
 283
 284 /**
 285   * A recommended size (in bytes) for the memory buffer to be passed to ubrk_saveClone().
 286   * @deprecated ICU 52. Do not rely on ubrk_safeClone() cloning into any provided buffer.
 287   */
 288 #define U_BRK_SAFECLONE_BUFFERSIZE 1
 289
 290 #endif /* U_HIDE_DEPRECATED_API */
 291
 292 /**
 293 * Close a UBreakIterator.
 294 * Once closed, a UBreakIterator may no longer be used.
 295 * @param bi The break iterator to close.
 296  * @stable ICU 2.0
 297 */
 298 U_STABLE void U_EXPORT2
 299 ubrk_close(UBreakIterator *bi);
 300
 301 #if U_SHOW_CPLUSPLUS_API
 302
 303 U_NAMESPACE_BEGIN
 304
 305 /**
 306  * \class LocalUBreakIteratorPointer
 307  * "Smart pointer" class, closes a UBreakIterator via ubrk_close().
 308  * For most methods see the LocalPointerBase base class.
 309  *
 310  * @see LocalPointerBase
 311  * @see LocalPointer
 312  * @stable ICU 4.4
 313  */
 314 U_DEFINE_LOCAL_OPEN_POINTER(LocalUBreakIteratorPointer, UBreakIterator, ubrk_close);
 315
 316 U_NAMESPACE_END
 317
 318 #endif
 319
 320 /**
 321  * Sets an existing iterator to point to a new piece of text
 322  * @param bi The iterator to use
 323  * @param text The text to be set
 324  * @param textLength The length of the text
 325  * @param status The error code
 326  * @stable ICU 2.0
 327  */
 328 U_STABLE void U_EXPORT2
 329 ubrk_setText(UBreakIterator* bi,
 330              const UChar*    text,
 331              int32_t         textLength,
 332              UErrorCode*     status);
 333
 334
 335 /**
 336  * Sets an existing iterator to point to a new piece of text.
 337  *
 338  * All index positions returned by break iterator functions are
 339  * native indices from the UText. For example, when breaking UTF-8
 340  * encoded text, the break positions returned by \ref ubrk_next, \ref ubrk_previous, etc.
 341  * will be UTF-8 string indices, not UTF-16 positions.
 342  *
 343  * @param bi The iterator to use
 344  * @param text The text to be set.
 345  *             This function makes a shallow clone of the supplied UText.  This means
 346  *             that the caller is free to immediately close or otherwise reuse the
 347  *             UText that was passed as a parameter, but that the underlying text itself
 348  *             must not be altered while being referenced by the break iterator.
 349  * @param status The error code
 350  * @stable ICU 3.4
 351  */
 352 U_STABLE void U_EXPORT2
 353 ubrk_setUText(UBreakIterator* bi,
 354              UText*          text,
 355              UErrorCode*     status);
 356
 357
 358
 359 /**
 360  * Determine the most recently-returned text boundary.
 361  *
 362  * @param bi The break iterator to use.
 363  * @return The character index most recently returned by \ref ubrk_next, \ref ubrk_previous,
 364  * \ref ubrk_first, or \ref ubrk_last.
 365  * @stable ICU 2.0
 366  */
 367 U_STABLE int32_t U_EXPORT2
 368 ubrk_current(const UBreakIterator *bi);
 369
 370 /**
 371  * Advance the iterator to the boundary following the current boundary.
 372  *
 373  * @param bi The break iterator to use.
 374  * @return The character index of the next text boundary, or UBRK_DONE
 375  * if all text boundaries have been returned.
 376  * @see ubrk_previous
 377  * @stable ICU 2.0
 378  */
 379 U_STABLE int32_t U_EXPORT2
 380 ubrk_next(UBreakIterator *bi);
 381
 382 /**
 383  * Set the iterator position to the boundary preceding the current boundary.
 384  *
 385  * @param bi The break iterator to use.
 386  * @return The character index of the preceding text boundary, or UBRK_DONE
 387  * if all text boundaries have been returned.
 388  * @see ubrk_next
 389  * @stable ICU 2.0
 390  */
 391 U_STABLE int32_t U_EXPORT2
 392 ubrk_previous(UBreakIterator *bi);
 393
 394 /**
 395  * Set the iterator position to zero, the start of the text being scanned.
 396  * @param bi The break iterator to use.
 397  * @return The new iterator position (zero).
 398  * @see ubrk_last
 399  * @stable ICU 2.0
 400  */
 401 U_STABLE int32_t U_EXPORT2
 402 ubrk_first(UBreakIterator *bi);
 403
 404 /**
 405  * Set the iterator position to the index immediately <EM>beyond</EM> the last character in the text being scanned.
 406  * This is not the same as the last character.
 407  * @param bi The break iterator to use.
 408  * @return The character offset immediately <EM>beyond</EM> the last character in the
 409  * text being scanned.
 410  * @see ubrk_first
 411  * @stable ICU 2.0
 412  */
 413 U_STABLE int32_t U_EXPORT2
 414 ubrk_last(UBreakIterator *bi);
 415
 416 /**
 417  * Set the iterator position to the first boundary preceding the specified offset.
 418  * The new position is always smaller than offset, or UBRK_DONE.
 419  * @param bi The break iterator to use.
 420  * @param offset The offset to begin scanning.
 421  * @return The text boundary preceding offset, or UBRK_DONE.
 422  * @see ubrk_following
 423  * @stable ICU 2.0
 424  */
 425 U_STABLE int32_t U_EXPORT2
 426 ubrk_preceding(UBreakIterator *bi,
 427            int32_t offset);
 428
 429 /**
 430  * Advance the iterator to the first boundary following the specified offset.
 431  * The value returned is always greater than offset, or UBRK_DONE.
 432  * @param bi The break iterator to use.
 433  * @param offset The offset to begin scanning.
 434  * @return The text boundary following offset, or UBRK_DONE.
 435  * @see ubrk_preceding
 436  * @stable ICU 2.0
 437  */
 438 U_STABLE int32_t U_EXPORT2
 439 ubrk_following(UBreakIterator *bi,
 440            int32_t offset);
 441
 442 /**
 443 * Get a locale for which text breaking information is available.
 444 * A UBreakIterator in a locale returned by this function will perform the correct
 445 * text breaking for the locale.
 446 * @param index The index of the desired locale.
 447 * @return A locale for which number text breaking information is available, or 0 if none.
 448 * @see ubrk_countAvailable
 449 * @stable ICU 2.0
 450 */
 451 U_STABLE const char* U_EXPORT2
 452 ubrk_getAvailable(int32_t index);
 453
 454 /**
 455 * Determine how many locales have text breaking information available.
 456 * This function is most useful as determining the loop ending condition for
 457 * calls to \ref ubrk_getAvailable.
 458 * @return The number of locales for which text breaking information is available.
 459 * @see ubrk_getAvailable
 460 * @stable ICU 2.0
 461 */
 462 U_STABLE int32_t U_EXPORT2
 463 ubrk_countAvailable(void);
 464
 465
 466 /**
 467 * Returns true if the specfied position is a boundary position.  As a side
 468 * effect, leaves the iterator pointing to the first boundary position at
 469 * or after "offset".
 470 * @param bi The break iterator to use.
 471 * @param offset the offset to check.
 472 * @return True if "offset" is a boundary position.
 473 * @stable ICU 2.0
 474 */
 475 U_STABLE  UBool U_EXPORT2
 476 ubrk_isBoundary(UBreakIterator *bi, int32_t offset);
 477
 478 /**
 479  * Return the status from the break rule that determined the most recently
 480  * returned break position.  The values appear in the rule source
 481  * within brackets, {123}, for example.  For rules that do not specify a
 482  * status, a default value of 0 is returned.
 483  * <p>
 484  * For word break iterators, the possible values are defined in enum UWordBreak.
 485  * @stable ICU 2.2
 486  */
 487 U_STABLE  int32_t U_EXPORT2
 488 ubrk_getRuleStatus(UBreakIterator *bi);
 489
 490 /**
 491  * Get the statuses from the break rules that determined the most recently
 492  * returned break position.  The values appear in the rule source
 493  * within brackets, {123}, for example.  The default status value for rules
 494  * that do not explicitly provide one is zero.
 495  * <p>
 496  * For word break iterators, the possible values are defined in enum UWordBreak.
 497  * @param bi        The break iterator to use
 498  * @param fillInVec an array to be filled in with the status values.
 499  * @param capacity  the length of the supplied vector.  A length of zero causes
 500  *                  the function to return the number of status values, in the
 501  *                  normal way, without attemtping to store any values.
 502  * @param status    receives error codes.
 503  * @return          The number of rule status values from rules that determined
 504  *                  the most recent boundary returned by the break iterator.
 505  * @stable ICU 3.0
 506  */
 507 U_STABLE  int32_t U_EXPORT2
 508 ubrk_getRuleStatusVec(UBreakIterator *bi, int32_t *fillInVec, int32_t capacity, UErrorCode *status);
 509
 510 /**
 511  * Return the locale of the break iterator. You can choose between the valid and
 512  * the actual locale.
 513  * @param bi break iterator
 514  * @param type locale type (valid or actual)
 515  * @param status error code
 516  * @return locale string
 517  * @stable ICU 2.8
 518  */
 519 U_STABLE const char* U_EXPORT2
 520 ubrk_getLocaleByType(const UBreakIterator *bi, ULocDataLocaleType type, UErrorCode* status);
 521
 522 /**
 523   *  Set the subject text string upon which the break iterator is operating
 524   *  without changing any other aspect of the state.
 525   *  The new and previous text strings must have the same content.
 526   *
 527   *  This function is intended for use in environments where ICU is operating on
 528   *  strings that may move around in memory.  It provides a mechanism for notifying
 529   *  ICU that the string has been relocated, and providing a new UText to access the
 530   *  string in its new position.
 531   *
 532   *  Note that the break iterator never copies the underlying text
 533   *  of a string being processed, but always operates directly on the original text
 534   *  provided by the user. Refreshing simply drops the references to the old text
 535   *  and replaces them with references to the new.
 536   *
 537   *  Caution:  this function is normally used only by very specialized
 538   *            system-level code.   One example use case is with garbage collection
 539   *            that moves the text in memory.
 540   *
 541   * @param bi         The break iterator.
 542   * @param text       The new (moved) text string.
 543   * @param status     Receives errors detected by this function.
 544   *
 545   * @stable ICU 49
 546   */
 547 U_STABLE void U_EXPORT2
 548 ubrk_refreshUText(UBreakIterator *bi,
 549                        UText          *text,
 550                        UErrorCode     *status);
 551
 552 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
 553
 554 #endif