icuSources/common/unicode/rbbi.h

   1 /*
   2 ***************************************************************************
   3 *   Copyright (C) 1999-2004 International Business Machines Corporation   *
   4 *   and others. All rights reserved.                                      *
   5 ***************************************************************************
   6
   7 **********************************************************************
   8 *   Date        Name        Description
   9 *   10/22/99    alan        Creation.
  10 *   11/11/99    rgillam     Complete port from Java.
  11 **********************************************************************
  12 */
  13
  14 #ifndef RBBI_H
  15 #define RBBI_H
  16
  17 #include "unicode/utypes.h"
  18
  19 #if !UCONFIG_NO_BREAK_ITERATION
  20
  21 #include "unicode/brkiter.h"
  22 #include "unicode/udata.h"
  23 #include "unicode/parseerr.h"
  24
  25 struct UTrie;
  26
  27 U_NAMESPACE_BEGIN
  28
  29 /** @internal */
  30 struct RBBIDataHeader;
  31 class  RuleBasedBreakIteratorTables;
  32 class  BreakIterator;
  33 class  RBBIDataWrapper;
  34 struct RBBIStateTable;
  35
  36
  37
  38 /**
  39  * A subclass of BreakIterator whose behavior is specified using a list of rules.
  40  * <p>Instances of this class are most commonly created by the factory methods of
  41  *  BreakIterator::createWordInstance(), BreakIterator::createLineInstance(), etc.,
  42  *  and then used via the abstract API in class BreakIterator</p>
  43  *
  44  * <p>See the ICU User Guide for information on Break Iterator Rules.</p>
  45  *
  46  * <p>This class is not intended to be subclassed.  (Class DictionaryBasedBreakIterator
  47  *    is a subclass, but that relationship is effectively internal to the ICU
  48  *    implementation.  The subclassing interface to RulesBasedBreakIterator is
  49  *    not part of the ICU API, and may not remain stable.</p>
  50  *
  51  */
  52 class U_COMMON_API RuleBasedBreakIterator : public BreakIterator {
  53
  54 protected:
  55     /**
  56      * The character iterator through which this BreakIterator accesses the text
  57      * @internal
  58      */
  59     CharacterIterator*  fText;
  60
  61     /**
  62      * The rule data for this BreakIterator instance
  63      * @internal
  64      */
  65     RBBIDataWrapper    *fData;
  66
  67     /** Index of the Rule {tag} values for the most recent match.
  68      *  @internal
  69     */
  70     int32_t             fLastRuleStatusIndex;
  71
  72     /**
  73      * Rule tag value valid flag.
  74      * Some iterator operations don't intrinsically set the correct tag value.
  75      * This flag lets us lazily compute the value if we are ever asked for it.
  76      * @internal
  77      */
  78     UBool               fLastStatusIndexValid;
  79
  80     /**
  81      * Counter for the number of characters encountered with the "dictionary"
  82      *   flag set.  Normal RBBI iterators don't use it, although the code
  83      *   for updating it is live.  Dictionary Based break iterators (a subclass
  84      *   of us) access this field directly.
  85      * @internal
  86      */
  87     uint32_t           fDictionaryCharCount;
  88
  89     /**
  90      * Debugging flag.  Trace operation of state machine when true.
  91      * @internal
  92      */
  93     static UBool        fTrace;
  94
  95
  96 protected:
  97     //=======================================================================
  98     // constructors
  99     //=======================================================================
 100
 101     /**
 102      * Constructor from a flattened set of RBBI data in malloced memory.
 103      *             RulesBasedBreakIterators built from a custom set of rules
 104      *             are created via this constructor; the rules are compiled
 105      *             into memory, then the break iterator is constructed here.
 106      *
 107      *             The break iterator adopts the memory, and will
 108      *             free it when done.
 109      * @internal
 110      */
 111     RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
 112
 113     friend class RBBIRuleBuilder; /** @internal */
 114     friend class BreakIterator;
 115
 116
 117
 118 public:
 119
 120     /** Default constructor.  Creates an empty shell of an iterator, with no
 121      *  rules or text to iterate over.   Object can subsequently be assigned to.
 122      *  @stable ICU 2.2
 123      */
 124     RuleBasedBreakIterator();
 125
 126     /**
 127      * Copy constructor.  Will produce a break iterator with the same behavior,
 128      * and which iterates over the same text, as the one passed in.
 129      * @param that The RuleBasedBreakIterator passed to be copied
 130      * @stable ICU 2.0
 131      */
 132     RuleBasedBreakIterator(const RuleBasedBreakIterator& that);
 133
 134     /**
 135      * Construct a RuleBasedBreakIterator from a set of rules supplied as a string.
 136      * @param rules The break rules to be used.
 137      * @param parseError  In the event of a syntax error in the rules, provides the location
 138      *                    within the rules of the problem.
 139      * @param status Information on any errors encountered.
 140      * @stable ICU 2.2
 141      */
 142     RuleBasedBreakIterator( const UnicodeString    &rules,
 143                              UParseError           &parseError,
 144                              UErrorCode            &status);
 145
 146
 147     /**
 148      * This constructor uses the udata interface to create a BreakIterator
 149      * whose internal tables live in a memory-mapped file.  "image" is an
 150      * ICU UDataMemory handle for the pre-compiled break iterator tables.
 151      * @param image handle to the memory image for the break iterator data.
 152      *        Ownership of the UDataMemory handle passes to the Break Iterator,
 153      *        which will be responsible for closing it when it is no longer needed.
 154      * @param status Information on any errors encountered.
 155      * @see udata_open
 156      * @see #getBinaryRules
 157      * @draft ICU 2.8
 158      */
 159     RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status);
 160
 161     /**
 162      * Destructor
 163      *  @stable ICU 2.0
 164      */
 165     virtual ~RuleBasedBreakIterator();
 166
 167     /**
 168      * Assignment operator.  Sets this iterator to have the same behavior,
 169      * and iterate over the same text, as the one passed in.
 170      * @param that The RuleBasedBreakItertor passed in
 171      * @return the newly created RuleBasedBreakIterator
 172      *  @stable ICU 2.0
 173      */
 174     RuleBasedBreakIterator& operator=(const RuleBasedBreakIterator& that);
 175
 176     /**
 177      * Equality operator.  Returns TRUE if both BreakIterators are of the
 178      * same class, have the same behavior, and iterate over the same text.
 179      * @param that The BreakIterator to be compared for equality
 180      * @return TRUE if both BreakIterators are of the
 181      * same class, have the same behavior, and iterate over the same text.
 182      *  @stable ICU 2.0
 183      */
 184     virtual UBool operator==(const BreakIterator& that) const;
 185
 186     /**
 187      * Not-equal operator.  If operator== returns TRUE, this returns FALSE,
 188      * and vice versa.
 189      * @param that The BreakIterator to be compared for inequality
 190      * @return TRUE if both BreakIterators are not same.
 191      *  @stable ICU 2.0
 192      */
 193     UBool operator!=(const BreakIterator& that) const;
 194
 195     /**
 196      * Returns a newly-constructed RuleBasedBreakIterator with the same
 197      * behavior, and iterating over the same text, as this one.
 198      * Differs from the copy constructor in that it is polymorphic, and
 199      * will correctly clone (copy) a derived class.
 200      * clone() is thread safe.  Multiple threads may simultaeneously
 201      * clone the same source break iterator.
 202      * @return a newly-constructed RuleBasedBreakIterator
 203      * @stable ICU 2.0
 204      */
 205     virtual BreakIterator* clone() const;
 206
 207     /**
 208      * Compute a hash code for this BreakIterator
 209      * @return A hash code
 210      *  @stable ICU 2.0
 211      */
 212     virtual int32_t hashCode(void) const;
 213
 214     /**
 215      * Returns the description used to create this iterator
 216      * @return the description used to create this iterator
 217      *  @stable ICU 2.0
 218      */
 219     virtual const UnicodeString& getRules(void) const;
 220
 221     //=======================================================================
 222     // BreakIterator overrides
 223     //=======================================================================
 224
 225     /**
 226      * Return a CharacterIterator over the text being analyzed.  This version
 227      * of this method returns the actual CharacterIterator we're using internally.
 228      * Changing the state of this iterator can have undefined consequences.  If
 229      * you need to change it, clone it first.
 230      * @return An iterator over the text being analyzed.
 231      *  @stable ICU 2.0
 232      */
 233     virtual const CharacterIterator& getText(void) const;
 234
 235
 236     /**
 237      * Set the iterator to analyze a new piece of text.  This function resets
 238      * the current iteration position to the beginning of the text.
 239      * @param newText An iterator over the text to analyze.  The BreakIterator
 240      * takes ownership of the character iterator.  The caller MUST NOT delete it!
 241      *  @stable ICU 2.0
 242      */
 243     virtual void adoptText(CharacterIterator* newText);
 244
 245     /**
 246      * Set the iterator to analyze a new piece of text.  This function resets
 247      * the current iteration position to the beginning of the text.
 248      * @param newText The text to analyze.
 249      *  @stable ICU 2.0
 250      */
 251     virtual void setText(const UnicodeString& newText);
 252
 253     /**
 254      * Sets the current iteration position to the beginning of the text.
 255      * (i.e., the CharacterIterator's starting offset).
 256      * @return The offset of the beginning of the text.
 257      *  @stable ICU 2.0
 258      */
 259     virtual int32_t first(void);
 260
 261     /**
 262      * Sets the current iteration position to the end of the text.
 263      * (i.e., the CharacterIterator's ending offset).
 264      * @return The text's past-the-end offset.
 265      *  @stable ICU 2.0
 266      */
 267     virtual int32_t last(void);
 268
 269     /**
 270      * Advances the iterator either forward or backward the specified number of steps.
 271      * Negative values move backward, and positive values move forward.  This is
 272      * equivalent to repeatedly calling next() or previous().
 273      * @param n The number of steps to move.  The sign indicates the direction
 274      * (negative is backwards, and positive is forwards).
 275      * @return The character offset of the boundary position n boundaries away from
 276      * the current one.
 277      *  @stable ICU 2.0
 278      */
 279     virtual int32_t next(int32_t n);
 280
 281     /**
 282      * Advances the iterator to the next boundary position.
 283      * @return The position of the first boundary after this one.
 284      *  @stable ICU 2.0
 285      */
 286     virtual int32_t next(void);
 287
 288     /**
 289      * Moves the iterator backwards, to the last boundary preceding this one.
 290      * @return The position of the last boundary position preceding this one.
 291      *  @stable ICU 2.0
 292      */
 293     virtual int32_t previous(void);
 294
 295     /**
 296      * Sets the iterator to refer to the first boundary position following
 297      * the specified position.
 298      * @param offset The position from which to begin searching for a break position.
 299      * @return The position of the first break after the current position.
 300      *  @stable ICU 2.0
 301      */
 302     virtual int32_t following(int32_t offset);
 303
 304     /**
 305      * Sets the iterator to refer to the last boundary position before the
 306      * specified position.
 307      * @param offset The position to begin searching for a break from.
 308      * @return The position of the last boundary before the starting position.
 309      *  @stable ICU 2.0
 310      */
 311     virtual int32_t preceding(int32_t offset);
 312
 313     /**
 314      * Returns true if the specfied position is a boundary position.  As a side
 315      * effect, leaves the iterator pointing to the first boundary position at
 316      * or after "offset".
 317      * @param offset the offset to check.
 318      * @return True if "offset" is a boundary position.
 319      *  @stable ICU 2.0
 320      */
 321     virtual UBool isBoundary(int32_t offset);
 322
 323     /**
 324      * Returns the current iteration position.
 325      * @return The current iteration position.
 326      * @stable ICU 2.0
 327      */
 328     virtual int32_t current(void) const;
 329
 330
 331     /**
 332      * Return the status tag from the break rule that determined the most recently
 333      * returned break position.  For break rules that do not specify a
 334      * status, a default value of 0 is returned.  If more than one break rule
 335      * would cause a boundary to be located at some position in the text,
 336      * the numerically largest of the applicable status values is returned.
 337      * <p>
 338      * Of the standard types of ICU break iterators, only word break and
 339      * line break provide status values.  The values are defined in
 340      * the header file ubrk.h.  For Word breaks, the status allows distinguishing between words
 341      * that contain alphabetic letters, "words" that appear to be numbers,
 342      * punctuation and spaces, words containing ideographic characters, and
 343      * more.  For Line Break, the status distinguishes between hard (mandatory) breaks
 344      * and soft (potential) break positions.
 345      * <p>
 346      * <code>getRuleStatus()</code> can be called after obtaining a boundary
 347      * position from <code>next()</code>, <code>previous()</code>, or
 348      * any other break iterator functions that returns a boundary position.
 349      * <p>
 350      * When creating custom break rules, one is free to define whatever
 351      * status values may be convenient for the application.
 352      * <p>
 353      * Note: this function is not thread safe.  It should not have been
 354      *       declared const, and the const remains only for compatibility
 355      *       reasons.  (The function is logically const, but not bit-wise const).
 356      * <p>
 357      * @return the status from the break rule that determined the most recently
 358      * returned break position.
 359      *
 360      * @see UWordBreak
 361      * @stable ICU 2.2
 362      */
 363     virtual int32_t getRuleStatus() const;
 364
 365    /**
 366     * Get the status (tag) values from the break rule(s) that determined the most
 367     * recently returned break position.
 368     * <p>
 369     * The returned status value(s) are stored into an array provided by the caller.
 370     * The values are stored in sorted (ascending) order.
 371     * If the capacity of the output array is insufficient to hold the data,
 372     *  the output will be truncated to the available length, and a
 373     *  U_BUFFER_OVERFLOW_ERROR will be signaled.
 374     *
 375     * @param fillInVec an array to be filled in with the status values.
 376     * @param capacity  the length of the supplied vector.  A length of zero causes
 377     *                  the function to return the number of status values, in the
 378     *                  normal way, without attemtping to store any values.
 379     * @param status    receives error codes.
 380     * @return          The number of rule status values from rules that determined
 381     *                  the most recent boundary returned by the break iterator.
 382     *                  In the event of a U_BUFFER_OVERFLOW_ERROR, the return value
 383     *                  is the total number of status values that were available,
 384     *                  not the reduced number that were actually returned.
 385     * @see getRuleStatus
 386     * @draft ICU 3.0
 387     */
 388     virtual int32_t getRuleStatusVec(int32_t *fillInVec, int32_t capacity, UErrorCode &status);
 389
 390     /**
 391      * Returns a unique class ID POLYMORPHICALLY.  Pure virtual override.
 392      * This method is to implement a simple version of RTTI, since not all
 393      * C++ compilers support genuine RTTI.  Polymorphic operator==() and
 394      * clone() methods call this method.
 395      *
 396      * @return          The class ID for this object. All objects of a
 397      *                  given class have the same class ID.  Objects of
 398      *                  other classes have different class IDs.
 399      * @stable ICU 2.0
 400      */
 401     virtual UClassID getDynamicClassID(void) const;
 402
 403     /**
 404      * Returns the class ID for this class.  This is useful only for
 405      * comparing to a return value from getDynamicClassID().  For example:
 406      *
 407      *      Base* polymorphic_pointer = createPolymorphicObject();
 408      *      if (polymorphic_pointer->getDynamicClassID() ==
 409      *          Derived::getStaticClassID()) ...
 410      *
 411      * @return          The class ID for all objects of this class.
 412      * @stable ICU 2.0
 413      */
 414     static UClassID U_EXPORT2 getStaticClassID(void);
 415
 416     /*
 417      * Create a clone (copy) of this break iterator in memory provided
 418      *  by the caller.  The idea is to increase performance by avoiding
 419      *  a storage allocation.  Use of this functoin is NOT RECOMMENDED.
 420      *  Performance gains are minimal, and correct buffer management is
 421      *  tricky.  Use clone() instead.
 422      *
 423      * @param stackBuffer  The pointer to the memory into which the cloned object
 424      *                     should be placed.  If NULL,  allocate heap memory
 425      *                     for the cloned object.
 426      * @param BufferSize   The size of the buffer.  If zero, return the required
 427      *                     buffer size, but do not clone the object.  If the
 428      *                     size was too small (but not zero), allocate heap
 429      *                     storage for the cloned object.
 430      *
 431      * @param status       Error status.  U_SAFECLONE_ALLOCATED_WARNING will be
 432      *                     returned if the the provided buffer was too small, and
 433      *                     the clone was therefore put on the heap.
 434      *
 435      * @return  Pointer to the clone object.  This may differ from the stackBuffer
 436      *          address if the byte alignment of the stack buffer was not suitable
 437      *          or if the stackBuffer was too small to hold the clone.
 438      * @stable ICU 2.0
 439      */
 440     virtual BreakIterator *  createBufferClone(void *stackBuffer,
 441                                                int32_t &BufferSize,
 442                                                UErrorCode &status);
 443
 444
 445     /**
 446      * Return the binary form of compiled break rules,
 447      * which can then be used to create a new break iterator at some
 448      * time in the future.  Creating a break iterator from pre-compiled rules
 449      * is much faster than building one from the source form of the
 450      * break rules.
 451      *
 452      * The binary data can only be used with the same version of ICU
 453      *  and on the same platform type (processor endian-ness)
 454      *
 455      * @param length Returns the length of the binary data.  (Out paramter.)
 456      *
 457      * @return   A pointer to the binary (compiled) rule data.  The storage
 458      *           belongs to the RulesBasedBreakIterator object, not the
 459      *           caller, and must not be modified or deleted.
 460      * @internal
 461      */
 462     virtual const uint8_t *getBinaryRules(uint32_t &length);
 463
 464
 465 protected:
 466     //=======================================================================
 467     // implementation
 468     //=======================================================================
 469     /**
 470      * This method is the actual implementation of the next() method.  All iteration
 471      * vectors through here.  This method initializes the state machine to state 1
 472      * and advances through the text character by character until we reach the end
 473      * of the text or the state machine transitions to state 0.  We update our return
 474      * value every time the state machine passes through a possible end state.
 475      * @internal
 476      */
 477     virtual int32_t handleNext(void);
 478
 479     /**
 480      * This method backs the iterator back up to a "safe position" in the text.
 481      * This is a position that we know, without any context, must be a break position.
 482      * The various calling methods then iterate forward from this safe position to
 483      * the appropriate position to return.  (For more information, see the description
 484      * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
 485      * @internal
 486      */
 487     virtual int32_t handlePrevious(void);
 488
 489     /**
 490      * Dumps caches and performs other actions associated with a complete change
 491      * in text or iteration position.  This function is a no-op in RuleBasedBreakIterator,
 492      * but subclasses can and do override it.
 493      * @internal
 494      */
 495     virtual void reset(void);
 496
 497     /**
 498       * Return true if the category lookup for this char
 499       * indicates that it is in the set of dictionary lookup chars.
 500       * This function is intended for use by dictionary based break iterators.
 501       * @return true if the category lookup for this char
 502       * indicates that it is in the set of dictionary lookup chars.
 503       * @internal
 504       */
 505     virtual UBool isDictionaryChar(UChar32);
 506
 507     /**
 508       * Common initialization function, used by constructors and bufferClone.
 509       *   (Also used by DictionaryBasedBreakIterator::createBufferClone().)
 510       * @internal
 511       */
 512     void init();
 513
 514 private:
 515
 516     /**
 517      * This method backs the iterator back up to a "safe position" in the text.
 518      * This is a position that we know, without any context, must be a break position.
 519      * The various calling methods then iterate forward from this safe position to
 520      * the appropriate position to return.  (For more information, see the description
 521      * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
 522      * @param statetable state table used of moving backwards
 523      * @internal
 524      */
 525     int32_t handlePrevious(const RBBIStateTable *statetable);
 526
 527     /**
 528      * This method is the actual implementation of the next() method.  All iteration
 529      * vectors through here.  This method initializes the state machine to state 1
 530      * and advances through the text character by character until we reach the end
 531      * of the text or the state machine transitions to state 0.  We update our return
 532      * value every time the state machine passes through a possible end state.
 533      * @param statetable state table used of moving forwards
 534      * @internal
 535      */
 536     int32_t handleNext(const RBBIStateTable *statetable);
 537
 538     /**
 539      *  @internal
 540      */
 541     void makeRuleStatusValid();
 542
 543 };
 544
 545 //------------------------------------------------------------------------------
 546 //
 547 //   Inline Functions Definitions ...
 548 //
 549 //------------------------------------------------------------------------------
 550
 551 inline UBool RuleBasedBreakIterator::operator!=(const BreakIterator& that) const {
 552     return !operator==(that);
 553 }
 554
 555 U_NAMESPACE_END
 556
 557 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
 558
 559 #endif