/*
***************************************************************************
-* Copyright (C) 1999-2004 International Business Machines Corporation *
+* Copyright (C) 1999-2006 International Business Machines Corporation *
* and others. All rights reserved. *
***************************************************************************
#include "unicode/utypes.h"
+/**
+ * \file
+ * \brief C++ API: Rule Based Break Iterator
+ */
+
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/brkiter.h"
#include "unicode/udata.h"
#include "unicode/parseerr.h"
+#include "unicode/schriter.h"
+#include "unicode/uchriter.h"
+
struct UTrie;
class RuleBasedBreakIteratorTables;
class BreakIterator;
class RBBIDataWrapper;
+class UStack;
+class LanguageBreakEngine;
+class UnhandledEngine;
struct RBBIStateTable;
+
/**
+ *
* A subclass of BreakIterator whose behavior is specified using a list of rules.
* <p>Instances of this class are most commonly created by the factory methods of
* BreakIterator::createWordInstance(), BreakIterator::createLineInstance(), etc.,
protected:
/**
- * The character iterator through which this BreakIterator accesses the text
+ * The UText through which this BreakIterator accesses the text
* @internal
*/
- CharacterIterator* fText;
+ UText *fText;
+
+ /**
+ * A character iterator that refers to the same text as the UText, above.
+ * Only included for compatibility with old API, which was based on CharacterIterators.
+ * Value may be adopted from outside, or one of fSCharIter or fDCharIter, below.
+ */
+ CharacterIterator *fCharIter;
+
+ /**
+ * When the input text is provided by a UnicodeString, this will point to
+ * a characterIterator that wraps that data. Needed only for the
+ * implementation of getText(), a backwards compatibility issue.
+ */
+ StringCharacterIterator *fSCharIter;
+
+ /**
+ * When the input text is provided by a UText, this
+ * dummy CharacterIterator over an empty string will
+ * be returned from getText()
+ */
+ UCharCharacterIterator *fDCharIter;
/**
* The rule data for this BreakIterator instance
/**
* Counter for the number of characters encountered with the "dictionary"
- * flag set. Normal RBBI iterators don't use it, although the code
- * for updating it is live. Dictionary Based break iterators (a subclass
- * of us) access this field directly.
+ * flag set.
* @internal
*/
- uint32_t fDictionaryCharCount;
+ uint32_t fDictionaryCharCount;
/**
- * Debugging flag. Trace operation of state machine when true.
+ * When a range of characters is divided up using the dictionary, the break
+ * positions that are discovered are stored here, preventing us from having
+ * to use either the dictionary or the state table again until the iterator
+ * leaves this range of text. Has the most impact for line breaking.
* @internal
*/
- static UBool fTrace;
+ int32_t* fCachedBreakPositions;
+ /**
+ * The number of elements in fCachedBreakPositions
+ * @internal
+ */
+ int32_t fNumCachedBreakPositions;
+ /**
+ * if fCachedBreakPositions is not null, this indicates which item in the
+ * cache the current iteration position refers to
+ * @internal
+ */
+ int32_t fPositionInCache;
+
+ /**
+ *
+ * If present, UStack of LanguageBreakEngine objects that might handle
+ * dictionary characters. Searched from top to bottom to find an object to
+ * handle a given character.
+ * @internal
+ */
+ UStack *fLanguageBreakEngines;
+
+ /**
+ *
+ * If present, the special LanguageBreakEngine used for handling
+ * characters that are in the dictionary set, but not handled by any
+ * LangugageBreakEngine.
+ * @internal
+ */
+ UnhandledEngine *fUnhandledBreakEngine;
+
+ /**
+ *
+ * The type of the break iterator, or -1 if it has not been set.
+ * @internal
+ */
+ int32_t fBreakType;
+
protected:
//=======================================================================
// constructors
*/
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
- friend class RBBIRuleBuilder; /** @internal */
+
+ friend class RBBIRuleBuilder;
+ /** @internal */
friend class BreakIterator;
* @param status Information on any errors encountered.
* @see udata_open
* @see #getBinaryRules
- * @draft ICU 2.8
+ * @stable ICU 2.8
*/
RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status);
//=======================================================================
/**
- * Return a CharacterIterator over the text being analyzed. This version
- * of this method returns the actual CharacterIterator we're using internally.
- * Changing the state of this iterator can have undefined consequences. If
- * you need to change it, clone it first.
+ * <p>
+ * Return a CharacterIterator over the text being analyzed.
+ * The returned character iterator is owned by the break iterator, and must
+ * not be deleted by the caller. Repeated calls to this function may
+ * return the same CharacterIterator.
+ * </p>
+ * <p>
+ * The returned character iterator must not be used concurrently with
+ * the break iterator. If concurrent operation is needed, clone the
+ * returned character iterator first and operate on the clone.
+ * </p>
+ * <p>
+ * When the break iterator is operating on text supplied via a UText,
+ * this function will fail. Lacking any way to signal failures, it
+ * returns an CharacterIterator containing no text.
+ * The function getUText() provides similar functionality,
+ * is reliable, and is more efficient.
+ * </p>
+ *
+ * TODO: deprecate this function?
+ *
* @return An iterator over the text being analyzed.
- * @stable ICU 2.0
+ * @stable ICU 2.0
*/
- virtual const CharacterIterator& getText(void) const;
+ virtual CharacterIterator& getText(void) const;
+
+ /**
+ * Get a UText for the text being analyzed.
+ * The returned UText is a shallow clone of the UText used internally
+ * by the break iterator implementation. It can safely be used to
+ * access the text without impacting any break iterator operations,
+ * but the underlying text itself must not be altered.
+ *
+ * @param fillIn A UText to be filled in. If NULL, a new UText will be
+ * allocated to hold the result.
+ * @param status receives any error codes.
+ * @return The current UText for this break iterator. If an input
+ * UText was provided, it will always be returned.
+ * @draft ICU 3.4
+ */
+ virtual UText *getUText(UText *fillIn, UErrorCode &status) const;
/**
* Set the iterator to analyze a new piece of text. This function resets
*/
virtual void setText(const UnicodeString& newText);
+ /**
+ * Reset the break iterator to operate over the text represented by
+ * the UText. The iterator position is reset to the start.
+ *
+ * This function makes a shallow clone of the supplied UText. This means
+ * that the caller is free to immediately close or otherwise reuse the
+ * Utext that was passed as a parameter, but that the underlying text itself
+ * must not be altered while being referenced by the break iterator.
+ *
+ * @param text The UText used to change the text.
+ * @param status Receives any error codes.
+ * @draft ICU 3.4
+ */
+ virtual void setText(UText *text, UErrorCode &status);
+
/**
* Sets the current iteration position to the beginning of the text.
- * (i.e., the CharacterIterator's starting offset).
* @return The offset of the beginning of the text.
* @stable ICU 2.0
*/
/**
* Sets the current iteration position to the end of the text.
- * (i.e., the CharacterIterator's ending offset).
* @return The text's past-the-end offset.
* @stable ICU 2.0
*/
* is the total number of status values that were available,
* not the reduced number that were actually returned.
* @see getRuleStatus
- * @draft ICU 3.0
+ * @stable ICU 3.0
*/
virtual int32_t getRuleStatusVec(int32_t *fillInVec, int32_t capacity, UErrorCode &status);
//=======================================================================
// implementation
//=======================================================================
- /**
- * This method is the actual implementation of the next() method. All iteration
- * vectors through here. This method initializes the state machine to state 1
- * and advances through the text character by character until we reach the end
- * of the text or the state machine transitions to state 0. We update our return
- * value every time the state machine passes through a possible end state.
- * @internal
- */
- virtual int32_t handleNext(void);
-
- /**
- * This method backs the iterator back up to a "safe position" in the text.
- * This is a position that we know, without any context, must be a break position.
- * The various calling methods then iterate forward from this safe position to
- * the appropriate position to return. (For more information, see the description
- * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
- * @internal
- */
- virtual int32_t handlePrevious(void);
-
/**
* Dumps caches and performs other actions associated with a complete change
- * in text or iteration position. This function is a no-op in RuleBasedBreakIterator,
- * but subclasses can and do override it.
+ * in text or iteration position.
* @internal
*/
virtual void reset(void);
+#if 0
/**
* Return true if the category lookup for this char
* indicates that it is in the set of dictionary lookup chars.
*/
virtual UBool isDictionaryChar(UChar32);
+ /**
+ * Get the type of the break iterator.
+ * @internal
+ */
+ virtual int32_t getBreakType() const;
+#endif
+
+ /**
+ * Set the type of the break iterator.
+ * @internal
+ */
+ virtual void setBreakType(int32_t type);
+
/**
* Common initialization function, used by constructors and bufferClone.
* (Also used by DictionaryBasedBreakIterator::createBufferClone().)
*/
int32_t handleNext(const RBBIStateTable *statetable);
+protected:
+
+ /**
+ * This is the function that actually implements dictionary-based
+ * breaking. Covering at least the range from startPos to endPos,
+ * it checks for dictionary characters, and if it finds them determines
+ * the appropriate object to deal with them. It may cache found breaks in
+ * fCachedBreakPositions as it goes. It may well also look at text outside
+ * the range startPos to endPos.
+ * If going forward, endPos is the normal Unicode break result, and
+ * if goind in reverse, startPos is the normal Unicode break result
+ * @param startPos The start position of a range of text
+ * @param endPos The end position of a range of text
+ * @param reverse The call is for the reverse direction
+ * @internal
+ */
+ int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse);
+
+private:
+
+ /**
+ * This function returns the appropriate LanguageBreakEngine for a
+ * given character c.
+ * @param c A character in the dictionary set
+ * @internal
+ */
+ const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
+
/**
* @internal
*/
projects / apple / icu.git / blobdiff