/*
***************************************************************************
-* Copyright (C) 1999-2003 International Business Machines Corporation *
+* Copyright (C) 1999-2008 International Business Machines Corporation *
* and others. All rights reserved. *
***************************************************************************
#include "unicode/utypes.h"
+/**
+ * \file
+ * \brief C++ API: Rule Based Break Iterator
+ */
+
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/brkiter.h"
#include "unicode/udata.h"
#include "unicode/parseerr.h"
+#include "unicode/schriter.h"
+#include "unicode/uchriter.h"
+
struct UTrie;
U_NAMESPACE_BEGIN
+/** @internal */
struct RBBIDataHeader;
class RuleBasedBreakIteratorTables;
class BreakIterator;
class RBBIDataWrapper;
+class UStack;
+class LanguageBreakEngine;
+class UnhandledEngine;
+struct RBBIStateTable;
+
/**
+ *
* A subclass of BreakIterator whose behavior is specified using a list of rules.
* <p>Instances of this class are most commonly created by the factory methods of
* BreakIterator::createWordInstance(), BreakIterator::createLineInstance(), etc.,
* <p>See the ICU User Guide for information on Break Iterator Rules.</p>
*
* <p>This class is not intended to be subclassed. (Class DictionaryBasedBreakIterator
- * is a subclass, but that relationship is effectively internal to the ICU
+ * is a subclass, but that relationship is effectively internal to the ICU
* implementation. The subclassing interface to RulesBasedBreakIterator is
* not part of the ICU API, and may not remain stable.</p>
*
protected:
/**
- * The character iterator through which this BreakIterator accesses the text
+ * The UText through which this BreakIterator accesses the text
* @internal
*/
- CharacterIterator* fText;
+ UText *fText;
+
+ /**
+ * A character iterator that refers to the same text as the UText, above.
+ * Only included for compatibility with old API, which was based on CharacterIterators.
+ * Value may be adopted from outside, or one of fSCharIter or fDCharIter, below.
+ */
+ CharacterIterator *fCharIter;
+
+ /**
+ * When the input text is provided by a UnicodeString, this will point to
+ * a characterIterator that wraps that data. Needed only for the
+ * implementation of getText(), a backwards compatibility issue.
+ */
+ StringCharacterIterator *fSCharIter;
+
+ /**
+ * When the input text is provided by a UText, this
+ * dummy CharacterIterator over an empty string will
+ * be returned from getText()
+ */
+ UCharCharacterIterator *fDCharIter;
/**
* The rule data for this BreakIterator instance
* @internal
*/
RBBIDataWrapper *fData;
- /** @internal */
- UTrie *fCharMappings;
- /** Rule {tag} value for the most recent match.
+ /** Index of the Rule {tag} values for the most recent match.
* @internal
*/
- int32_t fLastBreakTag;
+ int32_t fLastRuleStatusIndex;
/**
* Rule tag value valid flag.
* This flag lets us lazily compute the value if we are ever asked for it.
* @internal
*/
- UBool fLastBreakTagValid;
+ UBool fLastStatusIndexValid;
/**
* Counter for the number of characters encountered with the "dictionary"
- * flag set. Normal RBBI iterators don't use it, although the code
- * for updating it is live. Dictionary Based break iterators (a subclass
- * of us) access this field directly.
+ * flag set.
* @internal
*/
- uint32_t fDictionaryCharCount;
+ uint32_t fDictionaryCharCount;
/**
- * Debugging flag. Trace operation of state machine when true.
+ * When a range of characters is divided up using the dictionary, the break
+ * positions that are discovered are stored here, preventing us from having
+ * to use either the dictionary or the state table again until the iterator
+ * leaves this range of text. Has the most impact for line breaking.
* @internal
*/
- static UBool fTrace;
-
-
+ int32_t* fCachedBreakPositions;
-private:
/**
- * Class ID
+ * The number of elements in fCachedBreakPositions
+ * @internal
*/
- static const char fgClassID;
+ int32_t fNumCachedBreakPositions;
+ /**
+ * if fCachedBreakPositions is not null, this indicates which item in the
+ * cache the current iteration position refers to
+ * @internal
+ */
+ int32_t fPositionInCache;
+
+ /**
+ *
+ * If present, UStack of LanguageBreakEngine objects that might handle
+ * dictionary characters. Searched from top to bottom to find an object to
+ * handle a given character.
+ * @internal
+ */
+ UStack *fLanguageBreakEngines;
+
+ /**
+ *
+ * If present, the special LanguageBreakEngine used for handling
+ * characters that are in the dictionary set, but not handled by any
+ * LangugageBreakEngine.
+ * @internal
+ */
+ UnhandledEngine *fUnhandledBreakEngine;
+
+ /**
+ *
+ * The type of the break iterator, or -1 if it has not been set.
+ * @internal
+ */
+ int32_t fBreakType;
+
protected:
//=======================================================================
// constructors
//=======================================================================
/**
- * This constructor uses the udata interface to create a BreakIterator
- * whose internal tables live in a memory-mapped file. "image" is a pointer
- * to the beginning of that file.
+ * Constant to be used in the constructor
+ * RuleBasedBreakIterator(RBBIDataHeader*, EDontAdopt, UErrorCode &);
+ * which does not adopt the memory indicated by the RBBIDataHeader*
+ * parameter.
+ *
* @internal
*/
- RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status);
+ enum EDontAdopt {
+ kDontAdopt
+ };
/**
* Constructor from a flattened set of RBBI data in malloced memory.
*/
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
- friend class RBBIRuleBuilder; /** @internal */
+ /**
+ * Constructor from a flattened set of RBBI data in memory which need not
+ * be malloced (e.g. it may be a memory-mapped file, etc.).
+ *
+ * This version does not adopt the memory, and does not
+ * free it when done.
+ * @internal
+ */
+ RuleBasedBreakIterator(const RBBIDataHeader* data, enum EDontAdopt dontAdopt, UErrorCode &status);
+
+
+ friend class RBBIRuleBuilder;
+ /** @internal */
friend class BreakIterator;
/** Default constructor. Creates an empty shell of an iterator, with no
* rules or text to iterate over. Object can subsequently be assigned to.
- * @draft ICU 2.2
+ * @stable ICU 2.2
*/
RuleBasedBreakIterator();
* @param parseError In the event of a syntax error in the rules, provides the location
* within the rules of the problem.
* @param status Information on any errors encountered.
- * @draft ICU 2.2
+ * @stable ICU 2.2
*/
RuleBasedBreakIterator( const UnicodeString &rules,
UParseError &parseError,
UErrorCode &status);
+
+
+ /**
+ * This constructor uses the udata interface to create a BreakIterator
+ * whose internal tables live in a memory-mapped file. "image" is an
+ * ICU UDataMemory handle for the pre-compiled break iterator tables.
+ * @param image handle to the memory image for the break iterator data.
+ * Ownership of the UDataMemory handle passes to the Break Iterator,
+ * which will be responsible for closing it when it is no longer needed.
+ * @param status Information on any errors encountered.
+ * @see udata_open
+ * @see #getBinaryRules
+ * @stable ICU 2.8
+ */
+ RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status);
+
/**
* Destructor
* @stable ICU 2.0
* Equality operator. Returns TRUE if both BreakIterators are of the
* same class, have the same behavior, and iterate over the same text.
* @param that The BreakIterator to be compared for equality
- * @Return TRUE if both BreakIterators are of the
+ * @return TRUE if both BreakIterators are of the
* same class, have the same behavior, and iterate over the same text.
* @stable ICU 2.0
*/
* Returns a newly-constructed RuleBasedBreakIterator with the same
* behavior, and iterating over the same text, as this one.
* Differs from the copy constructor in that it is polymorphic, and
- * will correctly clone (copy) a derived class.
+ * will correctly clone (copy) a derived class.
* clone() is thread safe. Multiple threads may simultaeneously
* clone the same source break iterator.
- * @stable ICU 2.0
+ * @return a newly-constructed RuleBasedBreakIterator
+ * @stable ICU 2.0
*/
virtual BreakIterator* clone() const;
//=======================================================================
/**
- * Return a CharacterIterator over the text being analyzed. This version
- * of this method returns the actual CharacterIterator we're using internally.
- * Changing the state of this iterator can have undefined consequences. If
- * you need to change it, clone it first.
+ * <p>
+ * Return a CharacterIterator over the text being analyzed.
+ * The returned character iterator is owned by the break iterator, and must
+ * not be deleted by the caller. Repeated calls to this function may
+ * return the same CharacterIterator.
+ * </p>
+ * <p>
+ * The returned character iterator must not be used concurrently with
+ * the break iterator. If concurrent operation is needed, clone the
+ * returned character iterator first and operate on the clone.
+ * </p>
+ * <p>
+ * When the break iterator is operating on text supplied via a UText,
+ * this function will fail. Lacking any way to signal failures, it
+ * returns an CharacterIterator containing no text.
+ * The function getUText() provides similar functionality,
+ * is reliable, and is more efficient.
+ * </p>
+ *
+ * TODO: deprecate this function?
+ *
* @return An iterator over the text being analyzed.
- * @stable ICU 2.0
+ * @stable ICU 2.0
*/
- virtual const CharacterIterator& getText(void) const;
+ virtual CharacterIterator& getText(void) const;
+ /**
+ * Get a UText for the text being analyzed.
+ * The returned UText is a shallow clone of the UText used internally
+ * by the break iterator implementation. It can safely be used to
+ * access the text without impacting any break iterator operations,
+ * but the underlying text itself must not be altered.
+ *
+ * @param fillIn A UText to be filled in. If NULL, a new UText will be
+ * allocated to hold the result.
+ * @param status receives any error codes.
+ * @return The current UText for this break iterator. If an input
+ * UText was provided, it will always be returned.
+ * @stable ICU 3.4
+ */
+ virtual UText *getUText(UText *fillIn, UErrorCode &status) const;
+
/**
* Set the iterator to analyze a new piece of text. This function resets
* the current iteration position to the beginning of the text.
*/
virtual void setText(const UnicodeString& newText);
+ /**
+ * Reset the break iterator to operate over the text represented by
+ * the UText. The iterator position is reset to the start.
+ *
+ * This function makes a shallow clone of the supplied UText. This means
+ * that the caller is free to immediately close or otherwise reuse the
+ * Utext that was passed as a parameter, but that the underlying text itself
+ * must not be altered while being referenced by the break iterator.
+ *
+ * @param text The UText used to change the text.
+ * @param status Receives any error codes.
+ * @stable ICU 3.4
+ */
+ virtual void setText(UText *text, UErrorCode &status);
+
/**
* Sets the current iteration position to the beginning of the text.
- * (i.e., the CharacterIterator's starting offset).
* @return The offset of the beginning of the text.
* @stable ICU 2.0
*/
/**
* Sets the current iteration position to the end of the text.
- * (i.e., the CharacterIterator's ending offset).
* @return The text's past-the-end offset.
* @stable ICU 2.0
*/
/**
* Return the status tag from the break rule that determined the most recently
- * returned break position. The values appear in the rule source
- * within brackets, {123}, for example. For rules that do not specify a
- * status, a default value of 0 is returned.
+ * returned break position. For break rules that do not specify a
+ * status, a default value of 0 is returned. If more than one break rule
+ * would cause a boundary to be located at some position in the text,
+ * the numerically largest of the applicable status values is returned.
* <p>
- * Of the standard types of ICU break iterators, only the word break
- * iterator provides status values. The values are defined in
- * <code>enum UWordBreak</code>, and allow distinguishing between words
+ * Of the standard types of ICU break iterators, only word break and
+ * line break provide status values. The values are defined in
+ * the header file ubrk.h. For Word breaks, the status allows distinguishing between words
* that contain alphabetic letters, "words" that appear to be numbers,
* punctuation and spaces, words containing ideographic characters, and
- * more. Call <code>getRuleStatus</code> after obtaining a boundary
- * position from <code>next()<code>, <code>previous()</code>, or
+ * more. For Line Break, the status distinguishes between hard (mandatory) breaks
+ * and soft (potential) break positions.
+ * <p>
+ * <code>getRuleStatus()</code> can be called after obtaining a boundary
+ * position from <code>next()</code>, <code>previous()</code>, or
* any other break iterator functions that returns a boundary position.
* <p>
+ * When creating custom break rules, one is free to define whatever
+ * status values may be convenient for the application.
+ * <p>
+ * Note: this function is not thread safe. It should not have been
+ * declared const, and the const remains only for compatibility
+ * reasons. (The function is logically const, but not bit-wise const).
+ * <p>
* @return the status from the break rule that determined the most recently
* returned break position.
*
* @see UWordBreak
- * @draft ICU 2.2
+ * @stable ICU 2.2
*/
virtual int32_t getRuleStatus() const;
+ /**
+ * Get the status (tag) values from the break rule(s) that determined the most
+ * recently returned break position.
+ * <p>
+ * The returned status value(s) are stored into an array provided by the caller.
+ * The values are stored in sorted (ascending) order.
+ * If the capacity of the output array is insufficient to hold the data,
+ * the output will be truncated to the available length, and a
+ * U_BUFFER_OVERFLOW_ERROR will be signaled.
+ *
+ * @param fillInVec an array to be filled in with the status values.
+ * @param capacity the length of the supplied vector. A length of zero causes
+ * the function to return the number of status values, in the
+ * normal way, without attemtping to store any values.
+ * @param status receives error codes.
+ * @return The number of rule status values from rules that determined
+ * the most recent boundary returned by the break iterator.
+ * In the event of a U_BUFFER_OVERFLOW_ERROR, the return value
+ * is the total number of status values that were available,
+ * not the reduced number that were actually returned.
+ * @see getRuleStatus
+ * @stable ICU 3.0
+ */
+ virtual int32_t getRuleStatusVec(int32_t *fillInVec, int32_t capacity, UErrorCode &status);
+
/**
* Returns a unique class ID POLYMORPHICALLY. Pure virtual override.
* This method is to implement a simple version of RTTI, since not all
* other classes have different class IDs.
* @stable ICU 2.0
*/
- inline virtual UClassID getDynamicClassID(void) const;
+ virtual UClassID getDynamicClassID(void) const;
/**
* Returns the class ID for this class. This is useful only for
* @return The class ID for all objects of this class.
* @stable ICU 2.0
*/
- inline static UClassID getStaticClassID(void);
+ static UClassID U_EXPORT2 getStaticClassID(void);
/*
* Create a clone (copy) of this break iterator in memory provided
* is much faster than building one from the source form of the
* break rules.
*
- * The binary data is can only be used with the same version of ICU
+ * The binary data can only be used with the same version of ICU
* and on the same platform type (processor endian-ness)
*
* @param length Returns the length of the binary data. (Out paramter.)
//=======================================================================
// implementation
//=======================================================================
- /**
- * This method is the actual implementation of the next() method. All iteration
- * vectors through here. This method initializes the state machine to state 1
- * and advances through the text character by character until we reach the end
- * of the text or the state machine transitions to state 0. We update our return
- * value every time the state machine passes through a possible end state.
- * @internal
- */
- virtual int32_t handleNext(void);
-
- /**
- * This method backs the iterator back up to a "safe position" in the text.
- * This is a position that we know, without any context, must be a break position.
- * The various calling methods then iterate forward from this safe position to
- * the appropriate position to return. (For more information, see the description
- * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
- * @internal
- */
- virtual int32_t handlePrevious(void);
-
/**
* Dumps caches and performs other actions associated with a complete change
- * in text or iteration position. This function is a no-op in RuleBasedBreakIterator,
- * but subclasses can and do override it.
+ * in text or iteration position.
* @internal
*/
virtual void reset(void);
+#if 0
/**
* Return true if the category lookup for this char
* indicates that it is in the set of dictionary lookup chars.
*/
virtual UBool isDictionaryChar(UChar32);
+ /**
+ * Get the type of the break iterator.
+ * @internal
+ */
+ virtual int32_t getBreakType() const;
+#endif
+
+ /**
+ * Set the type of the break iterator.
+ * @internal
+ */
+ virtual void setBreakType(int32_t type);
+
/**
* Common initialization function, used by constructors and bufferClone.
* (Also used by DictionaryBasedBreakIterator::createBufferClone().)
*/
void init();
+private:
+
+ /**
+ * This method backs the iterator back up to a "safe position" in the text.
+ * This is a position that we know, without any context, must be a break position.
+ * The various calling methods then iterate forward from this safe position to
+ * the appropriate position to return. (For more information, see the description
+ * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
+ * @param statetable state table used of moving backwards
+ * @internal
+ */
+ int32_t handlePrevious(const RBBIStateTable *statetable);
+
+ /**
+ * This method is the actual implementation of the next() method. All iteration
+ * vectors through here. This method initializes the state machine to state 1
+ * and advances through the text character by character until we reach the end
+ * of the text or the state machine transitions to state 0. We update our return
+ * value every time the state machine passes through a possible end state.
+ * @param statetable state table used of moving forwards
+ * @internal
+ */
+ int32_t handleNext(const RBBIStateTable *statetable);
+
+protected:
+
+ /**
+ * This is the function that actually implements dictionary-based
+ * breaking. Covering at least the range from startPos to endPos,
+ * it checks for dictionary characters, and if it finds them determines
+ * the appropriate object to deal with them. It may cache found breaks in
+ * fCachedBreakPositions as it goes. It may well also look at text outside
+ * the range startPos to endPos.
+ * If going forward, endPos is the normal Unicode break result, and
+ * if goind in reverse, startPos is the normal Unicode break result
+ * @param startPos The start position of a range of text
+ * @param endPos The end position of a range of text
+ * @param reverse The call is for the reverse direction
+ * @internal
+ */
+ int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse);
+
+private:
+
+ /**
+ * This function returns the appropriate LanguageBreakEngine for a
+ * given character c.
+ * @param c A character in the dictionary set
+ * @internal
+ */
+ const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
+
+ /**
+ * @internal
+ */
+ void makeRuleStatusValid();
+
};
-//----------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Inline Functions Definitions ...
//
-//----------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
inline UBool RuleBasedBreakIterator::operator!=(const BreakIterator& that) const {
return !operator==(that);
}
-inline UClassID RuleBasedBreakIterator::getStaticClassID(void) {
- return (UClassID)(&fgClassID);
-}
-
-inline UClassID RuleBasedBreakIterator::getDynamicClassID(void) const {
- return RuleBasedBreakIterator::getStaticClassID();
-}
-
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
projects / apple / icu.git / blobdiff