/*
***************************************************************************
-* Copyright (C) 1999-2004 International Business Machines Corporation *
+* Copyright (C) 1999-2013 International Business Machines Corporation *
* and others. All rights reserved. *
***************************************************************************
#include "unicode/utypes.h"
+/**
+ * \file
+ * \brief C++ API: Rule Based Break Iterator
+ */
+
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/brkiter.h"
#include "unicode/udata.h"
#include "unicode/parseerr.h"
+#include "unicode/schriter.h"
+#include "unicode/uchriter.h"
+
struct UTrie;
class RuleBasedBreakIteratorTables;
class BreakIterator;
class RBBIDataWrapper;
+class UStack;
+class LanguageBreakEngine;
+class UnhandledEngine;
struct RBBIStateTable;
+
/**
+ *
* A subclass of BreakIterator whose behavior is specified using a list of rules.
* <p>Instances of this class are most commonly created by the factory methods of
* BreakIterator::createWordInstance(), BreakIterator::createLineInstance(), etc.,
protected:
/**
- * The character iterator through which this BreakIterator accesses the text
+ * The UText through which this BreakIterator accesses the text
* @internal
*/
- CharacterIterator* fText;
+ UText *fText;
+
+ /**
+ * A character iterator that refers to the same text as the UText, above.
+ * Only included for compatibility with old API, which was based on CharacterIterators.
+ * Value may be adopted from outside, or one of fSCharIter or fDCharIter, below.
+ */
+ CharacterIterator *fCharIter;
+
+ /**
+ * When the input text is provided by a UnicodeString, this will point to
+ * a characterIterator that wraps that data. Needed only for the
+ * implementation of getText(), a backwards compatibility issue.
+ */
+ StringCharacterIterator *fSCharIter;
+
+ /**
+ * When the input text is provided by a UText, this
+ * dummy CharacterIterator over an empty string will
+ * be returned from getText()
+ */
+ UCharCharacterIterator *fDCharIter;
/**
* The rule data for this BreakIterator instance
/**
* Counter for the number of characters encountered with the "dictionary"
- * flag set. Normal RBBI iterators don't use it, although the code
- * for updating it is live. Dictionary Based break iterators (a subclass
- * of us) access this field directly.
+ * flag set.
* @internal
*/
- uint32_t fDictionaryCharCount;
+ uint32_t fDictionaryCharCount;
/**
- * Debugging flag. Trace operation of state machine when true.
+ * When a range of characters is divided up using the dictionary, the break
+ * positions that are discovered are stored here, preventing us from having
+ * to use either the dictionary or the state table again until the iterator
+ * leaves this range of text. Has the most impact for line breaking.
* @internal
*/
- static UBool fTrace;
+ int32_t* fCachedBreakPositions;
+ /**
+ * The number of elements in fCachedBreakPositions
+ * @internal
+ */
+ int32_t fNumCachedBreakPositions;
+ /**
+ * if fCachedBreakPositions is not null, this indicates which item in the
+ * cache the current iteration position refers to
+ * @internal
+ */
+ int32_t fPositionInCache;
+
+ /**
+ *
+ * If present, UStack of LanguageBreakEngine objects that might handle
+ * dictionary characters. Searched from top to bottom to find an object to
+ * handle a given character.
+ * @internal
+ */
+ UStack *fLanguageBreakEngines;
+
+ /**
+ *
+ * If present, the special LanguageBreakEngine used for handling
+ * characters that are in the dictionary set, but not handled by any
+ * LangugageBreakEngine.
+ * @internal
+ */
+ UnhandledEngine *fUnhandledBreakEngine;
+
+ /**
+ *
+ * The type of the break iterator, or -1 if it has not been set.
+ * @internal
+ */
+ int32_t fBreakType;
+
protected:
//=======================================================================
// constructors
//=======================================================================
+#ifndef U_HIDE_INTERNAL_API
+ /**
+ * Constant to be used in the constructor
+ * RuleBasedBreakIterator(RBBIDataHeader*, EDontAdopt, UErrorCode &);
+ * which does not adopt the memory indicated by the RBBIDataHeader*
+ * parameter.
+ *
+ * @internal
+ */
+ enum EDontAdopt {
+ kDontAdopt
+ };
+
/**
* Constructor from a flattened set of RBBI data in malloced memory.
* RulesBasedBreakIterators built from a custom set of rules
*/
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
- friend class RBBIRuleBuilder; /** @internal */
+ /**
+ * Constructor from a flattened set of RBBI data in memory which need not
+ * be malloced (e.g. it may be a memory-mapped file, etc.).
+ *
+ * This version does not adopt the memory, and does not
+ * free it when done.
+ * @internal
+ */
+ RuleBasedBreakIterator(const RBBIDataHeader* data, enum EDontAdopt dontAdopt, UErrorCode &status);
+#endif /* U_HIDE_INTERNAL_API */
+
+
+ friend class RBBIRuleBuilder;
+ /** @internal */
friend class BreakIterator;
UParseError &parseError,
UErrorCode &status);
+ /**
+ * Contruct a RuleBasedBreakIterator from a set of precompiled binary rules.
+ * Binary rules are obtained from RulesBasedBreakIterator::getBinaryRules().
+ * Construction of a break iterator in this way is substantially faster than
+ * constuction from source rules.
+ *
+ * Ownership of the storage containing the compiled rules remains with the
+ * caller of this function. The compiled rules must not be modified or
+ * deleted during the life of the break iterator.
+ *
+ * The compiled rules are not compatible across different major versions of ICU.
+ * The compiled rules are comaptible only between machines with the same
+ * byte ordering (little or big endian) and the same base character set family
+ * (ASCII or EBCDIC).
+ *
+ * @see #getBinaryRules
+ * @param compiledRules A pointer to the compiled break rules to be used.
+ * @param ruleLength The length of the compiled break rules, in bytes. This
+ * corresponds to the length value produced by getBinaryRules().
+ * @param status Information on any errors encountered, including invalid
+ * binary rules.
+ * @stable ICU 4.8
+ */
+ RuleBasedBreakIterator(const uint8_t *compiledRules,
+ uint32_t ruleLength,
+ UErrorCode &status);
/**
* This constructor uses the udata interface to create a BreakIterator
* @param status Information on any errors encountered.
* @see udata_open
* @see #getBinaryRules
- * @draft ICU 2.8
+ * @stable ICU 2.8
*/
RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status);
//=======================================================================
/**
- * Return a CharacterIterator over the text being analyzed. This version
- * of this method returns the actual CharacterIterator we're using internally.
- * Changing the state of this iterator can have undefined consequences. If
- * you need to change it, clone it first.
+ * <p>
+ * Return a CharacterIterator over the text being analyzed.
+ * The returned character iterator is owned by the break iterator, and must
+ * not be deleted by the caller. Repeated calls to this function may
+ * return the same CharacterIterator.
+ * </p>
+ * <p>
+ * The returned character iterator must not be used concurrently with
+ * the break iterator. If concurrent operation is needed, clone the
+ * returned character iterator first and operate on the clone.
+ * </p>
+ * <p>
+ * When the break iterator is operating on text supplied via a UText,
+ * this function will fail. Lacking any way to signal failures, it
+ * returns an CharacterIterator containing no text.
+ * The function getUText() provides similar functionality,
+ * is reliable, and is more efficient.
+ * </p>
+ *
+ * TODO: deprecate this function?
+ *
* @return An iterator over the text being analyzed.
- * @stable ICU 2.0
+ * @stable ICU 2.0
*/
- virtual const CharacterIterator& getText(void) const;
+ virtual CharacterIterator& getText(void) const;
+ /**
+ * Get a UText for the text being analyzed.
+ * The returned UText is a shallow clone of the UText used internally
+ * by the break iterator implementation. It can safely be used to
+ * access the text without impacting any break iterator operations,
+ * but the underlying text itself must not be altered.
+ *
+ * @param fillIn A UText to be filled in. If NULL, a new UText will be
+ * allocated to hold the result.
+ * @param status receives any error codes.
+ * @return The current UText for this break iterator. If an input
+ * UText was provided, it will always be returned.
+ * @stable ICU 3.4
+ */
+ virtual UText *getUText(UText *fillIn, UErrorCode &status) const;
+
/**
* Set the iterator to analyze a new piece of text. This function resets
* the current iteration position to the beginning of the text.
*/
virtual void setText(const UnicodeString& newText);
+ /**
+ * Reset the break iterator to operate over the text represented by
+ * the UText. The iterator position is reset to the start.
+ *
+ * This function makes a shallow clone of the supplied UText. This means
+ * that the caller is free to immediately close or otherwise reuse the
+ * Utext that was passed as a parameter, but that the underlying text itself
+ * must not be altered while being referenced by the break iterator.
+ *
+ * @param text The UText used to change the text.
+ * @param status Receives any error codes.
+ * @stable ICU 3.4
+ */
+ virtual void setText(UText *text, UErrorCode &status);
+
/**
* Sets the current iteration position to the beginning of the text.
- * (i.e., the CharacterIterator's starting offset).
* @return The offset of the beginning of the text.
* @stable ICU 2.0
*/
/**
* Sets the current iteration position to the end of the text.
- * (i.e., the CharacterIterator's ending offset).
* @return The text's past-the-end offset.
* @stable ICU 2.0
*/
* is the total number of status values that were available,
* not the reduced number that were actually returned.
* @see getRuleStatus
- * @draft ICU 3.0
+ * @stable ICU 3.0
*/
virtual int32_t getRuleStatusVec(int32_t *fillInVec, int32_t capacity, UErrorCode &status);
*/
static UClassID U_EXPORT2 getStaticClassID(void);
- /*
+ /**
+ * Deprecated functionality. Use clone() instead.
+ *
* Create a clone (copy) of this break iterator in memory provided
* by the caller. The idea is to increase performance by avoiding
* a storage allocation. Use of this functoin is NOT RECOMMENDED.
* @return Pointer to the clone object. This may differ from the stackBuffer
* address if the byte alignment of the stack buffer was not suitable
* or if the stackBuffer was too small to hold the clone.
- * @stable ICU 2.0
+ * @deprecated ICU 52. Use clone() instead.
*/
virtual BreakIterator * createBufferClone(void *stackBuffer,
int32_t &BufferSize,
* @return A pointer to the binary (compiled) rule data. The storage
* belongs to the RulesBasedBreakIterator object, not the
* caller, and must not be modified or deleted.
- * @internal
+ * @stable ICU 4.8
*/
virtual const uint8_t *getBinaryRules(uint32_t &length);
+ /**
+ * Set the subject text string upon which the break iterator is operating
+ * without changing any other aspect of the matching state.
+ * The new and previous text strings must have the same content.
+ *
+ * This function is intended for use in environments where ICU is operating on
+ * strings that may move around in memory. It provides a mechanism for notifying
+ * ICU that the string has been relocated, and providing a new UText to access the
+ * string in its new position.
+ *
+ * Note that the break iterator implementation never copies the underlying text
+ * of a string being processed, but always operates directly on the original text
+ * provided by the user. Refreshing simply drops the references to the old text
+ * and replaces them with references to the new.
+ *
+ * Caution: this function is normally used only by very specialized,
+ * system-level code. One example use case is with garbage collection that moves
+ * the text in memory.
+ *
+ * @param input The new (moved) text string.
+ * @param status Receives errors detected by this function.
+ * @return *this
+ *
+ * @stable ICU 49
+ */
+ virtual RuleBasedBreakIterator &refreshInputText(UText *input, UErrorCode &status);
+
protected:
//=======================================================================
// implementation
//=======================================================================
- /**
- * This method is the actual implementation of the next() method. All iteration
- * vectors through here. This method initializes the state machine to state 1
- * and advances through the text character by character until we reach the end
- * of the text or the state machine transitions to state 0. We update our return
- * value every time the state machine passes through a possible end state.
- * @internal
- */
- virtual int32_t handleNext(void);
-
- /**
- * This method backs the iterator back up to a "safe position" in the text.
- * This is a position that we know, without any context, must be a break position.
- * The various calling methods then iterate forward from this safe position to
- * the appropriate position to return. (For more information, see the description
- * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
- * @internal
- */
- virtual int32_t handlePrevious(void);
-
/**
* Dumps caches and performs other actions associated with a complete change
- * in text or iteration position. This function is a no-op in RuleBasedBreakIterator,
- * but subclasses can and do override it.
+ * in text or iteration position.
* @internal
*/
virtual void reset(void);
+#if 0
/**
* Return true if the category lookup for this char
* indicates that it is in the set of dictionary lookup chars.
*/
virtual UBool isDictionaryChar(UChar32);
+ /**
+ * Get the type of the break iterator.
+ * @internal
+ */
+ virtual int32_t getBreakType() const;
+#endif
+
+ /**
+ * Set the type of the break iterator.
+ * @internal
+ */
+ virtual void setBreakType(int32_t type);
+
+#ifndef U_HIDE_INTERNAL_API
/**
* Common initialization function, used by constructors and bufferClone.
- * (Also used by DictionaryBasedBreakIterator::createBufferClone().)
* @internal
*/
void init();
+#endif /* U_HIDE_INTERNAL_API */
private:
*/
int32_t handleNext(const RBBIStateTable *statetable);
+protected:
+
+#ifndef U_HIDE_INTERNAL_API
+ /**
+ * This is the function that actually implements dictionary-based
+ * breaking. Covering at least the range from startPos to endPos,
+ * it checks for dictionary characters, and if it finds them determines
+ * the appropriate object to deal with them. It may cache found breaks in
+ * fCachedBreakPositions as it goes. It may well also look at text outside
+ * the range startPos to endPos.
+ * If going forward, endPos is the normal Unicode break result, and
+ * if goind in reverse, startPos is the normal Unicode break result
+ * @param startPos The start position of a range of text
+ * @param endPos The end position of a range of text
+ * @param reverse The call is for the reverse direction
+ * @internal
+ */
+ int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse);
+#endif /* U_HIDE_INTERNAL_API */
+
+private:
+
+ /**
+ * This function returns the appropriate LanguageBreakEngine for a
+ * given character c.
+ * @param c A character in the dictionary set
+ * @internal
+ */
+ const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
+
/**
* @internal
*/
projects / apple / icu.git / blobdiff