icuSources/common/dictbe.h

   1 /**
   2  *******************************************************************************
   3  * Copyright (C) 2006-2014, International Business Machines Corporation   *
   4  * and others. All Rights Reserved.                                            *
   5  *******************************************************************************
   6  */
   7
   8 #ifndef DICTBE_H
   9 #define DICTBE_H
  10
  11 #include "unicode/utypes.h"
  12 #include "unicode/uniset.h"
  13 #include "unicode/utext.h"
  14
  15 #include "brkeng.h"
  16
  17 U_NAMESPACE_BEGIN
  18
  19 class DictionaryMatcher;
  20 class Normalizer2;
  21
  22 /*******************************************************************
  23  * DictionaryBreakEngine
  24  */
  25
  26 /**
  27  * <p>DictionaryBreakEngine is a kind of LanguageBreakEngine that uses a
  28  * dictionary to determine language-specific breaks.</p>
  29  *
  30  * <p>After it is constructed a DictionaryBreakEngine may be shared between
  31  * threads without synchronization.</p>
  32  */
  33 class DictionaryBreakEngine : public LanguageBreakEngine {
  34  private:
  35     /**
  36      * The set of characters handled by this engine
  37      * @internal
  38      */
  39
  40   UnicodeSet    fSet;
  41
  42     /**
  43      * The set of break types handled by this engine
  44      * @internal
  45      */
  46
  47   uint32_t      fTypes;
  48
  49   /**
  50    * <p>Default constructor.</p>
  51    *
  52    */
  53   DictionaryBreakEngine();
  54
  55  public:
  56
  57   /**
  58    * <p>Constructor setting the break types handled.</p>
  59    *
  60    * @param breakTypes A bitmap of types handled by the engine.
  61    */
  62   DictionaryBreakEngine( uint32_t breakTypes );
  63
  64   /**
  65    * <p>Virtual destructor.</p>
  66    */
  67   virtual ~DictionaryBreakEngine();
  68
  69   /**
  70    * <p>Indicate whether this engine handles a particular character for
  71    * a particular kind of break.</p>
  72    *
  73    * @param c A character which begins a run that the engine might handle
  74    * @param breakType The type of text break which the caller wants to determine
  75    * @return TRUE if this engine handles the particular character and break
  76    * type.
  77    */
  78   virtual UBool handles( UChar32 c, int32_t breakType ) const;
  79
  80   /**
  81    * <p>Find any breaks within a run in the supplied text.</p>
  82    *
  83    * @param text A UText representing the text. The iterator is left at
  84    * the end of the run of characters which the engine is capable of handling
  85    * that starts from the first (or last) character in the range.
  86    * @param startPos The start of the run within the supplied text.
  87    * @param endPos The end of the run within the supplied text.
  88    * @param reverse Whether the caller is looking for breaks in a reverse
  89    * direction.
  90    * @param breakType The type of break desired, or -1.
  91    * @param foundBreaks An allocated C array of the breaks found, if any
  92    * @return The number of breaks found.
  93    */
  94   virtual int32_t findBreaks( UText *text,
  95                               int32_t startPos,
  96                               int32_t endPos,
  97                               UBool reverse,
  98                               int32_t breakType,
  99                               UStack &foundBreaks ) const;
 100
 101  protected:
 102
 103  /**
 104   * <p>Set the character set handled by this engine.</p>
 105   *
 106   * @param set A UnicodeSet of the set of characters handled by the engine
 107   */
 108   virtual void setCharacters( const UnicodeSet &set );
 109
 110  /**
 111   * <p>Set the break types handled by this engine.</p>
 112   *
 113   * @param breakTypes A bitmap of types handled by the engine.
 114   */
 115 //  virtual void setBreakTypes( uint32_t breakTypes );
 116
 117  /**
 118   * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 119   *
 120   * @param text A UText representing the text
 121   * @param rangeStart The start of the range of dictionary characters
 122   * @param rangeEnd The end of the range of dictionary characters
 123   * @param foundBreaks Output of C array of int32_t break positions, or 0
 124   * @return The number of breaks found
 125   */
 126   virtual int32_t divideUpDictionaryRange( UText *text,
 127                                            int32_t rangeStart,
 128                                            int32_t rangeEnd,
 129                                            UStack &foundBreaks ) const = 0;
 130
 131 };
 132
 133 /*******************************************************************
 134  * ThaiBreakEngine
 135  */
 136
 137 /**
 138  * <p>ThaiBreakEngine is a kind of DictionaryBreakEngine that uses a
 139  * dictionary and heuristics to determine Thai-specific breaks.</p>
 140  *
 141  * <p>After it is constructed a ThaiBreakEngine may be shared between
 142  * threads without synchronization.</p>
 143  */
 144 class ThaiBreakEngine : public DictionaryBreakEngine {
 145  private:
 146     /**
 147      * The set of characters handled by this engine
 148      * @internal
 149      */
 150
 151   UnicodeSet                fThaiWordSet;
 152   UnicodeSet                fEndWordSet;
 153   UnicodeSet                fBeginWordSet;
 154   UnicodeSet                fSuffixSet;
 155   UnicodeSet                fMarkSet;
 156   DictionaryMatcher  *fDictionary;
 157
 158  public:
 159
 160   /**
 161    * <p>Default constructor.</p>
 162    *
 163    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 164    * engine is deleted.
 165    */
 166   ThaiBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 167
 168   /**
 169    * <p>Virtual destructor.</p>
 170    */
 171   virtual ~ThaiBreakEngine();
 172
 173  protected:
 174  /**
 175   * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 176   *
 177   * @param text A UText representing the text
 178   * @param rangeStart The start of the range of dictionary characters
 179   * @param rangeEnd The end of the range of dictionary characters
 180   * @param foundBreaks Output of C array of int32_t break positions, or 0
 181   * @return The number of breaks found
 182   */
 183   virtual int32_t divideUpDictionaryRange( UText *text,
 184                                            int32_t rangeStart,
 185                                            int32_t rangeEnd,
 186                                            UStack &foundBreaks ) const;
 187
 188 };
 189
 190 /*******************************************************************
 191  * LaoBreakEngine
 192  */
 193
 194 /**
 195  * <p>LaoBreakEngine is a kind of DictionaryBreakEngine that uses a
 196  * dictionary and heuristics to determine Lao-specific breaks.</p>
 197  *
 198  * <p>After it is constructed a LaoBreakEngine may be shared between
 199  * threads without synchronization.</p>
 200  */
 201 class LaoBreakEngine : public DictionaryBreakEngine {
 202  private:
 203     /**
 204      * The set of characters handled by this engine
 205      * @internal
 206      */
 207
 208   UnicodeSet                fLaoWordSet;
 209   UnicodeSet                fEndWordSet;
 210   UnicodeSet                fBeginWordSet;
 211   UnicodeSet                fMarkSet;
 212   DictionaryMatcher  *fDictionary;
 213
 214  public:
 215
 216   /**
 217    * <p>Default constructor.</p>
 218    *
 219    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 220    * engine is deleted.
 221    */
 222   LaoBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 223
 224   /**
 225    * <p>Virtual destructor.</p>
 226    */
 227   virtual ~LaoBreakEngine();
 228
 229  protected:
 230  /**
 231   * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 232   *
 233   * @param text A UText representing the text
 234   * @param rangeStart The start of the range of dictionary characters
 235   * @param rangeEnd The end of the range of dictionary characters
 236   * @param foundBreaks Output of C array of int32_t break positions, or 0
 237   * @return The number of breaks found
 238   */
 239   virtual int32_t divideUpDictionaryRange( UText *text,
 240                                            int32_t rangeStart,
 241                                            int32_t rangeEnd,
 242                                            UStack &foundBreaks ) const;
 243
 244 };
 245
 246 /*******************************************************************
 247  * BurmeseBreakEngine
 248  */
 249
 250 /**
 251  * <p>BurmeseBreakEngine is a kind of DictionaryBreakEngine that uses a
 252  * DictionaryMatcher and heuristics to determine Burmese-specific breaks.</p>
 253  *
 254  * <p>After it is constructed a BurmeseBreakEngine may be shared between
 255  * threads without synchronization.</p>
 256  */
 257 class BurmeseBreakEngine : public DictionaryBreakEngine {
 258  private:
 259     /**
 260      * The set of characters handled by this engine
 261      * @internal
 262      */
 263
 264   UnicodeSet                fBurmeseWordSet;
 265   UnicodeSet                fEndWordSet;
 266   UnicodeSet                fBeginWordSet;
 267   UnicodeSet                fMarkSet;
 268   DictionaryMatcher  *fDictionary;
 269
 270  public:
 271
 272   /**
 273    * <p>Default constructor.</p>
 274    *
 275    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 276    * engine is deleted.
 277    */
 278   BurmeseBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 279
 280   /**
 281    * <p>Virtual destructor.</p>
 282    */
 283   virtual ~BurmeseBreakEngine();
 284
 285  protected:
 286  /**
 287   * <p>Divide up a range of known dictionary characters.</p>
 288   *
 289   * @param text A UText representing the text
 290   * @param rangeStart The start of the range of dictionary characters
 291   * @param rangeEnd The end of the range of dictionary characters
 292   * @param foundBreaks Output of C array of int32_t break positions, or 0
 293   * @return The number of breaks found
 294   */
 295   virtual int32_t divideUpDictionaryRange( UText *text,
 296                                            int32_t rangeStart,
 297                                            int32_t rangeEnd,
 298                                            UStack &foundBreaks ) const;
 299
 300 };
 301
 302 /*******************************************************************
 303  * KhmerBreakEngine
 304  */
 305
 306 /**
 307  * <p>KhmerBreakEngine is a kind of DictionaryBreakEngine that uses a
 308  * DictionaryMatcher and heuristics to determine Khmer-specific breaks.</p>
 309  *
 310  * <p>After it is constructed a KhmerBreakEngine may be shared between
 311  * threads without synchronization.</p>
 312  */
 313 class KhmerBreakEngine : public DictionaryBreakEngine {
 314  private:
 315     /**
 316      * The set of characters handled by this engine
 317      * @internal
 318      */
 319
 320   UnicodeSet                fKhmerWordSet;
 321   UnicodeSet                fEndWordSet;
 322   UnicodeSet                fBeginWordSet;
 323   UnicodeSet                fMarkSet;
 324   DictionaryMatcher  *fDictionary;
 325
 326  public:
 327
 328   /**
 329    * <p>Default constructor.</p>
 330    *
 331    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 332    * engine is deleted.
 333    */
 334   KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 335
 336   /**
 337    * <p>Virtual destructor.</p>
 338    */
 339   virtual ~KhmerBreakEngine();
 340
 341  protected:
 342  /**
 343   * <p>Divide up a range of known dictionary characters.</p>
 344   *
 345   * @param text A UText representing the text
 346   * @param rangeStart The start of the range of dictionary characters
 347   * @param rangeEnd The end of the range of dictionary characters
 348   * @param foundBreaks Output of C array of int32_t break positions, or 0
 349   * @return The number of breaks found
 350   */
 351   virtual int32_t divideUpDictionaryRange( UText *text,
 352                                            int32_t rangeStart,
 353                                            int32_t rangeEnd,
 354                                            UStack &foundBreaks ) const;
 355
 356 };
 357
 358 #if !UCONFIG_NO_NORMALIZATION
 359
 360 /*******************************************************************
 361  * CjkBreakEngine
 362  */
 363
 364 //indicates language/script that the CjkBreakEngine will handle
 365 enum LanguageType {
 366     kKorean,
 367     kChineseJapanese
 368 };
 369
 370 /**
 371  * <p>CjkBreakEngine is a kind of DictionaryBreakEngine that uses a
 372  * dictionary with costs associated with each word and
 373  * Viterbi decoding to determine CJK-specific breaks.</p>
 374  */
 375 class CjkBreakEngine : public DictionaryBreakEngine {
 376  protected:
 377     /**
 378      * The set of characters handled by this engine
 379      * @internal
 380      */
 381   UnicodeSet                fHangulWordSet;
 382   UnicodeSet                fHanWordSet;
 383   UnicodeSet                fKatakanaWordSet;
 384   UnicodeSet                fHiraganaWordSet;
 385
 386   DictionaryMatcher        *fDictionary;
 387   const Normalizer2        *nfkcNorm2;
 388
 389  public:
 390
 391     /**
 392      * <p>Default constructor.</p>
 393      *
 394      * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 395      * engine is deleted. The DictionaryMatcher must contain costs for each word
 396      * in order for the dictionary to work properly.
 397      */
 398   CjkBreakEngine(DictionaryMatcher *adoptDictionary, LanguageType type, UErrorCode &status);
 399
 400     /**
 401      * <p>Virtual destructor.</p>
 402      */
 403   virtual ~CjkBreakEngine();
 404
 405  protected:
 406     /**
 407      * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 408      *
 409      * @param text A UText representing the text
 410      * @param rangeStart The start of the range of dictionary characters
 411      * @param rangeEnd The end of the range of dictionary characters
 412      * @param foundBreaks Output of C array of int32_t break positions, or 0
 413      * @return The number of breaks found
 414      */
 415   virtual int32_t divideUpDictionaryRange( UText *text,
 416           int32_t rangeStart,
 417           int32_t rangeEnd,
 418           UStack &foundBreaks ) const;
 419
 420 };
 421
 422 #endif
 423
 424 U_NAMESPACE_END
 425
 426     /* DICTBE_H */
 427 #endif