icuSources/common/dictbe.h

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /**
   4  *******************************************************************************
   5  * Copyright (C) 2006-2014, International Business Machines Corporation   *
   6  * and others. All Rights Reserved.                                            *
   7  *******************************************************************************
   8  */
   9
  10 #ifndef DICTBE_H
  11 #define DICTBE_H
  12
  13 #include "unicode/utypes.h"
  14 #include "unicode/uniset.h"
  15 #include "unicode/utext.h"
  16
  17 #include "brkeng.h"
  18 #include "uvectr32.h"
  19
  20 U_NAMESPACE_BEGIN
  21
  22 class DictionaryMatcher;
  23 class Normalizer2;
  24
  25 /*******************************************************************
  26  * DictionaryBreakEngine
  27  */
  28
  29 /**
  30  * <p>DictionaryBreakEngine is a kind of LanguageBreakEngine that uses a
  31  * dictionary to determine language-specific breaks.</p>
  32  *
  33  * <p>After it is constructed a DictionaryBreakEngine may be shared between
  34  * threads without synchronization.</p>
  35  */
  36 class DictionaryBreakEngine : public LanguageBreakEngine {
  37  private:
  38     /**
  39      * The set of characters handled by this engine
  40      * @internal
  41      */
  42
  43   UnicodeSet    fSet;
  44
  45  public:
  46
  47   /**
  48    * <p>Constructor </p>
  49    */
  50   DictionaryBreakEngine();
  51
  52   /**
  53    * <p>Virtual destructor.</p>
  54    */
  55   virtual ~DictionaryBreakEngine();
  56
  57   /**
  58    * <p>Indicate whether this engine handles a particular character for
  59    * a particular kind of break.</p>
  60    *
  61    * @param c A character which begins a run that the engine might handle
  62    * @return TRUE if this engine handles the particular character and break
  63    * type.
  64    */
  65   virtual UBool handles(UChar32 c) const;
  66
  67   /**
  68    * <p>Find any breaks within a run in the supplied text.</p>
  69    *
  70    * @param text A UText representing the text. The iterator is left at
  71    * the end of the run of characters which the engine is capable of handling
  72    * that starts from the first character in the range.
  73    * @param startPos The start of the run within the supplied text.
  74    * @param endPos The end of the run within the supplied text.
  75    * @param foundBreaks vector of int32_t to receive the break positions
  76    * @return The number of breaks found.
  77    */
  78   virtual int32_t findBreaks( UText *text,
  79                               int32_t startPos,
  80                               int32_t endPos,
  81                               UVector32 &foundBreaks ) const;
  82
  83  protected:
  84
  85  /**
  86   * <p>Set the character set handled by this engine.</p>
  87   *
  88   * @param set A UnicodeSet of the set of characters handled by the engine
  89   */
  90   virtual void setCharacters( const UnicodeSet &set );
  91
  92  /**
  93   * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
  94   *
  95   * @param text A UText representing the text
  96   * @param rangeStart The start of the range of dictionary characters
  97   * @param rangeEnd The end of the range of dictionary characters
  98   * @param foundBreaks Output of C array of int32_t break positions, or 0
  99   * @return The number of breaks found
 100   */
 101   virtual int32_t divideUpDictionaryRange( UText *text,
 102                                            int32_t rangeStart,
 103                                            int32_t rangeEnd,
 104                                            UVector32 &foundBreaks ) const = 0;
 105
 106 };
 107
 108 /*******************************************************************
 109  * ThaiBreakEngine
 110  */
 111
 112 /**
 113  * <p>ThaiBreakEngine is a kind of DictionaryBreakEngine that uses a
 114  * dictionary and heuristics to determine Thai-specific breaks.</p>
 115  *
 116  * <p>After it is constructed a ThaiBreakEngine may be shared between
 117  * threads without synchronization.</p>
 118  */
 119 class ThaiBreakEngine : public DictionaryBreakEngine {
 120  private:
 121     /**
 122      * The set of characters handled by this engine
 123      * @internal
 124      */
 125
 126   UnicodeSet                fThaiWordSet;
 127   UnicodeSet                fEndWordSet;
 128   UnicodeSet                fBeginWordSet;
 129   UnicodeSet                fSuffixSet;
 130   UnicodeSet                fMarkSet;
 131   DictionaryMatcher  *fDictionary;
 132
 133  public:
 134
 135   /**
 136    * <p>Default constructor.</p>
 137    *
 138    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 139    * engine is deleted.
 140    */
 141   ThaiBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 142
 143   /**
 144    * <p>Virtual destructor.</p>
 145    */
 146   virtual ~ThaiBreakEngine();
 147
 148  protected:
 149  /**
 150   * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 151   *
 152   * @param text A UText representing the text
 153   * @param rangeStart The start of the range of dictionary characters
 154   * @param rangeEnd The end of the range of dictionary characters
 155   * @param foundBreaks Output of C array of int32_t break positions, or 0
 156   * @return The number of breaks found
 157   */
 158   virtual int32_t divideUpDictionaryRange( UText *text,
 159                                            int32_t rangeStart,
 160                                            int32_t rangeEnd,
 161                                            UVector32 &foundBreaks ) const;
 162
 163 };
 164
 165 /*******************************************************************
 166  * LaoBreakEngine
 167  */
 168
 169 /**
 170  * <p>LaoBreakEngine is a kind of DictionaryBreakEngine that uses a
 171  * dictionary and heuristics to determine Lao-specific breaks.</p>
 172  *
 173  * <p>After it is constructed a LaoBreakEngine may be shared between
 174  * threads without synchronization.</p>
 175  */
 176 class LaoBreakEngine : public DictionaryBreakEngine {
 177  private:
 178     /**
 179      * The set of characters handled by this engine
 180      * @internal
 181      */
 182
 183   UnicodeSet                fLaoWordSet;
 184   UnicodeSet                fEndWordSet;
 185   UnicodeSet                fBeginWordSet;
 186   UnicodeSet                fMarkSet;
 187   DictionaryMatcher  *fDictionary;
 188
 189  public:
 190
 191   /**
 192    * <p>Default constructor.</p>
 193    *
 194    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 195    * engine is deleted.
 196    */
 197   LaoBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 198
 199   /**
 200    * <p>Virtual destructor.</p>
 201    */
 202   virtual ~LaoBreakEngine();
 203
 204  protected:
 205  /**
 206   * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 207   *
 208   * @param text A UText representing the text
 209   * @param rangeStart The start of the range of dictionary characters
 210   * @param rangeEnd The end of the range of dictionary characters
 211   * @param foundBreaks Output of C array of int32_t break positions, or 0
 212   * @return The number of breaks found
 213   */
 214   virtual int32_t divideUpDictionaryRange( UText *text,
 215                                            int32_t rangeStart,
 216                                            int32_t rangeEnd,
 217                                            UVector32 &foundBreaks ) const;
 218
 219 };
 220
 221 /*******************************************************************
 222  * BurmeseBreakEngine
 223  */
 224
 225 /**
 226  * <p>BurmeseBreakEngine is a kind of DictionaryBreakEngine that uses a
 227  * DictionaryMatcher and heuristics to determine Burmese-specific breaks.</p>
 228  *
 229  * <p>After it is constructed a BurmeseBreakEngine may be shared between
 230  * threads without synchronization.</p>
 231  */
 232 class BurmeseBreakEngine : public DictionaryBreakEngine {
 233  private:
 234     /**
 235      * The set of characters handled by this engine
 236      * @internal
 237      */
 238
 239   UnicodeSet                fBurmeseWordSet;
 240   UnicodeSet                fEndWordSet;
 241   UnicodeSet                fBeginWordSet;
 242   UnicodeSet                fMarkSet;
 243   DictionaryMatcher  *fDictionary;
 244
 245  public:
 246
 247   /**
 248    * <p>Default constructor.</p>
 249    *
 250    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 251    * engine is deleted.
 252    */
 253   BurmeseBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 254
 255   /**
 256    * <p>Virtual destructor.</p>
 257    */
 258   virtual ~BurmeseBreakEngine();
 259
 260  protected:
 261  /**
 262   * <p>Divide up a range of known dictionary characters.</p>
 263   *
 264   * @param text A UText representing the text
 265   * @param rangeStart The start of the range of dictionary characters
 266   * @param rangeEnd The end of the range of dictionary characters
 267   * @param foundBreaks Output of C array of int32_t break positions, or 0
 268   * @return The number of breaks found
 269   */
 270   virtual int32_t divideUpDictionaryRange( UText *text,
 271                                            int32_t rangeStart,
 272                                            int32_t rangeEnd,
 273                                            UVector32 &foundBreaks ) const;
 274
 275 };
 276
 277 /*******************************************************************
 278  * KhmerBreakEngine
 279  */
 280
 281 /**
 282  * <p>KhmerBreakEngine is a kind of DictionaryBreakEngine that uses a
 283  * DictionaryMatcher and heuristics to determine Khmer-specific breaks.</p>
 284  *
 285  * <p>After it is constructed a KhmerBreakEngine may be shared between
 286  * threads without synchronization.</p>
 287  */
 288 class KhmerBreakEngine : public DictionaryBreakEngine {
 289  private:
 290     /**
 291      * The set of characters handled by this engine
 292      * @internal
 293      */
 294
 295   UnicodeSet                fKhmerWordSet;
 296   UnicodeSet                fEndWordSet;
 297   UnicodeSet                fBeginWordSet;
 298   UnicodeSet                fMarkSet;
 299   DictionaryMatcher  *fDictionary;
 300
 301  public:
 302
 303   /**
 304    * <p>Default constructor.</p>
 305    *
 306    * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 307    * engine is deleted.
 308    */
 309   KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status);
 310
 311   /**
 312    * <p>Virtual destructor.</p>
 313    */
 314   virtual ~KhmerBreakEngine();
 315
 316  protected:
 317  /**
 318   * <p>Divide up a range of known dictionary characters.</p>
 319   *
 320   * @param text A UText representing the text
 321   * @param rangeStart The start of the range of dictionary characters
 322   * @param rangeEnd The end of the range of dictionary characters
 323   * @param foundBreaks Output of C array of int32_t break positions, or 0
 324   * @return The number of breaks found
 325   */
 326   virtual int32_t divideUpDictionaryRange( UText *text,
 327                                            int32_t rangeStart,
 328                                            int32_t rangeEnd,
 329                                            UVector32 &foundBreaks ) const;
 330
 331 };
 332
 333 #if !UCONFIG_NO_NORMALIZATION
 334
 335 /*******************************************************************
 336  * CjkBreakEngine
 337  */
 338
 339 //indicates language/script that the CjkBreakEngine will handle
 340 enum LanguageType {
 341     kKorean,
 342     kChineseJapanese
 343 };
 344
 345 /**
 346  * <p>CjkBreakEngine is a kind of DictionaryBreakEngine that uses a
 347  * dictionary with costs associated with each word and
 348  * Viterbi decoding to determine CJK-specific breaks.</p>
 349  */
 350 class CjkBreakEngine : public DictionaryBreakEngine {
 351  protected:
 352     /**
 353      * The set of characters handled by this engine
 354      * @internal
 355      */
 356   UnicodeSet                fHangulWordSet;
 357   UnicodeSet                fHanWordSet;
 358   UnicodeSet                fKatakanaWordSet;
 359   UnicodeSet                fHiraganaWordSet;
 360
 361   DictionaryMatcher        *fDictionary;
 362   const Normalizer2        *nfkcNorm2;
 363
 364  public:
 365
 366     /**
 367      * <p>Default constructor.</p>
 368      *
 369      * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the
 370      * engine is deleted. The DictionaryMatcher must contain costs for each word
 371      * in order for the dictionary to work properly.
 372      */
 373   CjkBreakEngine(DictionaryMatcher *adoptDictionary, LanguageType type, UErrorCode &status);
 374
 375     /**
 376      * <p>Virtual destructor.</p>
 377      */
 378   virtual ~CjkBreakEngine();
 379
 380  protected:
 381     /**
 382      * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
 383      *
 384      * @param text A UText representing the text
 385      * @param rangeStart The start of the range of dictionary characters
 386      * @param rangeEnd The end of the range of dictionary characters
 387      * @param foundBreaks Output of C array of int32_t break positions, or 0
 388      * @return The number of breaks found
 389      */
 390   virtual int32_t divideUpDictionaryRange( UText *text,
 391           int32_t rangeStart,
 392           int32_t rangeEnd,
 393           UVector32 &foundBreaks ) const;
 394
 395 };
 396
 397 #endif
 398
 399 U_NAMESPACE_END
 400
 401     /* DICTBE_H */
 402 #endif