]>
Commit | Line | Data |
---|---|---|
1 | // © 2016 and later: Unicode, Inc. and others. | |
2 | // License & terms of use: http://www.unicode.org/copyright.html | |
3 | /** | |
4 | ******************************************************************************* | |
5 | * Copyright (C) 2006-2014, International Business Machines Corporation * | |
6 | * and others. All Rights Reserved. * | |
7 | ******************************************************************************* | |
8 | */ | |
9 | ||
10 | #ifndef DICTBE_H | |
11 | #define DICTBE_H | |
12 | ||
13 | #include "unicode/utypes.h" | |
14 | #include "unicode/uniset.h" | |
15 | #include "unicode/utext.h" | |
16 | ||
17 | #include "brkeng.h" | |
18 | #include "uvectr32.h" | |
19 | ||
20 | U_NAMESPACE_BEGIN | |
21 | ||
22 | class DictionaryMatcher; | |
23 | class Normalizer2; | |
24 | ||
25 | /******************************************************************* | |
26 | * DictionaryBreakEngine | |
27 | */ | |
28 | ||
29 | /** | |
30 | * <p>DictionaryBreakEngine is a kind of LanguageBreakEngine that uses a | |
31 | * dictionary to determine language-specific breaks.</p> | |
32 | * | |
33 | * <p>After it is constructed a DictionaryBreakEngine may be shared between | |
34 | * threads without synchronization.</p> | |
35 | */ | |
36 | class DictionaryBreakEngine : public LanguageBreakEngine { | |
37 | private: | |
38 | /** | |
39 | * The set of characters handled by this engine | |
40 | * @internal | |
41 | */ | |
42 | ||
43 | UnicodeSet fSet; | |
44 | ||
45 | public: | |
46 | ||
47 | /** | |
48 | * <p>Constructor </p> | |
49 | */ | |
50 | DictionaryBreakEngine(); | |
51 | ||
52 | /** | |
53 | * <p>Virtual destructor.</p> | |
54 | */ | |
55 | virtual ~DictionaryBreakEngine(); | |
56 | ||
57 | /** | |
58 | * <p>Indicate whether this engine handles a particular character for | |
59 | * a particular kind of break.</p> | |
60 | * | |
61 | * @param c A character which begins a run that the engine might handle | |
62 | * @return TRUE if this engine handles the particular character and break | |
63 | * type. | |
64 | */ | |
65 | virtual UBool handles(UChar32 c) const; | |
66 | ||
67 | /** | |
68 | * <p>Find any breaks within a run in the supplied text.</p> | |
69 | * | |
70 | * @param text A UText representing the text. The iterator is left at | |
71 | * the end of the run of characters which the engine is capable of handling | |
72 | * that starts from the first character in the range. | |
73 | * @param startPos The start of the run within the supplied text. | |
74 | * @param endPos The end of the run within the supplied text. | |
75 | * @param foundBreaks vector of int32_t to receive the break positions | |
76 | * @return The number of breaks found. | |
77 | */ | |
78 | virtual int32_t findBreaks( UText *text, | |
79 | int32_t startPos, | |
80 | int32_t endPos, | |
81 | UVector32 &foundBreaks ) const; | |
82 | ||
83 | protected: | |
84 | ||
85 | /** | |
86 | * <p>Set the character set handled by this engine.</p> | |
87 | * | |
88 | * @param set A UnicodeSet of the set of characters handled by the engine | |
89 | */ | |
90 | virtual void setCharacters( const UnicodeSet &set ); | |
91 | ||
92 | /** | |
93 | * <p>Divide up a range of known dictionary characters handled by this break engine.</p> | |
94 | * | |
95 | * @param text A UText representing the text | |
96 | * @param rangeStart The start of the range of dictionary characters | |
97 | * @param rangeEnd The end of the range of dictionary characters | |
98 | * @param foundBreaks Output of C array of int32_t break positions, or 0 | |
99 | * @return The number of breaks found | |
100 | */ | |
101 | virtual int32_t divideUpDictionaryRange( UText *text, | |
102 | int32_t rangeStart, | |
103 | int32_t rangeEnd, | |
104 | UVector32 &foundBreaks ) const = 0; | |
105 | ||
106 | }; | |
107 | ||
108 | /******************************************************************* | |
109 | * ThaiBreakEngine | |
110 | */ | |
111 | ||
112 | /** | |
113 | * <p>ThaiBreakEngine is a kind of DictionaryBreakEngine that uses a | |
114 | * dictionary and heuristics to determine Thai-specific breaks.</p> | |
115 | * | |
116 | * <p>After it is constructed a ThaiBreakEngine may be shared between | |
117 | * threads without synchronization.</p> | |
118 | */ | |
119 | class ThaiBreakEngine : public DictionaryBreakEngine { | |
120 | private: | |
121 | /** | |
122 | * The set of characters handled by this engine | |
123 | * @internal | |
124 | */ | |
125 | ||
126 | UnicodeSet fThaiWordSet; | |
127 | UnicodeSet fEndWordSet; | |
128 | UnicodeSet fBeginWordSet; | |
129 | UnicodeSet fSuffixSet; | |
130 | UnicodeSet fMarkSet; | |
131 | DictionaryMatcher *fDictionary; | |
132 | ||
133 | public: | |
134 | ||
135 | /** | |
136 | * <p>Default constructor.</p> | |
137 | * | |
138 | * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the | |
139 | * engine is deleted. | |
140 | */ | |
141 | ThaiBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status); | |
142 | ||
143 | /** | |
144 | * <p>Virtual destructor.</p> | |
145 | */ | |
146 | virtual ~ThaiBreakEngine(); | |
147 | ||
148 | protected: | |
149 | /** | |
150 | * <p>Divide up a range of known dictionary characters handled by this break engine.</p> | |
151 | * | |
152 | * @param text A UText representing the text | |
153 | * @param rangeStart The start of the range of dictionary characters | |
154 | * @param rangeEnd The end of the range of dictionary characters | |
155 | * @param foundBreaks Output of C array of int32_t break positions, or 0 | |
156 | * @return The number of breaks found | |
157 | */ | |
158 | virtual int32_t divideUpDictionaryRange( UText *text, | |
159 | int32_t rangeStart, | |
160 | int32_t rangeEnd, | |
161 | UVector32 &foundBreaks ) const; | |
162 | ||
163 | }; | |
164 | ||
165 | /******************************************************************* | |
166 | * LaoBreakEngine | |
167 | */ | |
168 | ||
169 | /** | |
170 | * <p>LaoBreakEngine is a kind of DictionaryBreakEngine that uses a | |
171 | * dictionary and heuristics to determine Lao-specific breaks.</p> | |
172 | * | |
173 | * <p>After it is constructed a LaoBreakEngine may be shared between | |
174 | * threads without synchronization.</p> | |
175 | */ | |
176 | class LaoBreakEngine : public DictionaryBreakEngine { | |
177 | private: | |
178 | /** | |
179 | * The set of characters handled by this engine | |
180 | * @internal | |
181 | */ | |
182 | ||
183 | UnicodeSet fLaoWordSet; | |
184 | UnicodeSet fEndWordSet; | |
185 | UnicodeSet fBeginWordSet; | |
186 | UnicodeSet fMarkSet; | |
187 | DictionaryMatcher *fDictionary; | |
188 | ||
189 | public: | |
190 | ||
191 | /** | |
192 | * <p>Default constructor.</p> | |
193 | * | |
194 | * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the | |
195 | * engine is deleted. | |
196 | */ | |
197 | LaoBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status); | |
198 | ||
199 | /** | |
200 | * <p>Virtual destructor.</p> | |
201 | */ | |
202 | virtual ~LaoBreakEngine(); | |
203 | ||
204 | protected: | |
205 | /** | |
206 | * <p>Divide up a range of known dictionary characters handled by this break engine.</p> | |
207 | * | |
208 | * @param text A UText representing the text | |
209 | * @param rangeStart The start of the range of dictionary characters | |
210 | * @param rangeEnd The end of the range of dictionary characters | |
211 | * @param foundBreaks Output of C array of int32_t break positions, or 0 | |
212 | * @return The number of breaks found | |
213 | */ | |
214 | virtual int32_t divideUpDictionaryRange( UText *text, | |
215 | int32_t rangeStart, | |
216 | int32_t rangeEnd, | |
217 | UVector32 &foundBreaks ) const; | |
218 | ||
219 | }; | |
220 | ||
221 | /******************************************************************* | |
222 | * BurmeseBreakEngine | |
223 | */ | |
224 | ||
225 | /** | |
226 | * <p>BurmeseBreakEngine is a kind of DictionaryBreakEngine that uses a | |
227 | * DictionaryMatcher and heuristics to determine Burmese-specific breaks.</p> | |
228 | * | |
229 | * <p>After it is constructed a BurmeseBreakEngine may be shared between | |
230 | * threads without synchronization.</p> | |
231 | */ | |
232 | class BurmeseBreakEngine : public DictionaryBreakEngine { | |
233 | private: | |
234 | /** | |
235 | * The set of characters handled by this engine | |
236 | * @internal | |
237 | */ | |
238 | ||
239 | UnicodeSet fBurmeseWordSet; | |
240 | UnicodeSet fEndWordSet; | |
241 | UnicodeSet fBeginWordSet; | |
242 | UnicodeSet fMarkSet; | |
243 | DictionaryMatcher *fDictionary; | |
244 | ||
245 | public: | |
246 | ||
247 | /** | |
248 | * <p>Default constructor.</p> | |
249 | * | |
250 | * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the | |
251 | * engine is deleted. | |
252 | */ | |
253 | BurmeseBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status); | |
254 | ||
255 | /** | |
256 | * <p>Virtual destructor.</p> | |
257 | */ | |
258 | virtual ~BurmeseBreakEngine(); | |
259 | ||
260 | protected: | |
261 | /** | |
262 | * <p>Divide up a range of known dictionary characters.</p> | |
263 | * | |
264 | * @param text A UText representing the text | |
265 | * @param rangeStart The start of the range of dictionary characters | |
266 | * @param rangeEnd The end of the range of dictionary characters | |
267 | * @param foundBreaks Output of C array of int32_t break positions, or 0 | |
268 | * @return The number of breaks found | |
269 | */ | |
270 | virtual int32_t divideUpDictionaryRange( UText *text, | |
271 | int32_t rangeStart, | |
272 | int32_t rangeEnd, | |
273 | UVector32 &foundBreaks ) const; | |
274 | ||
275 | }; | |
276 | ||
277 | /******************************************************************* | |
278 | * KhmerBreakEngine | |
279 | */ | |
280 | ||
281 | /** | |
282 | * <p>KhmerBreakEngine is a kind of DictionaryBreakEngine that uses a | |
283 | * DictionaryMatcher and heuristics to determine Khmer-specific breaks.</p> | |
284 | * | |
285 | * <p>After it is constructed a KhmerBreakEngine may be shared between | |
286 | * threads without synchronization.</p> | |
287 | */ | |
288 | class KhmerBreakEngine : public DictionaryBreakEngine { | |
289 | private: | |
290 | /** | |
291 | * The set of characters handled by this engine | |
292 | * @internal | |
293 | */ | |
294 | ||
295 | UnicodeSet fKhmerWordSet; | |
296 | UnicodeSet fEndWordSet; | |
297 | UnicodeSet fBeginWordSet; | |
298 | UnicodeSet fMarkSet; | |
299 | DictionaryMatcher *fDictionary; | |
300 | ||
301 | public: | |
302 | ||
303 | /** | |
304 | * <p>Default constructor.</p> | |
305 | * | |
306 | * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the | |
307 | * engine is deleted. | |
308 | */ | |
309 | KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status); | |
310 | ||
311 | /** | |
312 | * <p>Virtual destructor.</p> | |
313 | */ | |
314 | virtual ~KhmerBreakEngine(); | |
315 | ||
316 | protected: | |
317 | /** | |
318 | * <p>Divide up a range of known dictionary characters.</p> | |
319 | * | |
320 | * @param text A UText representing the text | |
321 | * @param rangeStart The start of the range of dictionary characters | |
322 | * @param rangeEnd The end of the range of dictionary characters | |
323 | * @param foundBreaks Output of C array of int32_t break positions, or 0 | |
324 | * @return The number of breaks found | |
325 | */ | |
326 | virtual int32_t divideUpDictionaryRange( UText *text, | |
327 | int32_t rangeStart, | |
328 | int32_t rangeEnd, | |
329 | UVector32 &foundBreaks ) const; | |
330 | ||
331 | }; | |
332 | ||
333 | #if !UCONFIG_NO_NORMALIZATION | |
334 | ||
335 | /******************************************************************* | |
336 | * CjkBreakEngine | |
337 | */ | |
338 | ||
339 | //indicates language/script that the CjkBreakEngine will handle | |
340 | enum LanguageType { | |
341 | kKorean, | |
342 | kChineseJapanese | |
343 | }; | |
344 | ||
345 | /** | |
346 | * <p>CjkBreakEngine is a kind of DictionaryBreakEngine that uses a | |
347 | * dictionary with costs associated with each word and | |
348 | * Viterbi decoding to determine CJK-specific breaks.</p> | |
349 | */ | |
350 | class CjkBreakEngine : public DictionaryBreakEngine { | |
351 | protected: | |
352 | /** | |
353 | * The set of characters handled by this engine | |
354 | * @internal | |
355 | */ | |
356 | UnicodeSet fHangulWordSet; | |
357 | UnicodeSet fHanWordSet; | |
358 | UnicodeSet fKatakanaWordSet; | |
359 | UnicodeSet fHiraganaWordSet; | |
360 | ||
361 | DictionaryMatcher *fDictionary; | |
362 | const Normalizer2 *nfkcNorm2; | |
363 | ||
364 | public: | |
365 | ||
366 | /** | |
367 | * <p>Default constructor.</p> | |
368 | * | |
369 | * @param adoptDictionary A DictionaryMatcher to adopt. Deleted when the | |
370 | * engine is deleted. The DictionaryMatcher must contain costs for each word | |
371 | * in order for the dictionary to work properly. | |
372 | */ | |
373 | CjkBreakEngine(DictionaryMatcher *adoptDictionary, LanguageType type, UErrorCode &status); | |
374 | ||
375 | /** | |
376 | * <p>Virtual destructor.</p> | |
377 | */ | |
378 | virtual ~CjkBreakEngine(); | |
379 | ||
380 | protected: | |
381 | /** | |
382 | * <p>Divide up a range of known dictionary characters handled by this break engine.</p> | |
383 | * | |
384 | * @param text A UText representing the text | |
385 | * @param rangeStart The start of the range of dictionary characters | |
386 | * @param rangeEnd The end of the range of dictionary characters | |
387 | * @param foundBreaks Output of C array of int32_t break positions, or 0 | |
388 | * @return The number of breaks found | |
389 | */ | |
390 | virtual int32_t divideUpDictionaryRange( UText *text, | |
391 | int32_t rangeStart, | |
392 | int32_t rangeEnd, | |
393 | UVector32 &foundBreaks ) const; | |
394 | ||
395 | }; | |
396 | ||
397 | #endif | |
398 | ||
399 | U_NAMESPACE_END | |
400 | ||
401 | /* DICTBE_H */ | |
402 | #endif |