]>
Commit | Line | Data |
---|---|---|
b75a7d8f | 1 | # |
08b89b0a | 2 | # Copyright (C) 2002-2015, International Business Machines Corporation |
374ca955 | 3 | # and others. All Rights Reserved. |
b75a7d8f | 4 | # |
374ca955 | 5 | # file: word.txt |
b75a7d8f | 6 | # |
374ca955 | 7 | # ICU Word Break Rules |
b75a7d8f | 8 | # See Unicode Standard Annex #29. |
57a6839d | 9 | # These rules are based on UAX #29 Revision 22 for Unicode Version 6.3 |
b75a7d8f | 10 | # |
73c04bcf | 11 | # Note: Updates to word.txt will usually need to be merged into |
51004dcb | 12 | # word_POSIX.txt also. |
b75a7d8f | 13 | |
374ca955 | 14 | ############################################################################## |
b75a7d8f A |
15 | # |
16 | # Character class definitions from TR 29 | |
17 | # | |
374ca955 A |
18 | ############################################################################## |
19 | ||
20 | !!chain; | |
b331163b | 21 | !!RINoChain; |
374ca955 | 22 | |
b75a7d8f A |
23 | |
24 | # | |
25 | # Character Class Definitions. | |
b75a7d8f | 26 | # |
b75a7d8f | 27 | |
57a6839d A |
28 | $CR = [\p{Word_Break = CR}]; |
29 | $LF = [\p{Word_Break = LF}]; | |
30 | $Newline = [\p{Word_Break = Newline}]; | |
31 | $Extend = [\p{Word_Break = Extend}]; | |
b331163b | 32 | $RI = [\p{Word_Break = Regional_Indicator}]; |
57a6839d A |
33 | $Format = [\p{Word_Break = Format}]; |
34 | $Katakana = [\p{Word_Break = Katakana}]; | |
35 | $Hebrew_Letter = [\p{Word_Break = Hebrew_Letter}]; | |
36 | $ALetter = [\p{Word_Break = ALetter}]; | |
37 | $Single_Quote = [\p{Word_Break = Single_Quote}]; | |
38 | $Double_Quote = [\p{Word_Break = Double_Quote}]; | |
39 | $MidNumLet = [\p{Word_Break = MidNumLet}]; | |
40 | $MidLetter = [\p{Word_Break = MidLetter}]; | |
41 | $MidNum = [\p{Word_Break = MidNum}]; | |
42 | $Numeric = [\p{Word_Break = Numeric}]; | |
43 | $ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; | |
44 | ||
45 | $Han = [:Han:]; | |
46 | $Hiragana = [:Hiragana:]; | |
73c04bcf | 47 | |
08b89b0a A |
48 | # Special character classes for people & body part emoji: |
49 | # Subsets of $Extend: | |
50 | $ZWJ = \u200D; | |
51 | $EmojiVar = \uFE0F; | |
52 | # The following are subsets of \p{Word_Break = Other} | |
b801cf36 A |
53 | $EmojiForSeqs = [\u2764 \U0001F441 \U0001F466-\U0001F469 \U0001F48B \U0001F5E8]; |
54 | $EmojiForMods = [\u261D \u26F9 \u270A-\u270D \U0001F385 \U0001F3C3-\U0001F3C4 \U0001F3C7 \U0001F3CA-\U0001F3CB \U0001F442-\U0001F443 \U0001F446-\U0001F450 \U0001F466-\U0001F469 \U0001F46E-\U0001F478 \U0001F47C \U0001F481-\U0001F483 \U0001F485-\U0001F487 \U0001F4AA \U0001F590 \U0001F595 \U0001F596 \U0001F645-\U0001F647 \U0001F64B-\U0001F64F \U0001F6A3 \U0001F6B4-\U0001F6B6 \U0001F6C0 \U0001F918]; | |
08b89b0a | 55 | $EmojiMods = [\U0001F3FB-\U0001F3FF]; |
73c04bcf | 56 | |
73c04bcf A |
57 | # Dictionary character set, for triggering language-based break engines. Currently |
58 | # limited to LineBreak=Complex_Context. Note that this set only works in Unicode | |
59 | # 5.0 or later as the definition of Complex_Context was corrected to include all | |
60 | # characters requiring dictionary break. | |
61 | ||
46f4442e | 62 | $Control = [\p{Grapheme_Cluster_Break = Control}]; |
51004dcb A |
63 | $HangulSyllable = [\uac00-\ud7a3]; |
64 | $ComplexContext = [:LineBreak = Complex_Context:]; | |
65 | $KanaKanji = [$Han $Hiragana $Katakana]; | |
66 | $dictionaryCJK = [$KanaKanji $HangulSyllable]; | |
67 | $dictionary = [$ComplexContext $dictionaryCJK]; | |
68 | ||
69 | # leave CJK scripts out of ALetterPlus | |
70 | $ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]]; | |
71 | ||
73c04bcf A |
72 | |
73 | # | |
46f4442e A |
74 | # Rules 4 Ignore Format and Extend characters, |
75 | # except when they appear at the beginning of a region of text. | |
73c04bcf | 76 | # |
51004dcb | 77 | # TODO: check if handling of katakana in dictionary makes rules incorrect/void |
57a6839d A |
78 | $KatakanaEx = $Katakana ($Extend | $Format)*; |
79 | $Hebrew_LetterEx = $Hebrew_Letter ($Extend | $Format)*; | |
80 | $ALetterEx = $ALetterPlus ($Extend | $Format)*; | |
81 | $Single_QuoteEx = $Single_Quote ($Extend | $Format)*; | |
82 | $Double_QuoteEx = $Double_Quote ($Extend | $Format)*; | |
83 | $MidNumLetEx = $MidNumLet ($Extend | $Format)*; | |
84 | $MidLetterEx = $MidLetter ($Extend | $Format)*; | |
85 | $MidNumEx = $MidNum ($Extend | $Format)*; | |
86 | $NumericEx = $Numeric ($Extend | $Format)*; | |
87 | $ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; | |
73c04bcf | 88 | |
46f4442e | 89 | $Ideographic = [\p{Ideographic}]; |
73c04bcf A |
90 | $HiraganaEx = $Hiragana ($Extend | $Format)*; |
91 | $IdeographicEx = $Ideographic ($Extend | $Format)*; | |
b75a7d8f | 92 | |
374ca955 | 93 | ## ------------------------------------------------- |
b75a7d8f | 94 | |
374ca955 | 95 | !!forward; |
b75a7d8f | 96 | |
b75a7d8f | 97 | |
73c04bcf | 98 | # Rule 3 - CR x LF |
46f4442e A |
99 | # |
100 | $CR $LF; | |
73c04bcf A |
101 | |
102 | # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning | |
103 | # of a region of Text. The rule here comes into play when the start of text | |
104 | # begins with a group of Format chars, or with a "word" consisting of a single | |
105 | # char that is not in any of the listed word break categories followed by | |
51004dcb | 106 | # format char(s), or is not a CJK dictionary character. |
46f4442e | 107 | [^$CR $LF $Newline]? ($Extend | $Format)+; |
b75a7d8f | 108 | |
374ca955 A |
109 | $NumericEx {100}; |
110 | $ALetterEx {200}; | |
51004dcb | 111 | $HangulSyllable {200}; |
57a6839d | 112 | $Hebrew_LetterEx{200}; |
51004dcb A |
113 | $KatakanaEx {400}; # note: these status values override those from rule 5 |
114 | $HiraganaEx {400}; # by virtue of being numerically larger. | |
46f4442e | 115 | $IdeographicEx {400}; # |
b75a7d8f | 116 | |
46f4442e | 117 | # |
374ca955 | 118 | # rule 5 |
46f4442e A |
119 | # Do not break between most letters. |
120 | # | |
57a6839d | 121 | ($ALetterEx | $Hebrew_LetterEx) ($ALetterEx | $Hebrew_LetterEx) {200}; |
b75a7d8f | 122 | |
374ca955 | 123 | # rule 6 and 7 |
57a6839d A |
124 | ($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200}; |
125 | ||
126 | # rule 7a | |
127 | $Hebrew_LetterEx $Single_QuoteEx {200}; | |
128 | ||
129 | # rule 7b and 7c | |
130 | $Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200}; | |
b75a7d8f | 131 | |
374ca955 | 132 | # rule 8 |
b75a7d8f | 133 | |
73c04bcf | 134 | $NumericEx $NumericEx {100}; |
b75a7d8f | 135 | |
374ca955 | 136 | # rule 9 |
b75a7d8f | 137 | |
57a6839d | 138 | ($ALetterEx | $Hebrew_LetterEx) $NumericEx {200}; |
b75a7d8f | 139 | |
374ca955 | 140 | # rule 10 |
b75a7d8f | 141 | |
57a6839d | 142 | $NumericEx ($ALetterEx | $Hebrew_LetterEx) {200}; |
b75a7d8f | 143 | |
374ca955 A |
144 | # rule 11 and 12 |
145 | ||
57a6839d | 146 | $NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100}; |
374ca955 A |
147 | |
148 | # rule 13 | |
51004dcb A |
149 | # to be consistent with $KanaKanji $KanaKanhi, changed |
150 | # from 300 to 400. | |
151 | # See also TestRuleStatus in intltest/rbbiapts.cpp | |
152 | $KatakanaEx $KatakanaEx {400}; | |
374ca955 A |
153 | |
154 | # rule 13a/b | |
155 | ||
57a6839d A |
156 | $ALetterEx $ExtendNumLetEx {200}; # (13a) |
157 | $Hebrew_LetterEx $ExtendNumLetEx {200}; # (13a) | |
158 | $NumericEx $ExtendNumLetEx {100}; # (13a) | |
159 | $KatakanaEx $ExtendNumLetEx {400}; # (13a) | |
160 | $ExtendNumLetEx $ExtendNumLetEx {200}; # (13a) | |
374ca955 | 161 | |
57a6839d A |
162 | $ExtendNumLetEx $ALetterEx {200}; # (13b) |
163 | $ExtendNumLetEx $Hebrew_Letter {200}; # (13b) | |
164 | $ExtendNumLetEx $NumericEx {100}; # (13b) | |
165 | $ExtendNumLetEx $KatakanaEx {400}; # (13b) | |
51004dcb A |
166 | |
167 | # rule 13c | |
168 | ||
b331163b A |
169 | $RI $RI $Extend* / $RI; |
170 | $RI $RI $Extend*; | |
08b89b0a A |
171 | |
172 | # Special forward rules for people & body part emoji: | |
173 | # don't break $ZWJ from subsequent $EmojiForSeqs; don't break between relevant emoji and $EmojiMods | |
174 | $ZWJ $EmojiForSeqs; | |
175 | $EmojiForMods $EmojiVar? $EmojiMods; | |
51004dcb A |
176 | |
177 | # special handling for CJK characters: chain for later dictionary segmentation | |
178 | $HangulSyllable $HangulSyllable {200}; | |
179 | $KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found | |
374ca955 A |
180 | |
181 | ||
182 | ## ------------------------------------------------- | |
b75a7d8f | 183 | |
374ca955 A |
184 | !!reverse; |
185 | ||
57a6839d | 186 | $BackHebrew_LetterEx = ($Format | $Extend)* $Hebrew_Letter; |
51004dcb | 187 | $BackALetterEx = ($Format | $Extend)* $ALetterPlus; |
57a6839d A |
188 | $BackSingle_QuoteEx = ($Format | $Extend)* $Single_Quote; |
189 | $BackDouble_QuoteEx = ($Format | $Extend)* $Double_Quote; | |
51004dcb A |
190 | $BackMidNumLetEx = ($Format | $Extend)* $MidNumLet; |
191 | $BackNumericEx = ($Format | $Extend)* $Numeric; | |
192 | $BackMidNumEx = ($Format | $Extend)* $MidNum; | |
193 | $BackMidLetterEx = ($Format | $Extend)* $MidLetter; | |
194 | $BackKatakanaEx = ($Format | $Extend)* $Katakana; | |
195 | $BackHiraganaEx = ($Format | $Extend)* $Hiragana; | |
196 | $BackExtendNumLetEx = ($Format | $Extend)* $ExtendNumLet; | |
374ca955 | 197 | |
73c04bcf | 198 | # rule 3 |
46f4442e | 199 | $LF $CR; |
374ca955 | 200 | |
73c04bcf | 201 | # rule 4 |
46f4442e | 202 | ($Format | $Extend)* [^$CR $LF $Newline]?; |
374ca955 A |
203 | |
204 | # rule 5 | |
205 | ||
57a6839d | 206 | ($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx); |
374ca955 A |
207 | |
208 | # rule 6 and 7 | |
209 | ||
57a6839d A |
210 | ($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx); |
211 | ||
212 | # rule 7a | |
213 | $BackSingle_QuoteEx $BackHebrew_LetterEx; | |
374ca955 | 214 | |
57a6839d A |
215 | # Rule 7b and 7c |
216 | $BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx; | |
374ca955 A |
217 | |
218 | # rule 8 | |
219 | ||
73c04bcf | 220 | $BackNumericEx $BackNumericEx; |
374ca955 A |
221 | |
222 | # rule 9 | |
223 | ||
57a6839d | 224 | $BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx); |
374ca955 A |
225 | |
226 | # rule 10 | |
227 | ||
57a6839d | 228 | ($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx; |
374ca955 A |
229 | |
230 | # rule 11 and 12 | |
231 | ||
57a6839d | 232 | $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx; |
374ca955 A |
233 | |
234 | # rule 13 | |
235 | ||
73c04bcf | 236 | $BackKatakanaEx $BackKatakanaEx; |
374ca955 A |
237 | |
238 | # rules 13 a/b | |
b75a7d8f | 239 | # |
57a6839d A |
240 | $BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx); |
241 | ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; | |
374ca955 | 242 | |
51004dcb A |
243 | # rule 13c |
244 | ||
b331163b A |
245 | $Extend* $RI $RI / $Extend* $RI $RI; |
246 | $Extend* $RI $RI; | |
08b89b0a A |
247 | |
248 | # Special reverse rules for people & body part emoji: | |
249 | # don't break $ZWJ from subsequent $EmojiForSeqs; don't break between relevant emoji and $EmojiMods | |
250 | $EmojiForSeqs $ZWJ; | |
251 | $EmojiMods $EmojiVar? $EmojiForMods; | |
51004dcb A |
252 | |
253 | # special handling for CJK characters: chain for later dictionary segmentation | |
254 | $HangulSyllable $HangulSyllable; | |
255 | $KanaKanji $KanaKanji; #different rule status if both kanji and kana found | |
256 | ||
374ca955 A |
257 | ## ------------------------------------------------- |
258 | ||
259 | !!safe_reverse; | |
260 | ||
261 | # rule 3 | |
73c04bcf | 262 | ($Extend | $Format)+ .?; |
374ca955 A |
263 | |
264 | # rule 6 | |
57a6839d A |
265 | ($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx); |
266 | ||
267 | # rule 7b | |
268 | $Double_Quote $BackHebrew_LetterEx; | |
269 | ||
374ca955 A |
270 | |
271 | # rule 11 | |
57a6839d | 272 | ($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx; |
73c04bcf | 273 | |
b331163b | 274 | $RI $RI+; |
73c04bcf A |
275 | # For dictionary-based break |
276 | $dictionary $dictionary; | |
374ca955 A |
277 | |
278 | ## ------------------------------------------------- | |
279 | ||
280 | !!safe_forward; | |
281 | ||
374ca955 | 282 | # rule 4 |
73c04bcf | 283 | ($Extend | $Format)+ .?; |
374ca955 A |
284 | |
285 | # rule 6 | |
57a6839d A |
286 | ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx); |
287 | ||
288 | # rule 7b | |
289 | $Double_QuoteEx $Hebrew_LetterEx; | |
b75a7d8f | 290 | |
374ca955 | 291 | # rule 11 |
57a6839d | 292 | ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx; |
73c04bcf | 293 | |
b331163b | 294 | $RI $RI+; |
73c04bcf A |
295 | # For dictionary-based break |
296 | $dictionary $dictionary; |