icuSources/data/brkitr/word.txt

   1 #
   2 # Copyright (C) 2002-2013, International Business Machines Corporation
   3 # and others. All Rights Reserved.
   4 #
   5 # file:  word.txt
   6 #
   7 # ICU Word Break Rules
   8 #      See Unicode Standard Annex #29.
   9 #      These rules are based on UAX #29 Revision 22 for Unicode Version 6.3
  10 #
  11 # Note:  Updates to word.txt will usually need to be merged into
  12 #        word_POSIX.txt also.
  13
  14 ##############################################################################
  15 #
  16 #  Character class definitions from TR 29
  17 #
  18 ##############################################################################
  19
  20 !!chain;
  21
  22
  23 #
  24 #  Character Class Definitions.
  25 #
  26
  27 $CR                 = [\p{Word_Break = CR}];
  28 $LF                 = [\p{Word_Break = LF}];
  29 $Newline            = [\p{Word_Break = Newline}];
  30 $Extend             = [\p{Word_Break = Extend}];
  31 $Format             = [\p{Word_Break = Format}];
  32 $Katakana           = [\p{Word_Break = Katakana}];
  33 $Hebrew_Letter      = [\p{Word_Break = Hebrew_Letter}];
  34 $ALetter            = [\p{Word_Break = ALetter}];
  35 $Single_Quote       = [\p{Word_Break = Single_Quote}];
  36 $Double_Quote       = [\p{Word_Break = Double_Quote}];
  37 $MidNumLet          = [\p{Word_Break = MidNumLet}];
  38 $MidLetter          = [\p{Word_Break = MidLetter}];
  39 $MidNum             = [\p{Word_Break = MidNum}];
  40 $Numeric            = [\p{Word_Break = Numeric}];
  41 $ExtendNumLet       = [\p{Word_Break = ExtendNumLet}];
  42
  43 $Han                = [:Han:];
  44 $Hiragana           = [:Hiragana:];
  45
  46 $RI_A        = \U0001F1E6;  # Trail ERTU
  47 $RI_B        = \U0001F1E7;  # Trail EGR
  48 $RI_C        = \U0001F1E8;  # Trail AHLNZ
  49 $RI_D        = \U0001F1E9;  # Trail EK
  50 $RI_E        = \U0001F1EA;  # Trail GS
  51 $RI_F        = \U0001F1EB;  # Trail IR
  52 $RI_G        = \U0001F1EC;  # Trail BR
  53 $RI_H        = \U0001F1ED;  # Trail KU
  54 $RI_I        = \U0001F1EE;  # Trail DLNT
  55 $RI_J        = \U0001F1EF;  # Trail OP
  56 $RI_K        = \U0001F1F0;  # Trail R
  57 $RI_L        = \U0001F1F1;  # Trail B
  58 $RI_M        = \U0001F1F2;  # Trail OXY
  59 $RI_N        = \U0001F1F3;  # Trail LO
  60 $RI_P        = \U0001F1F5;  # Trail LT
  61 $RI_R        = \U0001F1F7;  # Trail OU
  62 $RI_S        = \U0001F1F8;  # Trail AEGK
  63 $RI_T        = \U0001F1F9;  # Trail HRW
  64 $RI_U        = \U0001F1FA;  # Trail AS
  65 $RI_V        = \U0001F1FB;      # Trail N
  66
  67 $RI_A_End    = [\U0001F1EA \U0001F1F7 \U0001F1F9 \U0001F1FA];   # ERTU
  68 $RI_B_End    = [\U0001F1EA \U0001F1EC \U0001F1F7];              # EGR
  69 $RI_C_End    = [\U0001F1E6 \U0001F1ED \U0001F1F1 \U0001F1F3 \U0001F1FF]; # AHLNZ
  70 $RI_D_End    = [\U0001F1EA \U0001F1F0];                         # EK
  71 $RI_E_End    = [\U0001F1EC \U0001F1F8];                         # GS
  72 $RI_F_End    = [\U0001F1EE \U0001F1F7];                         # IR
  73 $RI_G_End    = [\U0001F1E7 \U0001F1F7];                         # BR
  74 $RI_H_End    = [\U0001F1F0 \U0001F1FA];                         # KU
  75 $RI_I_End    = [\U0001F1E9 \U0001F1F1 \U0001F1F3 \U0001F1F9];   # DLNT
  76 $RI_J_End    = [\U0001F1F4 \U0001F1F5];                         # OP
  77 $RI_K_End    = \U0001F1F7;                                      # R
  78 $RI_L_End    = \U0001F1E7;                                      # B
  79 $RI_M_End    = [\U0001F1F4 \U0001F1FD \U0001F1FE];              # OXY
  80 $RI_N_End    = [\U0001F1F1 \U0001F1F4];                         # LO
  81 $RI_P_End    = [\U0001F1F1 \U0001F1F9];                         # LT
  82 $RI_R_End    = [\U0001F1F4 \U0001F1FA];                         # OU
  83 $RI_S_End    = [\U0001F1E6 \U0001F1EA \U0001F1EC \U0001F1F0];   # AEGK
  84 $RI_T_End    = [\U0001F1ED \U0001F1F7 \U0001F1FC];              # HRW
  85 $RI_U_End    = [\U0001F1E6 \U0001F1F8];                         # AS
  86 $RI_V_End    = \U0001F1F3;                                      # N
  87
  88
  89 #   Dictionary character set, for triggering language-based break engines. Currently
  90 #   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
  91 #   5.0 or later as the definition of Complex_Context was corrected to include all
  92 #   characters requiring dictionary break.
  93
  94 $Control        = [\p{Grapheme_Cluster_Break = Control}];
  95 $HangulSyllable = [\uac00-\ud7a3];
  96 $ComplexContext = [:LineBreak = Complex_Context:];
  97 $KanaKanji      = [$Han $Hiragana $Katakana];
  98 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
  99 $dictionary     = [$ComplexContext $dictionaryCJK];
 100
 101 # leave CJK scripts out of ALetterPlus
 102 $ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
 103
 104
 105 #
 106 #  Rules 4    Ignore Format and Extend characters,
 107 #             except when they appear at the beginning of a region of text.
 108 #
 109 # TODO: check if handling of katakana in dictionary makes rules incorrect/void
 110 $KatakanaEx           = $Katakana           ($Extend |  $Format)*;
 111 $Hebrew_LetterEx      = $Hebrew_Letter      ($Extend |  $Format)*;
 112 $ALetterEx            = $ALetterPlus        ($Extend |  $Format)*;
 113 $Single_QuoteEx       = $Single_Quote       ($Extend |  $Format)*;
 114 $Double_QuoteEx       = $Double_Quote       ($Extend |  $Format)*;
 115 $MidNumLetEx          = $MidNumLet          ($Extend |  $Format)*;
 116 $MidLetterEx          = $MidLetter          ($Extend |  $Format)*;
 117 $MidNumEx             = $MidNum             ($Extend |  $Format)*;
 118 $NumericEx            = $Numeric            ($Extend |  $Format)*;
 119 $ExtendNumLetEx       = $ExtendNumLet       ($Extend |  $Format)*;
 120
 121 $Ideographic    = [\p{Ideographic}];
 122 $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
 123 $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
 124
 125 ## -------------------------------------------------
 126
 127 !!forward;
 128
 129
 130 # Rule 3 - CR x LF
 131 #
 132 $CR $LF;
 133
 134 # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
 135 #          of a region of Text.   The rule here comes into play when the start of text
 136 #          begins with a group of Format chars, or with a "word" consisting of a single
 137 #          char that is not in any of the listed word break categories followed by
 138 #          format char(s), or is not a CJK dictionary character.
 139 [^$CR $LF $Newline]? ($Extend |  $Format)+;
 140
 141 $NumericEx {100};
 142 $ALetterEx {200};
 143 $HangulSyllable {200};
 144 $Hebrew_LetterEx{200};
 145 $KatakanaEx {400};       # note:  these status values override those from rule 5
 146 $HiraganaEx {400};       #        by virtue of being numerically larger.
 147 $IdeographicEx {400};    #
 148
 149 #
 150 # rule 5
 151 #    Do not break between most letters.
 152 #
 153 ($ALetterEx | $Hebrew_LetterEx)  ($ALetterEx | $Hebrew_LetterEx) {200};
 154
 155 # rule 6 and 7
 156 ($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};
 157
 158 # rule 7a
 159 $Hebrew_LetterEx $Single_QuoteEx {200};
 160
 161 # rule 7b and 7c
 162 $Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
 163
 164 # rule 8
 165
 166 $NumericEx $NumericEx {100};
 167
 168 # rule 9
 169
 170 ($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};
 171
 172 # rule 10
 173
 174 $NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
 175
 176 # rule 11 and 12
 177
 178 $NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
 179
 180 # rule 13
 181 # to be consistent with $KanaKanji $KanaKanhi, changed
 182 # from 300 to 400.
 183 # See also TestRuleStatus in intltest/rbbiapts.cpp
 184 $KatakanaEx  $KatakanaEx {400};
 185
 186 # rule 13a/b
 187
 188 $ALetterEx       $ExtendNumLetEx {200};    #  (13a)
 189 $Hebrew_LetterEx $ExtendNumLetEx {200};    #  (13a)
 190 $NumericEx       $ExtendNumLetEx {100};    #  (13a)
 191 $KatakanaEx      $ExtendNumLetEx {400};    #  (13a)
 192 $ExtendNumLetEx  $ExtendNumLetEx {200};    #  (13a)
 193
 194 $ExtendNumLetEx  $ALetterEx      {200};    #  (13b)
 195 $ExtendNumLetEx  $Hebrew_Letter  {200};    #  (13b)
 196 $ExtendNumLetEx  $NumericEx      {100};    #  (13b)
 197 $ExtendNumLetEx  $KatakanaEx     {400};    #  (13b)
 198
 199 # rule 13c
 200
 201 $RI_A ($Extend|$Format)* $RI_A_End ($Extend|$Format)*;
 202 $RI_B ($Extend|$Format)* $RI_B_End ($Extend|$Format)*;
 203 $RI_C ($Extend|$Format)* $RI_C_End ($Extend|$Format)*;
 204 $RI_D ($Extend|$Format)* $RI_D_End ($Extend|$Format)*;
 205 $RI_E ($Extend|$Format)* $RI_E_End ($Extend|$Format)*;
 206 $RI_F ($Extend|$Format)* $RI_F_End ($Extend|$Format)*;
 207 $RI_G ($Extend|$Format)* $RI_G_End ($Extend|$Format)*;
 208 $RI_H ($Extend|$Format)* $RI_H_End ($Extend|$Format)*;
 209 $RI_I ($Extend|$Format)* $RI_I_End ($Extend|$Format)*;
 210 $RI_J ($Extend|$Format)* $RI_J_End ($Extend|$Format)*;
 211 $RI_K ($Extend|$Format)* $RI_K_End ($Extend|$Format)*;
 212 $RI_L ($Extend|$Format)* $RI_L_End ($Extend|$Format)*;
 213 $RI_M ($Extend|$Format)* $RI_M_End ($Extend|$Format)*;
 214 $RI_N ($Extend|$Format)* $RI_N_End ($Extend|$Format)*;
 215 $RI_P ($Extend|$Format)* $RI_P_End ($Extend|$Format)*;
 216 $RI_R ($Extend|$Format)* $RI_R_End ($Extend|$Format)*;
 217 $RI_S ($Extend|$Format)* $RI_S_End ($Extend|$Format)*;
 218 $RI_T ($Extend|$Format)* $RI_T_End ($Extend|$Format)*;
 219 $RI_U ($Extend|$Format)* $RI_U_End ($Extend|$Format)*;
 220 $RI_V ($Extend|$Format)* $RI_V_End ($Extend|$Format)*;
 221
 222 # special handling for CJK characters: chain for later dictionary segmentation
 223 $HangulSyllable $HangulSyllable {200};
 224 $KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
 225
 226
 227 ## -------------------------------------------------
 228
 229 !!reverse;
 230
 231 $BackHebrew_LetterEx      = ($Format | $Extend)* $Hebrew_Letter;
 232 $BackALetterEx            = ($Format | $Extend)* $ALetterPlus;
 233 $BackSingle_QuoteEx       = ($Format | $Extend)* $Single_Quote;
 234 $BackDouble_QuoteEx       = ($Format | $Extend)* $Double_Quote;
 235 $BackMidNumLetEx          = ($Format | $Extend)* $MidNumLet;
 236 $BackNumericEx            = ($Format | $Extend)* $Numeric;
 237 $BackMidNumEx             = ($Format | $Extend)* $MidNum;
 238 $BackMidLetterEx          = ($Format | $Extend)* $MidLetter;
 239 $BackKatakanaEx           = ($Format | $Extend)* $Katakana;
 240 $BackHiraganaEx           = ($Format | $Extend)* $Hiragana;
 241 $BackExtendNumLetEx       = ($Format | $Extend)* $ExtendNumLet;
 242
 243 # rule 3
 244 $LF $CR;
 245
 246 # rule 4
 247 ($Format | $Extend)*  [^$CR $LF $Newline]?;
 248
 249 # rule 5
 250
 251 ($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
 252
 253 # rule 6 and 7
 254
 255 ($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
 256
 257 # rule 7a
 258 $BackSingle_QuoteEx $BackHebrew_LetterEx;
 259
 260 # Rule 7b and 7c
 261 $BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
 262
 263 # rule 8
 264
 265 $BackNumericEx $BackNumericEx;
 266
 267 # rule 9
 268
 269 $BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
 270
 271 # rule 10
 272
 273 ($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
 274
 275 # rule 11 and 12
 276
 277 $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
 278
 279 # rule 13
 280
 281 $BackKatakanaEx $BackKatakanaEx;
 282
 283 # rules 13 a/b
 284 #
 285 $BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
 286 ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
 287
 288 # rule 13c
 289
 290 ($Format|$Extend)* $RI_A_End ($Format|$Extend)* $RI_A;
 291 ($Format|$Extend)* $RI_B_End ($Format|$Extend)* $RI_B;
 292 ($Format|$Extend)* $RI_C_End ($Format|$Extend)* $RI_C;
 293 ($Format|$Extend)* $RI_D_End ($Format|$Extend)* $RI_D;
 294 ($Format|$Extend)* $RI_E_End ($Format|$Extend)* $RI_E;
 295 ($Format|$Extend)* $RI_F_End ($Format|$Extend)* $RI_F;
 296 ($Format|$Extend)* $RI_G_End ($Format|$Extend)* $RI_G;
 297 ($Format|$Extend)* $RI_H_End ($Format|$Extend)* $RI_H;
 298 ($Format|$Extend)* $RI_I_End ($Format|$Extend)* $RI_I;
 299 ($Format|$Extend)* $RI_J_End ($Format|$Extend)* $RI_J;
 300 ($Format|$Extend)* $RI_K_End ($Format|$Extend)* $RI_K;
 301 ($Format|$Extend)* $RI_L_End ($Format|$Extend)* $RI_L;
 302 ($Format|$Extend)* $RI_M_End ($Format|$Extend)* $RI_M;
 303 ($Format|$Extend)* $RI_N_End ($Format|$Extend)* $RI_N;
 304 ($Format|$Extend)* $RI_P_End ($Format|$Extend)* $RI_P;
 305 ($Format|$Extend)* $RI_R_End ($Format|$Extend)* $RI_R;
 306 ($Format|$Extend)* $RI_S_End ($Format|$Extend)* $RI_S;
 307 ($Format|$Extend)* $RI_T_End ($Format|$Extend)* $RI_T;
 308 ($Format|$Extend)* $RI_U_End ($Format|$Extend)* $RI_U;
 309 ($Format|$Extend)* $RI_V_End ($Format|$Extend)* $RI_V;
 310
 311 # special handling for CJK characters: chain for later dictionary segmentation
 312 $HangulSyllable $HangulSyllable;
 313 $KanaKanji $KanaKanji; #different rule status if both kanji and kana found
 314
 315 ## -------------------------------------------------
 316
 317 !!safe_reverse;
 318
 319 # rule 3
 320 ($Extend | $Format)+ .?;
 321
 322 # rule 6
 323 ($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
 324
 325 # rule 7b
 326 $Double_Quote $BackHebrew_LetterEx;
 327
 328
 329 # rule 11
 330 ($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
 331
 332 # For dictionary-based break
 333 $dictionary $dictionary;
 334
 335 ## -------------------------------------------------
 336
 337 !!safe_forward;
 338
 339 # rule 4
 340 ($Extend | $Format)+ .?;
 341
 342 # rule 6
 343 ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
 344
 345 # rule 7b
 346 $Double_QuoteEx $Hebrew_LetterEx;
 347
 348 # rule 11
 349 ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
 350
 351 # For dictionary-based break
 352 $dictionary $dictionary;