icuSources/data/brkitr/word_POSIX.txt

   1 #
   2 # Copyright (C) 2002-2015, International Business Machines Corporation
   3 # and others. All Rights Reserved.
   4 #
   5 # file:  word_POSIX.txt
   6 #
   7 # ICU Word Break Rules, POSIX locale.
   8 #      See Unicode Standard Annex #29.
   9 #      These rules are based on UAX #29 Revision 22 for Unicode Version 6.3
  10 #
  11 # Note:  Updates to word.txt will usually need to be merged into
  12 #        word_POSIX.txt also.
  13
  14 ##############################################################################
  15 #
  16 #  Character class definitions from TR 29
  17 #
  18 ##############################################################################
  19
  20 !!chain;
  21
  22
  23 #
  24 #  Character Class Definitions.
  25 #
  26
  27 $CR                 = [\p{Word_Break = CR}];
  28 $LF                 = [\p{Word_Break = LF}];
  29 $Newline            = [\p{Word_Break = Newline}];
  30 $Extend             = [\p{Word_Break = Extend}];
  31 $Format             = [\p{Word_Break = Format}];
  32 $Katakana           = [\p{Word_Break = Katakana}];
  33 $Hebrew_Letter      = [\p{Word_Break = Hebrew_Letter}];
  34 $ALetter            = [\p{Word_Break = ALetter}];
  35 $Single_Quote       = [\p{Word_Break = Single_Quote}];
  36 $Double_Quote       = [\p{Word_Break = Double_Quote}];
  37 $MidNumLet          = [\p{Word_Break = MidNumLet} - [.]];
  38 $MidLetter          = [\p{Word_Break = MidLetter} - [\:]];
  39 $MidNum             = [\p{Word_Break = MidNum} [.]];
  40 $Numeric            = [\p{Word_Break = Numeric}];
  41 $ExtendNumLet       = [\p{Word_Break = ExtendNumLet}];
  42
  43 $Han                = [:Han:];
  44 $Hiragana           = [:Hiragana:];
  45
  46 $RI_A        = \U0001F1E6;  # Trail ETU
  47 $RI_B        = \U0001F1E7;  # Trail ER
  48 $RI_C        = \U0001F1E8;  # Trail AHLNO
  49 $RI_D        = \U0001F1E9;  # Trail EK
  50 $RI_E        = \U0001F1EA;  # Trail S
  51 $RI_F        = \U0001F1EB;  # Trail IR
  52 $RI_G        = \U0001F1EC;  # Trail B
  53 $RI_H        = \U0001F1ED;  # Trail K
  54 $RI_I        = \U0001F1EE;  # Trail DELNT
  55 $RI_J        = \U0001F1EF;  # Trail OP
  56 $RI_K        = \U0001F1F0;  # Trail R
  57 $RI_M        = \U0001F1F2;  # Trail OXY
  58 $RI_N        = \U0001F1F3;  # Trail LOZ
  59 $RI_P        = \U0001F1F5;  # Trail HLRT
  60 $RI_R        = \U0001F1F7;  # Trail U
  61 $RI_S        = \U0001F1F8;  # Trail AEG
  62 $RI_T        = \U0001F1F9;  # Trail R
  63 $RI_U        = \U0001F1FA;  # Trail S
  64 $RI_V        = \U0001F1FB;      # Trail N
  65 $RI_Z        = \U0001F1FF;      # Trail A
  66
  67 $RI_A_End    = [\U0001F1EA \U0001F1F9 \U0001F1FA];              # ETU
  68 $RI_B_End    = [\U0001F1EA \U0001F1F7];                         # ER
  69 $RI_C_End    = [\U0001F1E6 \U0001F1ED \U0001F1F1 \U0001F1F3 \U0001F1F4]; # AHLNO
  70 $RI_D_End    = [\U0001F1EA \U0001F1F0];                         # EK
  71 $RI_E_End    = \U0001F1F8;                                      # S
  72 $RI_F_End    = [\U0001F1EE \U0001F1F7];                         # IR
  73 $RI_G_End    = \U0001F1E7;                                      # B
  74 $RI_H_End    = \U0001F1F0;                                      # K
  75 $RI_I_End    = [\U0001F1E9 \U0001F1EA \U0001F1F1 \U0001F1F3 \U0001F1F9]; # DELNT
  76 $RI_J_End    = [\U0001F1F5 \U0001F1F4];                         # OP
  77 $RI_K_End    = \U0001F1F7;                                      # R
  78 $RI_M_End    = [\U0001F1F4 \U0001F1FD \U0001F1FE];              # OXY
  79 $RI_N_End    = [\U0001F1F1 \U0001F1F4 \U0001F1FF];              # LOZ
  80 $RI_P_End    = [\U0001F1ED \U0001F1F1 \U0001F1F7 \U0001F1F9];   # HLRT
  81 $RI_R_End    = \U0001F1FA;                                      # U
  82 $RI_S_End    = [\U0001F1E6 \U0001F1EA \U0001F1EC];              # AEG
  83 $RI_T_End    = \U0001F1F7;                                      # R
  84 $RI_U_End    = \U0001F1F8;                                      # S
  85 $RI_V_End    = \U0001F1F3;                                      # N
  86 $RI_Z_End    = \U0001F1E6;                                      # A
  87
  88 # Special character classes for people & body part emoji:
  89 # Subsets of $Extend:
  90 $ZWJ = \u200D;
  91 $EmojiVar = \uFE0F;
  92 # The following are subsets of \p{Word_Break = Other}
  93 $EmojiForSeqs = [\u2764 \U0001F466-\U0001F469 \U0001F48B];
  94 $EmojiForMods = [\u261D \u270A-\u270C \U0001F385 \U0001F3C3-\U0001F3C4 \U0001F3C7 \U0001F3CA \U0001F442-\U0001F443 \U0001F446-\U0001F450 \U0001F466-\U0001F469 \U0001F46E-\U0001F478 \U0001F47C \U0001F481-\U0001F483 \U0001F485-\U0001F487 \U0001F4AA \U0001F596 \U0001F645-\U0001F647 \U0001F64B-\U0001F64F \U0001F6A3 \U0001F6B4-\U0001F6B6 \U0001F6C0];
  95 $EmojiMods = [\U0001F3FB-\U0001F3FF];
  96
  97 #   Dictionary character set, for triggering language-based break engines. Currently
  98 #   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
  99 #   5.0 or later as the definition of Complex_Context was corrected to include all
 100 #   characters requiring dictionary break.
 101
 102 $Control        = [\p{Grapheme_Cluster_Break = Control}];
 103 $HangulSyllable = [\uac00-\ud7a3];
 104 $ComplexContext = [:LineBreak = Complex_Context:];
 105 $KanaKanji      = [$Han $Hiragana $Katakana];
 106 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
 107 $dictionary     = [$ComplexContext $dictionaryCJK];
 108
 109 # leave CJK scripts out of ALetterPlus
 110 $ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
 111
 112
 113 #
 114 #  Rules 4    Ignore Format and Extend characters,
 115 #             except when they appear at the beginning of a region of text.
 116 #
 117 # TODO: check if handling of katakana in dictionary makes rules incorrect/void
 118 $KatakanaEx           = $Katakana           ($Extend |  $Format)*;
 119 $Hebrew_LetterEx      = $Hebrew_Letter      ($Extend |  $Format)*;
 120 $ALetterEx            = $ALetterPlus        ($Extend |  $Format)*;
 121 $Single_QuoteEx       = $Single_Quote       ($Extend |  $Format)*;
 122 $Double_QuoteEx       = $Double_Quote       ($Extend |  $Format)*;
 123 $MidNumLetEx          = $MidNumLet          ($Extend |  $Format)*;
 124 $MidLetterEx          = $MidLetter          ($Extend |  $Format)*;
 125 $MidNumEx             = $MidNum             ($Extend |  $Format)*;
 126 $NumericEx            = $Numeric            ($Extend |  $Format)*;
 127 $ExtendNumLetEx       = $ExtendNumLet       ($Extend |  $Format)*;
 128
 129 $Ideographic    = [\p{Ideographic}];
 130 $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
 131 $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
 132
 133 ## -------------------------------------------------
 134
 135 !!forward;
 136
 137
 138 # Rule 3 - CR x LF
 139 #
 140 $CR $LF;
 141
 142 # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
 143 #          of a region of Text.   The rule here comes into play when the start of text
 144 #          begins with a group of Format chars, or with a "word" consisting of a single
 145 #          char that is not in any of the listed word break categories followed by
 146 #          format char(s), or is not a CJK dictionary character.
 147 [^$CR $LF $Newline]? ($Extend |  $Format)+;
 148
 149 $NumericEx {100};
 150 $ALetterEx {200};
 151 $HangulSyllable {200};
 152 $Hebrew_LetterEx{200};
 153 $KatakanaEx {400};       # note:  these status values override those from rule 5
 154 $HiraganaEx {400};       #        by virtue of being numerically larger.
 155 $IdeographicEx {400};    #
 156
 157 #
 158 # rule 5
 159 #    Do not break between most letters.
 160 #
 161 ($ALetterEx | $Hebrew_LetterEx)  ($ALetterEx | $Hebrew_LetterEx) {200};
 162
 163 # rule 6 and 7
 164 ($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};
 165
 166 # rule 7a
 167 $Hebrew_LetterEx $Single_QuoteEx {200};
 168
 169 # rule 7b and 7c
 170 $Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
 171
 172 # rule 8
 173
 174 $NumericEx $NumericEx {100};
 175
 176 # rule 9
 177
 178 ($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};
 179
 180 # rule 10
 181
 182 $NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
 183
 184 # rule 11 and 12
 185
 186 $NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
 187
 188 # rule 13
 189 # to be consistent with $KanaKanji $KanaKanhi, changed
 190 # from 300 to 400.
 191 # See also TestRuleStatus in intltest/rbbiapts.cpp
 192 $KatakanaEx  $KatakanaEx {400};
 193
 194 # rule 13a/b
 195
 196 $ALetterEx       $ExtendNumLetEx {200};    #  (13a)
 197 $Hebrew_LetterEx $ExtendNumLetEx {200};    #  (13a)
 198 $NumericEx       $ExtendNumLetEx {100};    #  (13a)
 199 $KatakanaEx      $ExtendNumLetEx {400};    #  (13a)
 200 $ExtendNumLetEx  $ExtendNumLetEx {200};    #  (13a)
 201
 202 $ExtendNumLetEx  $ALetterEx      {200};    #  (13b)
 203 $ExtendNumLetEx  $Hebrew_Letter  {200};    #  (13b)
 204 $ExtendNumLetEx  $NumericEx      {100};    #  (13b)
 205 $ExtendNumLetEx  $KatakanaEx     {400};    #  (13b)
 206
 207 # rule 13c
 208
 209 $RI_A ($Extend|$Format)* $RI_A_End ($Extend|$Format)*;
 210 $RI_B ($Extend|$Format)* $RI_B_End ($Extend|$Format)*;
 211 $RI_C ($Extend|$Format)* $RI_C_End ($Extend|$Format)*;
 212 $RI_D ($Extend|$Format)* $RI_D_End ($Extend|$Format)*;
 213 $RI_E ($Extend|$Format)* $RI_E_End ($Extend|$Format)*;
 214 $RI_F ($Extend|$Format)* $RI_F_End ($Extend|$Format)*;
 215 $RI_G ($Extend|$Format)* $RI_G_End ($Extend|$Format)*;
 216 $RI_H ($Extend|$Format)* $RI_H_End ($Extend|$Format)*;
 217 $RI_I ($Extend|$Format)* $RI_I_End ($Extend|$Format)*;
 218 $RI_J ($Extend|$Format)* $RI_J_End ($Extend|$Format)*;
 219 $RI_K ($Extend|$Format)* $RI_K_End ($Extend|$Format)*;
 220 $RI_M ($Extend|$Format)* $RI_M_End ($Extend|$Format)*;
 221 $RI_N ($Extend|$Format)* $RI_N_End ($Extend|$Format)*;
 222 $RI_P ($Extend|$Format)* $RI_P_End ($Extend|$Format)*;
 223 $RI_R ($Extend|$Format)* $RI_R_End ($Extend|$Format)*;
 224 $RI_S ($Extend|$Format)* $RI_S_End ($Extend|$Format)*;
 225 $RI_T ($Extend|$Format)* $RI_T_End ($Extend|$Format)*;
 226 $RI_U ($Extend|$Format)* $RI_U_End ($Extend|$Format)*;
 227 $RI_V ($Extend|$Format)* $RI_V_End ($Extend|$Format)*;
 228 $RI_Z ($Extend|$Format)* $RI_Z_End ($Extend|$Format)*;
 229
 230 # Special forward rules for people & body part emoji:
 231 # don't break $ZWJ from subsequent $EmojiForSeqs; don't break between relevant emoji and $EmojiMods
 232 $ZWJ $EmojiForSeqs;
 233 $EmojiForMods $EmojiVar? $EmojiMods;
 234
 235 # special handling for CJK characters: chain for later dictionary segmentation
 236 $HangulSyllable $HangulSyllable {200};
 237 $KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
 238
 239
 240 ## -------------------------------------------------
 241
 242 !!reverse;
 243
 244 $BackHebrew_LetterEx      = ($Format | $Extend)* $Hebrew_Letter;
 245 $BackALetterEx            = ($Format | $Extend)* $ALetterPlus;
 246 $BackSingle_QuoteEx       = ($Format | $Extend)* $Single_Quote;
 247 $BackDouble_QuoteEx       = ($Format | $Extend)* $Double_Quote;
 248 $BackMidNumLetEx          = ($Format | $Extend)* $MidNumLet;
 249 $BackNumericEx            = ($Format | $Extend)* $Numeric;
 250 $BackMidNumEx             = ($Format | $Extend)* $MidNum;
 251 $BackMidLetterEx          = ($Format | $Extend)* $MidLetter;
 252 $BackKatakanaEx           = ($Format | $Extend)* $Katakana;
 253 $BackHiraganaEx           = ($Format | $Extend)* $Hiragana;
 254 $BackExtendNumLetEx       = ($Format | $Extend)* $ExtendNumLet;
 255
 256 # rule 3
 257 $LF $CR;
 258
 259 # rule 4
 260 ($Format | $Extend)*  [^$CR $LF $Newline]?;
 261
 262 # rule 5
 263
 264 ($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
 265
 266 # rule 6 and 7
 267
 268 ($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
 269
 270 # rule 7a
 271 $BackSingle_QuoteEx $BackHebrew_LetterEx;
 272
 273 # Rule 7b and 7c
 274 $BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
 275
 276 # rule 8
 277
 278 $BackNumericEx $BackNumericEx;
 279
 280 # rule 9
 281
 282 $BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
 283
 284 # rule 10
 285
 286 ($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
 287
 288 # rule 11 and 12
 289
 290 $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
 291
 292 # rule 13
 293
 294 $BackKatakanaEx $BackKatakanaEx;
 295
 296 # rules 13 a/b
 297 #
 298 $BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
 299 ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
 300
 301 # rule 13c
 302
 303 ($Format|$Extend)* $RI_A_End ($Format|$Extend)* $RI_A;
 304 ($Format|$Extend)* $RI_B_End ($Format|$Extend)* $RI_B;
 305 ($Format|$Extend)* $RI_C_End ($Format|$Extend)* $RI_C;
 306 ($Format|$Extend)* $RI_D_End ($Format|$Extend)* $RI_D;
 307 ($Format|$Extend)* $RI_E_End ($Format|$Extend)* $RI_E;
 308 ($Format|$Extend)* $RI_F_End ($Format|$Extend)* $RI_F;
 309 ($Format|$Extend)* $RI_G_End ($Format|$Extend)* $RI_G;
 310 ($Format|$Extend)* $RI_H_End ($Format|$Extend)* $RI_H;
 311 ($Format|$Extend)* $RI_I_End ($Format|$Extend)* $RI_I;
 312 ($Format|$Extend)* $RI_J_End ($Format|$Extend)* $RI_J;
 313 ($Format|$Extend)* $RI_K_End ($Format|$Extend)* $RI_K;
 314 ($Format|$Extend)* $RI_M_End ($Format|$Extend)* $RI_M;
 315 ($Format|$Extend)* $RI_N_End ($Format|$Extend)* $RI_N;
 316 ($Format|$Extend)* $RI_P_End ($Format|$Extend)* $RI_P;
 317 ($Format|$Extend)* $RI_R_End ($Format|$Extend)* $RI_R;
 318 ($Format|$Extend)* $RI_S_End ($Format|$Extend)* $RI_S;
 319 ($Format|$Extend)* $RI_T_End ($Format|$Extend)* $RI_T;
 320 ($Format|$Extend)* $RI_U_End ($Format|$Extend)* $RI_U;
 321 ($Format|$Extend)* $RI_V_End ($Format|$Extend)* $RI_V;
 322 ($Format|$Extend)* $RI_Z_End ($Format|$Extend)* $RI_Z;
 323
 324 # Special reverse rules for people & body part emoji:
 325 # don't break $ZWJ from subsequent $EmojiForSeqs; don't break between relevant emoji and $EmojiMods
 326 $EmojiForSeqs $ZWJ;
 327 $EmojiMods $EmojiVar? $EmojiForMods;
 328
 329 # special handling for CJK characters: chain for later dictionary segmentation
 330 $HangulSyllable $HangulSyllable;
 331 $KanaKanji $KanaKanji; #different rule status if both kanji and kana found
 332
 333 ## -------------------------------------------------
 334
 335 !!safe_reverse;
 336
 337 # rule 3
 338 ($Extend | $Format)+ .?;
 339
 340 # rule 6
 341 ($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
 342
 343 # rule 7b
 344 $Double_Quote $BackHebrew_LetterEx;
 345
 346
 347 # rule 11
 348 ($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
 349
 350 # For dictionary-based break
 351 $dictionary $dictionary;
 352
 353 ## -------------------------------------------------
 354
 355 !!safe_forward;
 356
 357 # rule 4
 358 ($Extend | $Format)+ .?;
 359
 360 # rule 6
 361 ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
 362
 363 # rule 7b
 364 $Double_QuoteEx $Hebrew_LetterEx;
 365
 366 # rule 11
 367 ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
 368
 369 # For dictionary-based break
 370 $dictionary $dictionary;