icuSources/data/brkitr/word_POSIX.txt

   1 #
   2 # Copyright (C) 2002-2013, International Business Machines Corporation
   3 # and others. All Rights Reserved.
   4 #
   5 # file:  word_POSIX.txt
   6 #
   7 # ICU Word Break Rules, POSIX locale.
   8 #      See Unicode Standard Annex #29.
   9 #      These rules are based on UAX #29 Revision 20 for Unicode Version 6.2
  10 #
  11 # Note:  Updates to word.txt will usually need to be merged into
  12 #        word_POSIX.txt also.
  13
  14 ##############################################################################
  15 #
  16 #  Character class definitions from TR 29
  17 #
  18 ##############################################################################
  19
  20 !!chain;
  21
  22
  23 #
  24 #  Character Class Definitions.
  25 #
  26
  27 $CR           = [\p{Word_Break = CR}];
  28 $LF           = [\p{Word_Break = LF}];
  29 $Newline      = [\p{Word_Break = Newline}];
  30 $Extend       = [\p{Word_Break = Extend}];
  31 $Format       = [\p{Word_Break = Format}];
  32 $Hiragana     = [:Hiragana:];
  33 $Katakana     = [\p{Word_Break = Katakana}];
  34 $Han          = [:Han:];
  35 $ALetter      = [\p{Word_Break = ALetter}];
  36 $MidNumLet    = [\p{Word_Break = MidNumLet} - [.]];
  37 $MidLetter    = [\p{Word_Break = MidLetter} - [\:]];
  38 $MidNum       = [\p{Word_Break = MidNum} [.]];
  39 $Numeric      = [\p{Word_Break = Numeric}];
  40 $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
  41
  42 $RI_A        = \U0001F1E6;  # Trail ERTU
  43 $RI_B        = \U0001F1E7;  # Trail EGR
  44 $RI_C        = \U0001F1E8;  # Trail AHLNZ
  45 $RI_D        = \U0001F1E9;  # Trail EK
  46 $RI_E        = \U0001F1EA;  # Trail GS
  47 $RI_F        = \U0001F1EB;  # Trail IR
  48 $RI_G        = \U0001F1EC;  # Trail BR
  49 $RI_H        = \U0001F1ED;  # Trail KU
  50 $RI_I        = \U0001F1EE;  # Trail DLNT
  51 $RI_J        = \U0001F1EF;  # Trail OP
  52 $RI_K        = \U0001F1F0;  # Trail R
  53 $RI_L        = \U0001F1F1;  # Trail B
  54 $RI_M        = \U0001F1F2;  # Trail OXY
  55 $RI_N        = \U0001F1F3;  # Trail LO
  56 $RI_P        = \U0001F1F5;  # Trail LT
  57 $RI_R        = \U0001F1F7;  # Trail OU
  58 $RI_S        = \U0001F1F8;  # Trail AEGK
  59 $RI_T        = \U0001F1F9;  # Trail HRW
  60 $RI_U        = \U0001F1FA;  # Trail AS
  61 $RI_V        = \U0001F1FB;      # Trail N
  62
  63 $RI_A_End    = [\U0001F1EA \U0001F1F7 \U0001F1F9 \U0001F1FA];   # ERTU
  64 $RI_B_End    = [\U0001F1EA \U0001F1EC \U0001F1F7];              # EGR
  65 $RI_C_End    = [\U0001F1E6 \U0001F1ED \U0001F1F1 \U0001F1F3 \U0001F1FF]; # AHLNZ
  66 $RI_D_End    = [\U0001F1EA \U0001F1F0];                         # EK
  67 $RI_E_End    = [\U0001F1EC \U0001F1F8];                         # GS
  68 $RI_F_End    = [\U0001F1EE \U0001F1F7];                         # IR
  69 $RI_G_End    = [\U0001F1E7 \U0001F1F7];                         # BR
  70 $RI_H_End    = [\U0001F1F0 \U0001F1FA];                         # KU
  71 $RI_I_End    = [\U0001F1E9 \U0001F1F1 \U0001F1F3 \U0001F1F9];   # DLNT
  72 $RI_J_End    = [\U0001F1F4 \U0001F1F5];                         # OP
  73 $RI_K_End    = \U0001F1F7;                                      # R
  74 $RI_L_End    = \U0001F1E7;                                      # B
  75 $RI_M_End    = [\U0001F1F4 \U0001F1FD \U0001F1FE];              # OXY
  76 $RI_N_End    = [\U0001F1F1 \U0001F1F4];                         # LO
  77 $RI_P_End    = [\U0001F1F1 \U0001F1F9];                         # LT
  78 $RI_R_End    = [\U0001F1F4 \U0001F1FA];                         # OU
  79 $RI_S_End    = [\U0001F1E6 \U0001F1EA \U0001F1EC \U0001F1F0];   # AEGK
  80 $RI_T_End    = [\U0001F1ED \U0001F1F7 \U0001F1FC];              # HRW
  81 $RI_U_End    = [\U0001F1E6 \U0001F1F8];                         # AS
  82 $RI_V_End    = \U0001F1F3;                                      # N
  83
  84
  85 #   Dictionary character set, for triggering language-based break engines. Currently
  86 #   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
  87 #   5.0 or later as the definition of Complex_Context was corrected to include all
  88 #   characters requiring dictionary break.
  89
  90 $Control        = [\p{Grapheme_Cluster_Break = Control}];
  91 $HangulSyllable = [\uac00-\ud7a3];
  92 $ComplexContext = [:LineBreak = Complex_Context:];
  93 $KanaKanji      = [$Han $Hiragana $Katakana];
  94 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
  95 $dictionary     = [$ComplexContext $dictionaryCJK];
  96
  97 # leave CJK scripts out of ALetterPlus
  98 $ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
  99
 100
 101 #
 102 #  Rules 4    Ignore Format and Extend characters,
 103 #             except when they appear at the beginning of a region of text.
 104 #
 105 # TODO: check if handling of katakana in dictionary makes rules incorrect/void
 106 $KatakanaEx     = $Katakana     ($Extend |  $Format)*;
 107 $ALetterEx      = $ALetterPlus  ($Extend |  $Format)*;
 108 $MidNumLetEx    = $MidNumLet    ($Extend |  $Format)*;
 109 $MidLetterEx    = $MidLetter    ($Extend |  $Format)*;
 110 $MidNumEx       = $MidNum       ($Extend |  $Format)*;
 111 $NumericEx      = $Numeric      ($Extend |  $Format)*;
 112 $ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;
 113
 114 $Ideographic    = [\p{Ideographic}];
 115 $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
 116 $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
 117
 118 ## -------------------------------------------------
 119
 120 !!forward;
 121
 122
 123 # Rule 3 - CR x LF
 124 #
 125 $CR $LF;
 126
 127 # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
 128 #          of a region of Text.   The rule here comes into play when the start of text
 129 #          begins with a group of Format chars, or with a "word" consisting of a single
 130 #          char that is not in any of the listed word break categories followed by
 131 #          format char(s), or is not a CJK dictionary character.
 132 [^$CR $LF $Newline]? ($Extend |  $Format)+;
 133
 134 $NumericEx {100};
 135 $ALetterEx {200};
 136 $HangulSyllable {200};
 137 $KatakanaEx {400};       # note:  these status values override those from rule 5
 138 $HiraganaEx {400};       #        by virtue of being numerically larger.
 139 $IdeographicEx {400};    #
 140
 141 #
 142 # rule 5
 143 #    Do not break between most letters.
 144 #
 145 $ALetterEx $ALetterEx {200};
 146
 147 # rule 6 and 7
 148 $ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};
 149
 150 # rule 8
 151
 152 $NumericEx $NumericEx {100};
 153
 154 # rule 9
 155
 156 $ALetterEx $NumericEx {200};
 157
 158 # rule 10
 159
 160 $NumericEx $ALetterEx {200};
 161
 162 # rule 11 and 12
 163
 164 $NumericEx ($MidNumEx | $MidNumLetEx) $NumericEx {100};
 165
 166 # rule 13
 167 # to be consistent with $KanaKanji $KanaKanhi, changed
 168 # from 300 to 400.
 169 # See also TestRuleStatus in intltest/rbbiapts.cpp
 170 $KatakanaEx  $KatakanaEx {400};
 171
 172 # rule 13a/b
 173
 174 $ALetterEx      $ExtendNumLetEx {200};    #  (13a)
 175 $NumericEx      $ExtendNumLetEx {100};    #  (13a)
 176 $KatakanaEx     $ExtendNumLetEx {400};    #  (13a)
 177 $ExtendNumLetEx $ExtendNumLetEx {200};    #  (13a)
 178
 179 $ExtendNumLetEx $ALetterEx  {200};    #  (13b)
 180 $ExtendNumLetEx $NumericEx  {100};    #  (13b)
 181 $ExtendNumLetEx $KatakanaEx {400};    #  (13b)
 182
 183 # rule 13c
 184
 185 $RI_A ($Extend|$Format)* $RI_A_End ($Extend|$Format)*;
 186 $RI_B ($Extend|$Format)* $RI_B_End ($Extend|$Format)*;
 187 $RI_C ($Extend|$Format)* $RI_C_End ($Extend|$Format)*;
 188 $RI_D ($Extend|$Format)* $RI_D_End ($Extend|$Format)*;
 189 $RI_E ($Extend|$Format)* $RI_E_End ($Extend|$Format)*;
 190 $RI_F ($Extend|$Format)* $RI_F_End ($Extend|$Format)*;
 191 $RI_G ($Extend|$Format)* $RI_G_End ($Extend|$Format)*;
 192 $RI_H ($Extend|$Format)* $RI_H_End ($Extend|$Format)*;
 193 $RI_I ($Extend|$Format)* $RI_I_End ($Extend|$Format)*;
 194 $RI_J ($Extend|$Format)* $RI_J_End ($Extend|$Format)*;
 195 $RI_K ($Extend|$Format)* $RI_K_End ($Extend|$Format)*;
 196 $RI_L ($Extend|$Format)* $RI_L_End ($Extend|$Format)*;
 197 $RI_M ($Extend|$Format)* $RI_M_End ($Extend|$Format)*;
 198 $RI_N ($Extend|$Format)* $RI_N_End ($Extend|$Format)*;
 199 $RI_P ($Extend|$Format)* $RI_P_End ($Extend|$Format)*;
 200 $RI_R ($Extend|$Format)* $RI_R_End ($Extend|$Format)*;
 201 $RI_S ($Extend|$Format)* $RI_S_End ($Extend|$Format)*;
 202 $RI_T ($Extend|$Format)* $RI_T_End ($Extend|$Format)*;
 203 $RI_U ($Extend|$Format)* $RI_U_End ($Extend|$Format)*;
 204 $RI_V ($Extend|$Format)* $RI_V_End ($Extend|$Format)*;
 205
 206 # special handling for CJK characters: chain for later dictionary segmentation
 207 $HangulSyllable $HangulSyllable {200};
 208 $KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
 209
 210
 211 ## -------------------------------------------------
 212
 213 !!reverse;
 214
 215 $BackALetterEx            = ($Format | $Extend)* $ALetterPlus;
 216 $BackMidNumLetEx          = ($Format | $Extend)* $MidNumLet;
 217 $BackNumericEx            = ($Format | $Extend)* $Numeric;
 218 $BackMidNumEx             = ($Format | $Extend)* $MidNum;
 219 $BackMidLetterEx          = ($Format | $Extend)* $MidLetter;
 220 $BackKatakanaEx           = ($Format | $Extend)* $Katakana;
 221 $BackHiraganaEx           = ($Format | $Extend)* $Hiragana;
 222 $BackExtendNumLetEx       = ($Format | $Extend)* $ExtendNumLet;
 223
 224 # rule 3
 225 $LF $CR;
 226
 227 # rule 4
 228 ($Format | $Extend)*  [^$CR $LF $Newline]?;
 229
 230 # rule 5
 231
 232 $BackALetterEx $BackALetterEx;
 233
 234 # rule 6 and 7
 235
 236 $BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;
 237
 238
 239 # rule 8
 240
 241 $BackNumericEx $BackNumericEx;
 242
 243 # rule 9
 244
 245 $BackNumericEx $BackALetterEx;
 246
 247 # rule 10
 248
 249 $BackALetterEx $BackNumericEx;
 250
 251 # rule 11 and 12
 252
 253 $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx) $BackNumericEx;
 254
 255 # rule 13
 256
 257 $BackKatakanaEx $BackKatakanaEx;
 258
 259 # rules 13 a/b
 260 #
 261 $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
 262 ($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
 263
 264 # rule 13c
 265
 266 ($Format|$Extend)* $RI_A_End ($Format|$Extend)* $RI_A;
 267 ($Format|$Extend)* $RI_B_End ($Format|$Extend)* $RI_B;
 268 ($Format|$Extend)* $RI_C_End ($Format|$Extend)* $RI_C;
 269 ($Format|$Extend)* $RI_D_End ($Format|$Extend)* $RI_D;
 270 ($Format|$Extend)* $RI_E_End ($Format|$Extend)* $RI_E;
 271 ($Format|$Extend)* $RI_F_End ($Format|$Extend)* $RI_F;
 272 ($Format|$Extend)* $RI_G_End ($Format|$Extend)* $RI_G;
 273 ($Format|$Extend)* $RI_H_End ($Format|$Extend)* $RI_H;
 274 ($Format|$Extend)* $RI_I_End ($Format|$Extend)* $RI_I;
 275 ($Format|$Extend)* $RI_J_End ($Format|$Extend)* $RI_J;
 276 ($Format|$Extend)* $RI_K_End ($Format|$Extend)* $RI_K;
 277 ($Format|$Extend)* $RI_L_End ($Format|$Extend)* $RI_L;
 278 ($Format|$Extend)* $RI_M_End ($Format|$Extend)* $RI_M;
 279 ($Format|$Extend)* $RI_N_End ($Format|$Extend)* $RI_N;
 280 ($Format|$Extend)* $RI_P_End ($Format|$Extend)* $RI_P;
 281 ($Format|$Extend)* $RI_R_End ($Format|$Extend)* $RI_R;
 282 ($Format|$Extend)* $RI_S_End ($Format|$Extend)* $RI_S;
 283 ($Format|$Extend)* $RI_T_End ($Format|$Extend)* $RI_T;
 284 ($Format|$Extend)* $RI_U_End ($Format|$Extend)* $RI_U;
 285 ($Format|$Extend)* $RI_V_End ($Format|$Extend)* $RI_V;
 286
 287 # special handling for CJK characters: chain for later dictionary segmentation
 288 $HangulSyllable $HangulSyllable;
 289 $KanaKanji $KanaKanji; #different rule status if both kanji and kana found
 290
 291 ## -------------------------------------------------
 292
 293 !!safe_reverse;
 294
 295 # rule 3
 296 ($Extend | $Format)+ .?;
 297
 298 # rule 6
 299 ($MidLetter | $MidNumLet) $BackALetterEx;
 300
 301 # rule 11
 302 ($MidNum | $MidNumLet) $BackNumericEx;
 303
 304 # For dictionary-based break
 305 $dictionary $dictionary;
 306
 307 ## -------------------------------------------------
 308
 309 !!safe_forward;
 310
 311 # rule 4
 312 ($Extend | $Format)+ .?;
 313
 314 # rule 6
 315 ($MidLetterEx | $MidNumLetEx) $ALetterEx;
 316
 317 # rule 11
 318 ($MidNumEx | $MidNumLetEx) $NumericEx;
 319
 320 # For dictionary-based break
 321 $dictionary $dictionary;