icuSources/data/brkitr/word_POSIX.txt

   1 #
   2 # Copyright (C) 2002-2006, International Business Machines Corporation
   3 # and others. All Rights Reserved.
   4 #
   5 # file:  word.txt
   6 #
   7 # ICU Word Break Rules, POSIX locale.
   8 #      See Unicode Standard Annex #29.
   9 #      These rules are based on Unicode Version 5.0 0
  10 #        Includes post Unicode 5.0 change to treat Japanese half width voicing marks
  11 #        as Extend
  12 #
  13 # Note:  Updates to word.txt will usually need to be merged into
  14 #        word_POSIX.txt and word_ja.txt also.
  15
  16 ##############################################################################
  17 #
  18 #  Character class definitions from TR 29
  19 #
  20 ##############################################################################
  21
  22 !!chain;
  23
  24
  25 #
  26 #  Character Class Definitions.
  27 #
  28
  29 $VoiceMarks   = [\uff9e\uff9f];
  30 $Format       = [\p{Word_Break = Format}];
  31 $Katakana     = [\p{Word_Break = Katakana}-$VoiceMarks];
  32 $ALetter      = [\p{Word_Break = ALetter}];
  33 $MidLetter    = [\p{Word_Break = MidLetter} - [ \:]];
  34 $MidNum       = [\p{Word_Break = MidNum}];
  35 $Numeric      = [\p{Word_Break = Numeric}];
  36 $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
  37
  38
  39 $CR             = \u000d;
  40 $LF             = \u000a;
  41 $Extend         = [\p{Grapheme_Cluster_Break = Extend}$VoiceMarks];
  42 $Control        = [\p{Grapheme_Cluster_Break = Control}];
  43
  44 #   Dictionary character set, for triggering language-based break engines. Currently
  45 #   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
  46 #   5.0 or later as the definition of Complex_Context was corrected to include all
  47 #   characters requiring dictionary break.
  48
  49 $dictionary   = [:LineBreak = Complex_Context:];
  50 $ALetterPlus  = [$ALetter [$dictionary-$Extend-$Control]];
  51
  52
  53 #
  54 #  Rules 3    Grapheme Clusters behave like their first char.
  55 #  Rule  4    Ignore trailing Format characters  (Also see note in TR 29)
  56 #
  57 $KatakanaEx     = $Katakana     ($Extend |  $Format)*;
  58 $ALetterEx      = $ALetterPlus  ($Extend |  $Format)*;
  59 $MidLetterEx    = $MidLetter    ($Extend |  $Format)*;
  60 $MidNumEx       = $MidNum       ($Extend |  $Format)*;
  61 $NumericEx      = $Numeric      ($Extend |  $Format)*;
  62 $ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;
  63
  64 $Hiragana       = [:Hiragana:];
  65 $Ideographic    = [:IDEOGRAPHIC:];
  66 $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
  67 $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
  68
  69 ## -------------------------------------------------
  70
  71 !!forward;
  72
  73
  74 # Rule 3 - CR x LF
  75 #          see character breaks.
  76
  77 $CR $LF  ($Extend | $Format)*;
  78
  79 # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
  80 #          of a region of Text.   The rule here comes into play when the start of text
  81 #          begins with a group of Format chars, or with a "word" consisting of a single
  82 #          char that is not in any of the listed word break categories followed by
  83 #          format char(s).
  84 .? ($Extend |  $Format)+;
  85
  86
  87 $NumericEx {100};
  88 $ALetterEx {200};
  89 $KatakanaEx {300};
  90 $HiraganaEx {300};
  91 $IdeographicEx {400};
  92
  93 # rule 5
  94
  95 $ALetterEx $ALetterEx {200};
  96
  97 # rule 6 and 7
  98 $ALetterEx $MidLetterEx $ALetterEx {200};
  99
 100 # rule 8
 101
 102 $NumericEx $NumericEx {100};
 103
 104 # rule 9
 105
 106 $ALetterEx $Format* $NumericEx {200};
 107
 108 # rule 10
 109
 110 $NumericEx $ALetterEx {200};
 111
 112 # rule 11 and 12
 113
 114 $NumericEx $MidNumEx $NumericEx {100};
 115
 116 # rule 13
 117
 118 $KatakanaEx  $KatakanaEx {300};
 119
 120 # rule 13a/b
 121
 122 $ALetterEx      $ExtendNumLetEx {200};    #  (13a)
 123 $NumericEx      $ExtendNumLetEx {100};    #  (13a)
 124 $KatakanaEx     $ExtendNumLetEx {300};    #  (13a)
 125 $ExtendNumLetEx $ExtendNumLetEx{200}; #  (13a)
 126
 127 $ExtendNumLetEx $ALetterEx  {200};    #  (13b)
 128 $ExtendNumLetEx $NumericEx  {100};    #  (13b)
 129 $ExtendNumLetEx $KatakanaEx {300};    #  (13b)
 130
 131
 132
 133 ## -------------------------------------------------
 134
 135 !!reverse;
 136
 137 $BackALetterEx     = ($Format | $Extend)* $ALetterPlus;
 138 $BackNumericEx     = ($Format | $Extend)* $Numeric;
 139 $BackMidNumEx      = ($Format | $Extend)* $MidNum;
 140 $BackMidLetterEx   = ($Format | $Extend)* $MidLetter;
 141 $BackKatakanaEx    = ($Format | $Extend)* $Katakana;
 142 $BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;
 143
 144 # rule 3
 145 ($Format | $Extend)* $LF $CR;
 146
 147 # rule 4
 148 ($Format | $Extend)*  .?;
 149
 150 # rule 5
 151
 152 $BackALetterEx $BackALetterEx;
 153
 154 # rule 6 and 7
 155
 156 $BackALetterEx $BackMidLetterEx $BackALetterEx;
 157
 158
 159 # rule 8
 160
 161 $BackNumericEx $BackNumericEx;
 162
 163 # rule 9
 164
 165 $BackNumericEx $BackALetterEx;
 166
 167 # rule 10
 168
 169 $BackALetterEx $BackNumericEx;
 170
 171 # rule 11 and 12
 172
 173 $BackNumericEx $BackMidNumEx $BackNumericEx;
 174
 175 # rule 13
 176
 177 $BackKatakanaEx $BackKatakanaEx;
 178
 179 # rules 13 a/b
 180 #
 181 ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx) $BackExtendNumLetEx;
 182 $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx);
 183
 184 ## -------------------------------------------------
 185
 186 !!safe_reverse;
 187
 188 # rule 3
 189 ($Extend | $Format)+ .?;
 190
 191 # rule 6
 192 $MidLetter $BackALetterEx;
 193
 194 # rule 11
 195 $MidNum $BackNumericEx;
 196
 197 # For dictionary-based break
 198 $dictionary $dictionary;
 199
 200 ## -------------------------------------------------
 201
 202 !!safe_forward;
 203
 204 # rule 4
 205 ($Extend | $Format)+ .?;
 206
 207 # rule 6
 208 $MidLetterEx $ALetterEx;
 209
 210 # rule 11
 211 $MidNumEx $NumericEx;
 212
 213 # For dictionary-based break
 214 $dictionary $dictionary;