icuSources/test/testdata/wordNLLTu8.txt

   1 ## wordNLLTu8.txt
   2 ## The following corresponds to source file from CoreLP of 2018-Jan-17:
   3 ##   CoreNLP/Tagger.subproj/Tagger.subproj/Source/Tagging/Data/word.txt
   4 ## and binary files in CoreLP of 2018-Feb-03:
   5 ##   CoreNLP.framework/Resources/word.dat       [for ICU before 60, including ICU 57]
   6 ##   CoreNLP.framework/Resources/word_icu60.dat
   7 ##############################################################################
   8 #
   9 # Copyright (C) 2002-2016, International Business Machines Corporation
  10 # and others. All Rights Reserved.
  11 #
  12 # file:  word.txt
  13 #
  14 # ICU Word Break Rules
  15 #      See Unicode Standard Annex #29.
  16 #      These rules are based on UAX #29 Revision 27 for Unicode Version 8.0
  17 #      with additions from L2/16-011R3 for Emoji sequences.
  18 #
  19 # Note:  Updates to word.txt will usually need to be merged into
  20 #        word_POSIX.txt also.
  21
  22 ##############################################################################
  23 #
  24 #  Character class definitions from TR 29
  25 #
  26 ##############################################################################
  27
  28 !!chain;
  29
  30
  31 # Smart Tokenizer customization
  32 $Space              = [\u0020];
  33 $Fullstop       = [\u002E\uFE52\uFF0E];
  34 $Dash           = [\u002D\uFE63\uFF0D];
  35
  36 $Terminators    = [\u0021\u00A1\u00A1\u01C3\u01C3\u01C3\u203C\u2048\uFE15\uFF01\u003F];
  37 $Parenthesis    = [\u0028\u005B\u007B\u0029\u005D\u007D];
  38 $Punctuation    = [\u007E\u003D\u002A\u003C\u003E];
  39 $OtherSingle    = [\u0060\u0300\u030F];
  40
  41 $Hash           = [\u0023 \uFE5F \uFF03];
  42 $At             = [\u0040 \uFE6B \uFF20];
  43 $Dollar         = [\u0024];
  44
  45 $Semicolon      = [\u003B];
  46 $Colon          = [\u003A];
  47
  48 #
  49 #  Character Class Definitions.
  50 #
  51
  52 $E_Base = [\u261D\u26F9\u270A-\u270D\U0001F385\U0001F3C3-\U0001F3C4\U0001F3CA-\U0001F3CB\U0001F442-\U0001F443\U0001F446-\U0001F450\U0001F466-\U0001F469\U0001F46E\U0001F470-\U0001F478\U0001F47C\U0001F481-\U0001F483\U0001F485-\U0001F487\U0001F4AA\U0001F575\U0001F590\U0001F595-\U0001F596\U0001F645-\U0001F647\U0001F64B-\U0001F64F\U0001F6A3\U0001F6B4-\U0001F6B6\U0001F6C0\U0001F918];
  53 $E_Modifier = [\U0001F3FB-\U0001F3FF];
  54 $ZWJ = [\u200D];
  55 $GAZ = [\U0001F466-\U0001F469\U0001F48B\U0001F5E8\u2764];
  56
  57 $CR                 = [\p{Word_Break = CR}];
  58 $LF                 = [\p{Word_Break = LF}];
  59 $Newline            = [\p{Word_Break = Newline} ];
  60 $Extend             = [[\p{Word_Break = Extend}][:Block=Tags:]];
  61 $Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
  62 $Format             = [[\p{Word_Break = Format}] - [:Block=Tags:]];
  63 $Katakana           = [\p{Word_Break = Katakana}];
  64 $Hebrew_Letter      = [\p{Word_Break = Hebrew_Letter}];
  65 $ALetter            = [\p{Word_Break = ALetter}];
  66 $Single_Quote       = [\p{Word_Break = Single_Quote}\u2018\u2019];
  67 $Double_Quote       = [\p{Word_Break = Double_Quote}];
  68 $MidNumLet          = [\p{Word_Break = MidNumLet}-[$Fullstop $Single_Quote]];
  69 $MidLetter          = [\p{Word_Break = MidLetter}-[$Single_Quote]];
  70 $MidNum             = [\p{Word_Break = MidNum}];
  71 $Numeric            = [\p{Word_Break = Numeric}];
  72 $ExtendNumLet       = [\p{Word_Break = ExtendNumLet}];
  73
  74 $Han                = [:Han:];
  75 $Hiragana           = [:Hiragana:];
  76
  77
  78 #   Dictionary character set, for triggering language-based break engines. Currently
  79 #   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
  80 #   5.0 or later as the definition of Complex_Context was corrected to include all
  81 #   characters requiring dictionary break.
  82
  83 $Control        = [\p{Grapheme_Cluster_Break = Control}];
  84 $HangulSyllable = [\uac00-\ud7a3];
  85 $ComplexContext = [:LineBreak = Complex_Context:];
  86 $KanaKanji      = [$Han $Hiragana $Katakana];
  87 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
  88 $dictionary     = [$ComplexContext $dictionaryCJK];
  89
  90 # leave CJK scripts out of ALetterPlus
  91 $ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
  92
  93 #
  94 #  Rules 4    Ignore Format and Extend characters,
  95 #             except when they appear at the beginning of a region of text.
  96 #
  97 # TODO: check if handling of katakana in dictionary makes rules incorrect/void
  98 $KatakanaEx           = $Katakana           ($Extend |  $Format | $ZWJ)*;
  99 $Hebrew_LetterEx      = $Hebrew_Letter      ($Extend |  $Format | $ZWJ)*;
 100 $ALetterEx            = $ALetterPlus        ($Extend |  $Format | $ZWJ)*;
 101 $Single_QuoteEx       = $Single_Quote       ($Extend |  $Format | $ZWJ)*;
 102 $Double_QuoteEx       = $Double_Quote       ($Extend |  $Format | $ZWJ)*;
 103 $MidNumLetEx          = $MidNumLet          ($Extend |  $Format | $ZWJ)*;
 104 $MidLetterEx          = $MidLetter          ($Extend |  $Format | $ZWJ)*;
 105 $MidNumEx             = $MidNum             ($Extend |  $Format | $ZWJ)*;
 106 $NumericEx            = $Numeric            ($Extend |  $Format | $ZWJ)*;
 107 $ExtendNumLetEx       = $ExtendNumLet       ($Extend |  $Format | $ZWJ)*;
 108 $Regional_IndicatorEx = $Regional_Indicator ($Extend |  $Format | $ZWJ)*;
 109
 110 $Ideographic    = [\p{Ideographic}];
 111 $HiraganaEx     = $Hiragana     ($Extend |  $Format | $ZWJ)*;
 112 $IdeographicEx  = $Ideographic  ($Extend |  $Format | $ZWJ)*;
 113
 114 $Letter = [A-Za-z];
 115
 116 $Contraction = (N | n) $Single_Quote (T | t);
 117 $Yall = (Y | y) $Single_Quote (A | a) (L | l)*;
 118
 119 $Time = (0 | 1 | 2)? $Numeric $Colon (0 | 1 | 2 | 3 | 4 | 5) $Numeric;
 120 #$IPAddress = $Numeric? $Numeric? $Numeric $Fullstop $Numeric? $Numeric? $Numeric $Fullstop $Numeric? $Numeric? $Numeric $Fullstop $Numeric? $Numeric? $Numeric;
 121 $IPAddress = $Numeric? $Numeric? $Numeric $Fullstop $Numeric? $Numeric? $Numeric;
 122
 123 ## -------------------------------------------------
 124
 125 !!forward;
 126
 127 $Space* {0};
 128
 129 $Contraction {202};
 130 $Yall {202};
 131
 132 $Fullstop {20};
 133 $Dash {21};
 134 $Single_Quote {22};
 135
 136 $Fullstop $Fullstop ($Fullstop*) {40};
 137 $Dash $Dash ($Dash*) {40};
 138 $Terminators $Terminators ($Terminators*) {40};
 139 $Punctuation $Punctuation ($Punctuation*) {40};
 140 $OtherSingle $OtherSingle ($OtherSingle*) {40};
 141
 142 $Single_Quote $Single_Quote {40};
 143 $Single_Quote $Single_Quote $Single_Quote* {40};
 144
 145 $Hash ($ALetter | $Numeric)* {30};
 146 $At ($ALetter | $Numeric)* {31};
 147 $Dollar $ALetter ($ALetter | $Numeric)* {31};
 148
 149 $Time {50};
 150 $IPAddress {51};
 151
 152 $ALetter / $IPAddress;
 153 $ALetter / $Time;
 154
 155 $IPAddress / $ALetter;
 156 $Time / $ALetter;
 157
 158 ($Colon | $Semicolon) $Parenthesis {32};
 159
 160 $Fullstop $Numeric $Numeric;
 161
 162 # Rule 3 - CR x LF
 163 #
 164 $CR $LF;
 165
 166 # Rule 3c   ZWJ x GAZ.  Preceeds WB4, so no intervening Extend chars allowed.
 167 #
 168 $ZWJ $GAZ;
 169
 170
 171 # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
 172 #          of a region of Text.   The rule here comes into play when the start of text
 173 #          begins with a group of Format chars, or with a "word" consisting of a single
 174 #          char that is not in any of the listed word break categories followed by
 175 #          format char(s), or is not a CJK dictionary character.
 176 [^$CR $LF $Newline]? ($Extend |  $Format | $ZWJ)+;
 177
 178 $NumericEx {100};
 179 $ALetterEx {200};
 180 $HangulSyllable {200};
 181 $Hebrew_LetterEx{200};
 182 $KatakanaEx {400};       # note:  these status values override those from rule 5
 183 $HiraganaEx {400};       #        by virtue of being numerically larger.
 184 $IdeographicEx {400};    #
 185
 186 $E_Base ($Extend | $Format | $ZWJ)*;
 187 $E_Modifier ($Extend | $Format | $ZWJ)*;
 188 $GAZ ($Extend | $Format | $ZWJ)*;
 189
 190 #
 191 # rule 5
 192 #    Do not break between most letters.
 193 #
 194 ($ALetterEx | $Hebrew_LetterEx)  ($ALetterEx | $Hebrew_LetterEx) {200};
 195
 196 # rule 6 and 7
 197 #($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};
 198 ($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx) ($ALetterEx | $Hebrew_LetterEx) {200};
 199
 200 # rule 7a
 201 #$Hebrew_LetterEx $Single_QuoteEx {200};
 202
 203 # rule 7b and 7c
 204 $Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
 205
 206 # rule 8
 207
 208 $NumericEx $NumericEx {100};
 209
 210 # rule 9
 211
 212 ($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};
 213
 214 # rule 10
 215
 216 $NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
 217
 218 # rule 11 and 12
 219
 220 $NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
 221
 222 # rule 13
 223 # to be consistent with $KanaKanji $KanaKanhi, changed
 224 # from 300 to 400.
 225 # See also TestRuleStatus in intltest/rbbiapts.cpp
 226 $KatakanaEx  $KatakanaEx {400};
 227
 228 # rule 13a/b
 229
 230 $ALetterEx       $ExtendNumLetEx {200};    #  (13a)
 231 $Hebrew_LetterEx $ExtendNumLetEx {200};    #  (13a)
 232 $NumericEx       $ExtendNumLetEx {100};    #  (13a)
 233 $KatakanaEx      $ExtendNumLetEx {400};    #  (13a)
 234 $ExtendNumLetEx  $ExtendNumLetEx {200};    #  (13a)
 235
 236 $ExtendNumLetEx  $ALetterEx      {200};    #  (13b)
 237 $ExtendNumLetEx  $Hebrew_Letter  {200};    #  (13b)
 238 $ExtendNumLetEx  $NumericEx      {100};    #  (13b)
 239 $ExtendNumLetEx  $KatakanaEx     {400};    #  (13b)
 240
 241 # rule 13c
 242 #    Pairs of Regional Indicators stay together.
 243 #    With rule chaining disabled by ^, this rule will match exactly two of them.
 244 #    No other rule begins with a Regional_Indicator, so chaining cannot extend the match.
 245 #
 246 ^$Regional_IndicatorEx $Regional_IndicatorEx;
 247
 248 # special handling for CJK characters: chain for later dictionary segmentation
 249 $HangulSyllable $HangulSyllable {200};
 250 $KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
 251
 252 # rule 13d
 253 #    E_Base x E_Modifier
 254 #
 255 ($E_Base | $GAZ) ($Format | $Extend | $ZWJ)* $E_Modifier;
 256
 257 ## -------------------------------------------------
 258
 259 !!reverse;
 260
 261 $BackHebrew_LetterEx      = ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
 262 $BackALetterEx            = ($Format | $Extend | $ZWJ)* $ALetterPlus;
 263 $BackSingle_QuoteEx       = ($Format | $Extend | $ZWJ)* $Single_Quote;
 264 $BackDouble_QuoteEx       = ($Format | $Extend | $ZWJ)* $Double_Quote;
 265 $BackMidNumLetEx          = ($Format | $Extend | $ZWJ)* $MidNumLet;
 266 $BackNumericEx            = ($Format | $Extend | $ZWJ)* $Numeric;
 267 $BackMidNumEx             = ($Format | $Extend | $ZWJ)* $MidNum;
 268 $BackMidLetterEx          = ($Format | $Extend | $ZWJ)* $MidLetter;
 269 $BackKatakanaEx           = ($Format | $Extend | $ZWJ)* $Katakana;
 270 $BackHiraganaEx           = ($Format | $Extend | $ZWJ)* $Hiragana;
 271 $BackExtendNumLetEx       = ($Format | $Extend | $ZWJ)* $ExtendNumLet;
 272 $BackRegional_IndicatorEx = ($Format | $Extend | $ZWJ)* $Regional_Indicator;
 273
 274 # rule 3
 275 $LF $CR;
 276
 277 # Rule 3c   ZWJ x GAZ.  Preceeds WB4, so no intervening Extend chars allowed.
 278 #
 279 $GAZ $ZWJ;
 280
 281 # rule 4
 282 ($Format | $Extend | $ZWJ)*  [^$CR $LF $Newline]?;
 283
 284 # rule 5
 285
 286 ($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
 287
 288 # rule 6 and 7
 289
 290 #($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
 291 ($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx) ($BackALetterEx | $BackHebrew_LetterEx);
 292
 293 # rule 7a
 294 #$BackSingle_QuoteEx $BackHebrew_LetterEx;
 295
 296 # Rule 7b and 7c
 297 $BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
 298
 299 # rule 8
 300
 301 $BackNumericEx $BackNumericEx;
 302
 303 # rule 9
 304
 305 $BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
 306
 307 # rule 10
 308
 309 ($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
 310
 311 # rule 11 and 12
 312
 313 $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
 314
 315 # rule 13
 316
 317 $BackKatakanaEx $BackKatakanaEx;
 318
 319 # rules 13 a/b
 320 #
 321 $BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
 322 ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
 323
 324 # rule 13c
 325
 326 ^$BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
 327 ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
 328 ^$BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
 329 ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
 330
 331 $GAZ $ZWJ $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
 332 ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
 333 $GAZ $ZWJ $BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
 334 ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
 335
 336 # special handling for CJK characters: chain for later dictionary segmentation
 337 $HangulSyllable $HangulSyllable;
 338 $KanaKanji $KanaKanji; #different rule status if both kanji and kana found
 339
 340 # rule 13d
 341
 342 $E_Modifier ($Format | $Extend | $ZWJ)* ($E_Base | $GAZ);
 343
 344
 345
 346 ## -------------------------------------------------
 347
 348 !!safe_reverse;
 349
 350
 351 # rule 3
 352 ($Extend | $Format | $ZWJ)+ .?;
 353
 354 # rule 6
 355 #($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
 356 ($MidLetter | $MidNumLet) ($BackALetterEx | $BackHebrew_LetterEx);
 357
 358 # rule 7b
 359 $Double_Quote $BackHebrew_LetterEx;
 360
 361
 362 # rule 11
 363 #($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
 364 ($MidNum | $MidNumLet) $BackNumericEx;
 365
 366 # rule 13c
 367 $BackRegional_IndicatorEx*;
 368
 369 # For dictionary-based break
 370 $dictionary $dictionary;
 371
 372 ## -------------------------------------------------
 373
 374 !!safe_forward;
 375
 376
 377 # rule 4
 378 ($Extend | $Format | $ZWJ)+ .?;
 379
 380 # rule 6
 381 #($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
 382 ($MidLetterEx | $MidNumLetEx) ($ALetterEx | $Hebrew_LetterEx);
 383
 384 # rule 7b
 385 $Double_QuoteEx $Hebrew_LetterEx;
 386
 387 # rule 11
 388 #($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
 389 ($MidNumEx | $MidNumLetEx) $NumericEx;
 390
 391 # rule 13c
 392 $Regional_IndicatorEx*;
 393
 394 # For dictionary-based break
 395 $dictionary $dictionary;