ICU-400.37.tar.gz

[apple/icu.git] / icuSources / data / brkitr / word.txt

diff --git a/icuSources/data/brkitr/word.txt b/icuSources/data/brkitr/word.txt
index 1fdaa3a0a9aa33b181b5bbbf0be2f0a167545512..db045c9a7b50a37c5a4d4f0c16d3a393d0c43de7 100644 (file)
--- a/icuSources/data/brkitr/word.txt
+++ b/icuSources/data/brkitr/word.txt
@@ -1,13 +1,15 @@
 #
 #

-# Copyright (C) 2002-2004, International Business Machines Corporation 
+# Copyright (C) 2002-2008, International Business Machines Corporation 

 # and others. All Rights Reserved.
 #
 # file:  word.txt
 #
 # ICU Word Break Rules
 #      See Unicode Standard Annex #29.
 # and others. All Rights Reserved.
 #
 # file:  word.txt
 #
 # ICU Word Break Rules
 #      See Unicode Standard Annex #29.

-#      These rules are based on Version 4.1 draft, dated 2004-11-11
+#      These rules are based on UAX-29 Revision 12 for Unicode 5.1

 #
 #

+# Note:  Updates to word.txt will usually need to be merged into
+#        word_POSIX.txt and word_ja.txt also.

 
 ##############################################################################
 #
 
 ##############################################################################
 #


@@ -17,119 +19,113 @@
 
 !!chain;
 
 
 !!chain;
 

-$Katakana  = [[:Script = KATAKANA:]
-                         [:name = VERTICAL KANA REPEAT MARK:]
-                         [:name = VERTICAL KANA REPEAT WITH VOICED SOUND MARK:]
-                         [:name = VERTICAL KANA REPEAT MARK UPPER HALF:]
-                         [:name = VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HALF:]
-                         [:name = VERTICAL KANA REPEAT MARK LOWER HALF:]
-                         [:name = KATAKANA-HIRAGANA VOICED SOUND MARK:]
-                         [:name = KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK:]
-                         [:name = KATAKANA-HIRAGANA DOUBLE HYPHEN:]
-                         [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
-                         [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
-                         [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
-                         [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
-
-
-$ALetter   = [[:Alphabetic:] [:name= HEBREW PUNCTUATION GERESH:]
-                                                  - [:Ideographic:]
-                                                  - $Katakana
-                                                  - [:Script = Hiragana:]
-                                                  - [:Script = Lao:]
-                                                  - [:Grapheme_Extend = TRUE:]];
-                                                  
-$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:]
-                         [:name = HEBREW PUNCTUATION GERSHAYIM:]
-                         [:name = RIGHT SINGLE QUOTATION MARK:]
-                         [:name = HYPHENATION POINT:]
-                         [:name = COLON:]];
-
-                       
-$MidNum    = [[:LineBreak = Infix_Numeric:] - [:name = COLON:]];
-$Numeric   = [[:Nd:][:name = ARABIC DECIMAL SEPARATOR:][:name = ARABIC THOUSANDS SEPARATOR:]];
-$ExtendNumLet = [[:Connector_Punctuation:] 
-                       - [:name = KATAKANA MIDDLE DOT:]
-                       - [:name = HALFWIDTH KATAKANA MIDDLE DOT:]];
-                       
-                       

 
 #
 #  Character Class Definitions.
 
 #
 #  Character Class Definitions.

-#    The names are those from TR29.

 #
 
 #
 

-$CR             = \u000d;
-$LF             = \u000a;
-$Extend         = [[:Grapheme_Extend = TRUE:]];
-$Control        = [[:Zl:] [:Zp:] [:Cc:] [:Cf:] - $Extend];
-$Format         = [[:Cf:] - $Extend];
-$Hiragana       = [:Hiragana:];
-$Ideographic    = [:IDEOGRAPHIC:];
-
-$ALetterEx      = $ALetter     $Extend*;
-$NumericEx      = $Numeric     $Extend*;
-$MidNumEx       = $MidNum      $Extend*;
-$MidLetterEx    = $MidLetter   $Extend*;
-$KatakanaEx     = $Katakana    $Extend*;
-$ExtendNumLetEx = $ExtendNumLet $Extend*;
+$CR           = [\p{Word_Break = CR}];
+$LF           = [\p{Word_Break = LF}];
+$Newline      = [\p{Word_Break = Newline}];
+$Extend       = [\p{Word_Break = Extend}];
+$Format       = [\p{Word_Break = Format}];
+$Katakana     = [\p{Word_Break = Katakana}];
+$ALetter      = [\p{Word_Break = ALetter}];
+$MidNumLet    = [\p{Word_Break = MidNumLet}];
+$MidLetter    = [\p{Word_Break = MidLetter}];
+$MidNum       = [\p{Word_Break = MidNum}];
+$Numeric      = [\p{Word_Break = Numeric}];
+$ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
+
+
+#   Dictionary character set, for triggering language-based break engines. Currently
+#   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
+#   5.0 or later as the definition of Complex_Context was corrected to include all
+#   characters requiring dictionary break.
+
+$dictionary   = [:LineBreak = Complex_Context:];
+$Control        = [\p{Grapheme_Cluster_Break = Control}]; 
+$ALetterPlus  = [$ALetter [$dictionary-$Extend-$Control]];   # Note:  default ALetter does not
+                                                             #  include the dictionary characters.
+
+#
+#  Rules 4    Ignore Format and Extend characters, 
+#             except when they appear at the beginning of a region of text.
+#
+$KatakanaEx     = $Katakana     ($Extend |  $Format)*;
+$ALetterEx      = $ALetterPlus  ($Extend |  $Format)*;
+$MidNumLetEx    = $MidNumLet    ($Extend |  $Format)*;
+$MidLetterEx    = $MidLetter    ($Extend |  $Format)*;
+$MidNumEx       = $MidNum       ($Extend |  $Format)*;
+$NumericEx      = $Numeric      ($Extend |  $Format)*;
+$ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;
+
+$Hiragana       = [\p{script=Hiragana}];
+$Ideographic    = [\p{Ideographic}];
+$HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
+$IdeographicEx  = $Ideographic  ($Extend |  $Format)*;

 
 ## -------------------------------------------------
 
 !!forward;
 
 
 
 ## -------------------------------------------------
 
 !!forward;
 
 

-# Rule 3 - don't break grapheme clusters.
-# see character breaks
-
+# Rule 3 - CR x LF
+#

 $CR $LF;
 $CR $LF;

-#[^$Control] $Extend*;
-#$NumericEx $Extend* {100};
-#$ALetterEx $Extend* {200};
-[^$Control] $Extend+;
+
+# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
+#          of a region of Text.   The rule here comes into play when the start of text
+#          begins with a group of Format chars, or with a "word" consisting of a single
+#          char that is not in any of the listed word break categories followed by
+#          format char(s).
+[^$CR $LF $Newline]? ($Extend |  $Format)+;
+

 $NumericEx {100};
 $ALetterEx {200};
 $NumericEx {100};
 $ALetterEx {200};

-$KatakanaEx {300};
+$KatakanaEx {300};       # note:  these status values override those from rule 5
+$HiraganaEx {300};       #        by virtual of being numerically larger.
+$IdeographicEx {400};    #

 
 

+#

 # rule 5
 # rule 5

-
-$ALetterEx $Format* $ALetterEx {200};
+#    Do not break between most letters.
+#
+$ALetterEx $ALetterEx {200};

 
 # rule 6 and 7
 
 # rule 6 and 7

-$ALetterEx $Format* $MidLetterEx  $Format* $ALetterEx {200};
+$ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};

 
 # rule 8
 
 
 # rule 8
 

-$NumericEx $Format* $NumericEx {100};
+$NumericEx $NumericEx {100};

 
 # rule 9
 
 
 # rule 9
 

-$ALetterEx $Format* $NumericEx {200};
+$ALetterEx $NumericEx {200};

 
 # rule 10
 
 
 # rule 10
 

-$NumericEx $Format* $ALetterEx {200};
+$NumericEx $ALetterEx {200};

 
 # rule 11 and 12 
 
 
 # rule 11 and 12 
 

-$NumericEx $Format* $MidNumEx $Format* $NumericEx {100};
+$NumericEx ($MidNumEx | $MidNumLetEx) $NumericEx {100};

 
 # rule 13
 
 
 # rule 13
 

-$KatakanaEx  $Format* $KatakanaEx {300};
-$Hiragana    $Extend* {300};
-$Ideographic $Extend* {400};
+$KatakanaEx  $KatakanaEx {300};

 
 # rule 13a/b
 
 
 # rule 13a/b
 

-$ALetterEx  $Format* $ExtendNumLetEx {200};    #  (13a)
-$NumericEx  $Format* $ExtendNumLetEx {100};    #  (13a)
-$KatakanaEx $Format* $ExtendNumLetEx {300};    #  (13a)
-$ExtendNumLetEx $Format* $ExtendNumLetEx{200}; #  (13a)
+$ALetterEx      $ExtendNumLetEx {200};    #  (13a)
+$NumericEx      $ExtendNumLetEx {100};    #  (13a)
+$KatakanaEx     $ExtendNumLetEx {300};    #  (13a)
+$ExtendNumLetEx $ExtendNumLetEx {200};    #  (13a)

 
 

-$ExtendNumLetEx $Format* $ALetterEx  {200};    #  (13b)
-$ExtendNumLetEx $Format* $NumericEx  {100};    #  (13b)
-$ExtendNumLetEx $Format* $KatakanaEx {300};    #  (13b)
+$ExtendNumLetEx $ALetterEx  {200};    #  (13b)
+$ExtendNumLetEx $NumericEx  {100};    #  (13b)
+$ExtendNumLetEx $KatakanaEx {300};    #  (13b)

  
 
 
  
 
 


@@ -137,100 +133,82 @@ $ExtendNumLetEx $Format* $KatakanaEx {300};    #  (13b)
 
 !!reverse;
 
 
 !!reverse;
 

-$BackALetterEx     = $Extend* $ALetter;
-$BackNumericEx     = $Extend* $Numeric;
-$BackMidNumEx      = $Extend* $MidNum;
-$BackMidLetterEx   = $Extend* $MidLetter;
-$BackKatakanaEx    = $Extend* $Katakana;
-$BackExtendNumLetEx= $Extend* $ExtendNumLet;
+$BackALetterEx     = ($Format | $Extend)* $ALetterPlus;
+$BackMidNumLetEx   = ($Format | $Extend)* $MidNumLet;
+$BackNumericEx     = ($Format | $Extend)* $Numeric;
+$BackMidNumEx      = ($Format | $Extend)* $MidNum;
+$BackMidLetterEx   = ($Format | $Extend)* $MidLetter;
+$BackKatakanaEx    = ($Format | $Extend)* $Katakana;
+$BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;

 
 

+# rule 3

 $LF $CR;
 
 $LF $CR;
 

-# see character breaks
-
-$Extend* [^$Control];
+# rule 4
+($Format | $Extend)*  [^$CR $LF $Newline]?;

 
 # rule 5
 
 
 # rule 5
 

-$BackALetterEx $Format* $BackALetterEx;
+$BackALetterEx $BackALetterEx;

 
 # rule 6 and 7
 
 
 # rule 6 and 7
 

-$BackALetterEx $Format* $BackMidLetterEx $Format* $BackALetterEx;
+$BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;

 
 
 # rule 8
 
 
 
 # rule 8
 

-$BackNumericEx $Format* $BackNumericEx;
+$BackNumericEx $BackNumericEx;

 
 # rule 9
 
 
 # rule 9
 

-$BackNumericEx  $Format* $BackALetterEx;
+$BackNumericEx $BackALetterEx;

 
 # rule 10
 
 
 # rule 10
 

-$BackALetterEx $Format* $BackNumericEx;
+$BackALetterEx $BackNumericEx;

 
 # rule 11 and 12
 
 
 # rule 11 and 12
 

-$BackNumericEx $Format* $BackMidNumEx $Format* $BackNumericEx;
+$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx) $BackNumericEx;

 
 # rule 13
 
 
 # rule 13
 

-$BackKatakanaEx $Format* $BackKatakanaEx;
+$BackKatakanaEx $BackKatakanaEx;

 
 # rules 13 a/b
 #
 
 # rules 13 a/b
 #

-($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx) $Format* $BackExtendNumLetEx; 
-$BackExtendNumLetEx $Format* ($BackALetterEx | $BackNumericEx | $BackKatakanaEx);        
+$BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
+($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; 

 
 ## -------------------------------------------------
 
 !!safe_reverse;
 
 # rule 3
 
 ## -------------------------------------------------
 
 !!safe_reverse;
 
 # rule 3

-$Extend+ [^$Extend];
-$Extend+;               # comes into play when buffer _begins_ with an $Extend+.
-
-# rule 4
-$Format+ $BackALetterEx;
-$Format+ $BackNumericEx;
-$Format+ $BackMidLetterEx;
-$Format+ $BackMidNumEx;
-$Format+ $BackKatakanaEx;
-$Format+ $BackExtendNumLetEx;
-
+($Extend | $Format)+ .?;

 
 # rule 6
 
 # rule 6

-$MidLetter $Format* $BackALetterEx;
+($MidLetter | $MidNumLet) $BackALetterEx;

 
 # rule 11
 
 # rule 11

-$MidNum $Format* $BackNumericEx;
+($MidNum | $MidNumLet) $BackNumericEx;
+
+# For dictionary-based break
+$dictionary $dictionary;

 
 ## -------------------------------------------------
 
 !!safe_forward;
 
 
 ## -------------------------------------------------
 
 !!safe_forward;
 

-# rule 3
-$Extend+;
-

 # rule 4
 # rule 4

-$Extend* $Format+ $ALetterEx;
-$Extend* $Format+ $NumericEx;
-$Extend* $Format+ $MidLetterEx;
-$Extend* $Format+ $MidNumEx;
-$Extend* $Format+ $KatakanaEx;
-$Extend* $Format+ $ExtendNumLetEx;
-
-$Extend+ $Format* $ALetterEx;
-$Extend+ $Format* $NumericEx;
-$Extend+ $Format* $MidLetterEx;
-$Extend+ $Format* $MidNumEx;
-$Extend+ $Format* $KatakanaEx;
-$Extend+ $Format* $ExtendNumLetEx;
+($Extend | $Format)+ .?;

 
 # rule 6
 
 # rule 6

-$MidLetterEx $Format* $ALetterEx;
+($MidLetterEx | $MidNumLetEx) $ALetterEx;

 
 # rule 11
 
 # rule 11

-$MidNumEx $Format* $NumericEx;
+($MidNumEx | $MidNumLetEx) $NumericEx;
+
+# For dictionary-based break
+$dictionary $dictionary;