X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/374ca955a76ecab1204ca8bfa63ff9238d998416..46f4442e9a5a4f3b98b7c1083586332f6a8a99a4:/icuSources/data/brkitr/word.txt?ds=sidebyside diff --git a/icuSources/data/brkitr/word.txt b/icuSources/data/brkitr/word.txt index 1fdaa3a0..db045c9a 100644 --- a/icuSources/data/brkitr/word.txt +++ b/icuSources/data/brkitr/word.txt @@ -1,13 +1,15 @@ # -# Copyright (C) 2002-2004, International Business Machines Corporation +# Copyright (C) 2002-2008, International Business Machines Corporation # and others. All Rights Reserved. # # file: word.txt # # ICU Word Break Rules # See Unicode Standard Annex #29. -# These rules are based on Version 4.1 draft, dated 2004-11-11 +# These rules are based on UAX-29 Revision 12 for Unicode 5.1 # +# Note: Updates to word.txt will usually need to be merged into +# word_POSIX.txt and word_ja.txt also. ############################################################################## # @@ -17,119 +19,113 @@ !!chain; -$Katakana = [[:Script = KATAKANA:] - [:name = VERTICAL KANA REPEAT MARK:] - [:name = VERTICAL KANA REPEAT WITH VOICED SOUND MARK:] - [:name = VERTICAL KANA REPEAT MARK UPPER HALF:] - [:name = VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HALF:] - [:name = VERTICAL KANA REPEAT MARK LOWER HALF:] - [:name = KATAKANA-HIRAGANA VOICED SOUND MARK:] - [:name = KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK:] - [:name = KATAKANA-HIRAGANA DOUBLE HYPHEN:] - [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:] - [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:] - [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:] - [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]]; - - -$ALetter = [[:Alphabetic:] [:name= HEBREW PUNCTUATION GERESH:] - - [:Ideographic:] - - $Katakana - - [:Script = Hiragana:] - - [:Script = Lao:] - - [:Grapheme_Extend = TRUE:]]; - -$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:] - [:name = HEBREW PUNCTUATION GERSHAYIM:] - [:name = RIGHT SINGLE QUOTATION MARK:] - [:name = HYPHENATION POINT:] - [:name = COLON:]]; - - -$MidNum = [[:LineBreak = Infix_Numeric:] - [:name = COLON:]]; -$Numeric = [[:Nd:][:name = ARABIC DECIMAL SEPARATOR:][:name = ARABIC THOUSANDS SEPARATOR:]]; -$ExtendNumLet = [[:Connector_Punctuation:] - - [:name = KATAKANA MIDDLE DOT:] - - [:name = HALFWIDTH KATAKANA MIDDLE DOT:]]; - - # # Character Class Definitions. -# The names are those from TR29. # -$CR = \u000d; -$LF = \u000a; -$Extend = [[:Grapheme_Extend = TRUE:]]; -$Control = [[:Zl:] [:Zp:] [:Cc:] [:Cf:] - $Extend]; -$Format = [[:Cf:] - $Extend]; -$Hiragana = [:Hiragana:]; -$Ideographic = [:IDEOGRAPHIC:]; - -$ALetterEx = $ALetter $Extend*; -$NumericEx = $Numeric $Extend*; -$MidNumEx = $MidNum $Extend*; -$MidLetterEx = $MidLetter $Extend*; -$KatakanaEx = $Katakana $Extend*; -$ExtendNumLetEx = $ExtendNumLet $Extend*; +$CR = [\p{Word_Break = CR}]; +$LF = [\p{Word_Break = LF}]; +$Newline = [\p{Word_Break = Newline}]; +$Extend = [\p{Word_Break = Extend}]; +$Format = [\p{Word_Break = Format}]; +$Katakana = [\p{Word_Break = Katakana}]; +$ALetter = [\p{Word_Break = ALetter}]; +$MidNumLet = [\p{Word_Break = MidNumLet}]; +$MidLetter = [\p{Word_Break = MidLetter}]; +$MidNum = [\p{Word_Break = MidNum}]; +$Numeric = [\p{Word_Break = Numeric}]; +$ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; + + +# Dictionary character set, for triggering language-based break engines. Currently +# limited to LineBreak=Complex_Context. Note that this set only works in Unicode +# 5.0 or later as the definition of Complex_Context was corrected to include all +# characters requiring dictionary break. + +$dictionary = [:LineBreak = Complex_Context:]; +$Control = [\p{Grapheme_Cluster_Break = Control}]; +$ALetterPlus = [$ALetter [$dictionary-$Extend-$Control]]; # Note: default ALetter does not + # include the dictionary characters. + +# +# Rules 4 Ignore Format and Extend characters, +# except when they appear at the beginning of a region of text. +# +$KatakanaEx = $Katakana ($Extend | $Format)*; +$ALetterEx = $ALetterPlus ($Extend | $Format)*; +$MidNumLetEx = $MidNumLet ($Extend | $Format)*; +$MidLetterEx = $MidLetter ($Extend | $Format)*; +$MidNumEx = $MidNum ($Extend | $Format)*; +$NumericEx = $Numeric ($Extend | $Format)*; +$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; + +$Hiragana = [\p{script=Hiragana}]; +$Ideographic = [\p{Ideographic}]; +$HiraganaEx = $Hiragana ($Extend | $Format)*; +$IdeographicEx = $Ideographic ($Extend | $Format)*; ## ------------------------------------------------- !!forward; -# Rule 3 - don't break grapheme clusters. -# see character breaks - +# Rule 3 - CR x LF +# $CR $LF; -#[^$Control] $Extend*; -#$NumericEx $Extend* {100}; -#$ALetterEx $Extend* {200}; -[^$Control] $Extend+; + +# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning +# of a region of Text. The rule here comes into play when the start of text +# begins with a group of Format chars, or with a "word" consisting of a single +# char that is not in any of the listed word break categories followed by +# format char(s). +[^$CR $LF $Newline]? ($Extend | $Format)+; + $NumericEx {100}; $ALetterEx {200}; -$KatakanaEx {300}; +$KatakanaEx {300}; # note: these status values override those from rule 5 +$HiraganaEx {300}; # by virtual of being numerically larger. +$IdeographicEx {400}; # +# # rule 5 - -$ALetterEx $Format* $ALetterEx {200}; +# Do not break between most letters. +# +$ALetterEx $ALetterEx {200}; # rule 6 and 7 -$ALetterEx $Format* $MidLetterEx $Format* $ALetterEx {200}; +$ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200}; # rule 8 -$NumericEx $Format* $NumericEx {100}; +$NumericEx $NumericEx {100}; # rule 9 -$ALetterEx $Format* $NumericEx {200}; +$ALetterEx $NumericEx {200}; # rule 10 -$NumericEx $Format* $ALetterEx {200}; +$NumericEx $ALetterEx {200}; # rule 11 and 12 -$NumericEx $Format* $MidNumEx $Format* $NumericEx {100}; +$NumericEx ($MidNumEx | $MidNumLetEx) $NumericEx {100}; # rule 13 -$KatakanaEx $Format* $KatakanaEx {300}; -$Hiragana $Extend* {300}; -$Ideographic $Extend* {400}; +$KatakanaEx $KatakanaEx {300}; # rule 13a/b -$ALetterEx $Format* $ExtendNumLetEx {200}; # (13a) -$NumericEx $Format* $ExtendNumLetEx {100}; # (13a) -$KatakanaEx $Format* $ExtendNumLetEx {300}; # (13a) -$ExtendNumLetEx $Format* $ExtendNumLetEx{200}; # (13a) +$ALetterEx $ExtendNumLetEx {200}; # (13a) +$NumericEx $ExtendNumLetEx {100}; # (13a) +$KatakanaEx $ExtendNumLetEx {300}; # (13a) +$ExtendNumLetEx $ExtendNumLetEx {200}; # (13a) -$ExtendNumLetEx $Format* $ALetterEx {200}; # (13b) -$ExtendNumLetEx $Format* $NumericEx {100}; # (13b) -$ExtendNumLetEx $Format* $KatakanaEx {300}; # (13b) +$ExtendNumLetEx $ALetterEx {200}; # (13b) +$ExtendNumLetEx $NumericEx {100}; # (13b) +$ExtendNumLetEx $KatakanaEx {300}; # (13b) @@ -137,100 +133,82 @@ $ExtendNumLetEx $Format* $KatakanaEx {300}; # (13b) !!reverse; -$BackALetterEx = $Extend* $ALetter; -$BackNumericEx = $Extend* $Numeric; -$BackMidNumEx = $Extend* $MidNum; -$BackMidLetterEx = $Extend* $MidLetter; -$BackKatakanaEx = $Extend* $Katakana; -$BackExtendNumLetEx= $Extend* $ExtendNumLet; +$BackALetterEx = ($Format | $Extend)* $ALetterPlus; +$BackMidNumLetEx = ($Format | $Extend)* $MidNumLet; +$BackNumericEx = ($Format | $Extend)* $Numeric; +$BackMidNumEx = ($Format | $Extend)* $MidNum; +$BackMidLetterEx = ($Format | $Extend)* $MidLetter; +$BackKatakanaEx = ($Format | $Extend)* $Katakana; +$BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet; +# rule 3 $LF $CR; -# see character breaks - -$Extend* [^$Control]; +# rule 4 +($Format | $Extend)* [^$CR $LF $Newline]?; # rule 5 -$BackALetterEx $Format* $BackALetterEx; +$BackALetterEx $BackALetterEx; # rule 6 and 7 -$BackALetterEx $Format* $BackMidLetterEx $Format* $BackALetterEx; +$BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx; # rule 8 -$BackNumericEx $Format* $BackNumericEx; +$BackNumericEx $BackNumericEx; # rule 9 -$BackNumericEx $Format* $BackALetterEx; +$BackNumericEx $BackALetterEx; # rule 10 -$BackALetterEx $Format* $BackNumericEx; +$BackALetterEx $BackNumericEx; # rule 11 and 12 -$BackNumericEx $Format* $BackMidNumEx $Format* $BackNumericEx; +$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx) $BackNumericEx; # rule 13 -$BackKatakanaEx $Format* $BackKatakanaEx; +$BackKatakanaEx $BackKatakanaEx; # rules 13 a/b # -($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx) $Format* $BackExtendNumLetEx; -$BackExtendNumLetEx $Format* ($BackALetterEx | $BackNumericEx | $BackKatakanaEx); +$BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx); +($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; ## ------------------------------------------------- !!safe_reverse; # rule 3 -$Extend+ [^$Extend]; -$Extend+; # comes into play when buffer _begins_ with an $Extend+. - -# rule 4 -$Format+ $BackALetterEx; -$Format+ $BackNumericEx; -$Format+ $BackMidLetterEx; -$Format+ $BackMidNumEx; -$Format+ $BackKatakanaEx; -$Format+ $BackExtendNumLetEx; - +($Extend | $Format)+ .?; # rule 6 -$MidLetter $Format* $BackALetterEx; +($MidLetter | $MidNumLet) $BackALetterEx; # rule 11 -$MidNum $Format* $BackNumericEx; +($MidNum | $MidNumLet) $BackNumericEx; + +# For dictionary-based break +$dictionary $dictionary; ## ------------------------------------------------- !!safe_forward; -# rule 3 -$Extend+; - # rule 4 -$Extend* $Format+ $ALetterEx; -$Extend* $Format+ $NumericEx; -$Extend* $Format+ $MidLetterEx; -$Extend* $Format+ $MidNumEx; -$Extend* $Format+ $KatakanaEx; -$Extend* $Format+ $ExtendNumLetEx; - -$Extend+ $Format* $ALetterEx; -$Extend+ $Format* $NumericEx; -$Extend+ $Format* $MidLetterEx; -$Extend+ $Format* $MidNumEx; -$Extend+ $Format* $KatakanaEx; -$Extend+ $Format* $ExtendNumLetEx; +($Extend | $Format)+ .?; # rule 6 -$MidLetterEx $Format* $ALetterEx; +($MidLetterEx | $MidNumLetEx) $ALetterEx; # rule 11 -$MidNumEx $Format* $NumericEx; +($MidNumEx | $MidNumLetEx) $NumericEx; + +# For dictionary-based break +$dictionary $dictionary;