X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/374ca955a76ecab1204ca8bfa63ff9238d998416..efa1e6592fb03ce23b15276b2b91d885a3ee7da5:/icuSources/data/brkitr/word.txt diff --git a/icuSources/data/brkitr/word.txt b/icuSources/data/brkitr/word.txt index 1fdaa3a0..48e67ee4 100644 --- a/icuSources/data/brkitr/word.txt +++ b/icuSources/data/brkitr/word.txt @@ -1,13 +1,15 @@ # -# Copyright (C) 2002-2004, International Business Machines Corporation +# Copyright (C) 2002-2013, International Business Machines Corporation # and others. All Rights Reserved. # # file: word.txt # # ICU Word Break Rules # See Unicode Standard Annex #29. -# These rules are based on Version 4.1 draft, dated 2004-11-11 +# These rules are based on UAX #29 Revision 20 for Unicode Version 6.2 # +# Note: Updates to word.txt will usually need to be merged into +# word_POSIX.txt also. ############################################################################## # @@ -17,220 +19,303 @@ !!chain; -$Katakana = [[:Script = KATAKANA:] - [:name = VERTICAL KANA REPEAT MARK:] - [:name = VERTICAL KANA REPEAT WITH VOICED SOUND MARK:] - [:name = VERTICAL KANA REPEAT MARK UPPER HALF:] - [:name = VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HALF:] - [:name = VERTICAL KANA REPEAT MARK LOWER HALF:] - [:name = KATAKANA-HIRAGANA VOICED SOUND MARK:] - [:name = KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK:] - [:name = KATAKANA-HIRAGANA DOUBLE HYPHEN:] - [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:] - [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:] - [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:] - [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]]; - - -$ALetter = [[:Alphabetic:] [:name= HEBREW PUNCTUATION GERESH:] - - [:Ideographic:] - - $Katakana - - [:Script = Hiragana:] - - [:Script = Lao:] - - [:Grapheme_Extend = TRUE:]]; - -$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:] - [:name = HEBREW PUNCTUATION GERSHAYIM:] - [:name = RIGHT SINGLE QUOTATION MARK:] - [:name = HYPHENATION POINT:] - [:name = COLON:]]; - - -$MidNum = [[:LineBreak = Infix_Numeric:] - [:name = COLON:]]; -$Numeric = [[:Nd:][:name = ARABIC DECIMAL SEPARATOR:][:name = ARABIC THOUSANDS SEPARATOR:]]; -$ExtendNumLet = [[:Connector_Punctuation:] - - [:name = KATAKANA MIDDLE DOT:] - - [:name = HALFWIDTH KATAKANA MIDDLE DOT:]]; - - # # Character Class Definitions. -# The names are those from TR29. # -$CR = \u000d; -$LF = \u000a; -$Extend = [[:Grapheme_Extend = TRUE:]]; -$Control = [[:Zl:] [:Zp:] [:Cc:] [:Cf:] - $Extend]; -$Format = [[:Cf:] - $Extend]; -$Hiragana = [:Hiragana:]; -$Ideographic = [:IDEOGRAPHIC:]; - -$ALetterEx = $ALetter $Extend*; -$NumericEx = $Numeric $Extend*; -$MidNumEx = $MidNum $Extend*; -$MidLetterEx = $MidLetter $Extend*; -$KatakanaEx = $Katakana $Extend*; -$ExtendNumLetEx = $ExtendNumLet $Extend*; +$CR = [\p{Word_Break = CR}]; +$LF = [\p{Word_Break = LF}]; +$Newline = [\p{Word_Break = Newline}]; +$Extend = [\p{Word_Break = Extend}]; +$Format = [\p{Word_Break = Format}]; +$Hiragana = [:Hiragana:]; +$Katakana = [\p{Word_Break = Katakana}]; +$Han = [:Han:]; +$ALetter = [\p{Word_Break = ALetter}]; +$MidNumLet = [\p{Word_Break = MidNumLet}]; +$MidLetter = [\p{Word_Break = MidLetter}]; +$MidNum = [\p{Word_Break = MidNum}]; +$Numeric = [\p{Word_Break = Numeric}]; +$ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; + +$RI_A = \U0001F1E6; # Trail ERTU +$RI_B = \U0001F1E7; # Trail EGR +$RI_C = \U0001F1E8; # Trail AHLNZ +$RI_D = \U0001F1E9; # Trail EK +$RI_E = \U0001F1EA; # Trail GS +$RI_F = \U0001F1EB; # Trail IR +$RI_G = \U0001F1EC; # Trail BR +$RI_H = \U0001F1ED; # Trail KU +$RI_I = \U0001F1EE; # Trail DLNT +$RI_J = \U0001F1EF; # Trail OP +$RI_K = \U0001F1F0; # Trail R +$RI_L = \U0001F1F1; # Trail B +$RI_M = \U0001F1F2; # Trail OXY +$RI_N = \U0001F1F3; # Trail LO +$RI_P = \U0001F1F5; # Trail LT +$RI_R = \U0001F1F7; # Trail OU +$RI_S = \U0001F1F8; # Trail AEGK +$RI_T = \U0001F1F9; # Trail HRW +$RI_U = \U0001F1FA; # Trail AS +$RI_V = \U0001F1FB; # Trail N + +$RI_A_End = [\U0001F1EA \U0001F1F7 \U0001F1F9 \U0001F1FA]; # ERTU +$RI_B_End = [\U0001F1EA \U0001F1EC \U0001F1F7]; # EGR +$RI_C_End = [\U0001F1E6 \U0001F1ED \U0001F1F1 \U0001F1F3 \U0001F1FF]; # AHLNZ +$RI_D_End = [\U0001F1EA \U0001F1F0]; # EK +$RI_E_End = [\U0001F1EC \U0001F1F8]; # GS +$RI_F_End = [\U0001F1EE \U0001F1F7]; # IR +$RI_G_End = [\U0001F1E7 \U0001F1F7]; # BR +$RI_H_End = [\U0001F1F0 \U0001F1FA]; # KU +$RI_I_End = [\U0001F1E9 \U0001F1F1 \U0001F1F3 \U0001F1F9]; # DLNT +$RI_J_End = [\U0001F1F4 \U0001F1F5]; # OP +$RI_K_End = \U0001F1F7; # R +$RI_L_End = \U0001F1E7; # B +$RI_M_End = [\U0001F1F4 \U0001F1FD \U0001F1FE]; # OXY +$RI_N_End = [\U0001F1F1 \U0001F1F4]; # LO +$RI_P_End = [\U0001F1F1 \U0001F1F9]; # LT +$RI_R_End = [\U0001F1F4 \U0001F1FA]; # OU +$RI_S_End = [\U0001F1E6 \U0001F1EA \U0001F1EC \U0001F1F0]; # AEGK +$RI_T_End = [\U0001F1ED \U0001F1F7 \U0001F1FC]; # HRW +$RI_U_End = [\U0001F1E6 \U0001F1F8]; # AS +$RI_V_End = \U0001F1F3; # N + + +# Dictionary character set, for triggering language-based break engines. Currently +# limited to LineBreak=Complex_Context. Note that this set only works in Unicode +# 5.0 or later as the definition of Complex_Context was corrected to include all +# characters requiring dictionary break. + +$Control = [\p{Grapheme_Cluster_Break = Control}]; +$HangulSyllable = [\uac00-\ud7a3]; +$ComplexContext = [:LineBreak = Complex_Context:]; +$KanaKanji = [$Han $Hiragana $Katakana]; +$dictionaryCJK = [$KanaKanji $HangulSyllable]; +$dictionary = [$ComplexContext $dictionaryCJK]; + +# leave CJK scripts out of ALetterPlus +$ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]]; + + +# +# Rules 4 Ignore Format and Extend characters, +# except when they appear at the beginning of a region of text. +# +# TODO: check if handling of katakana in dictionary makes rules incorrect/void +$KatakanaEx = $Katakana ($Extend | $Format)*; +$ALetterEx = $ALetterPlus ($Extend | $Format)*; +$MidNumLetEx = $MidNumLet ($Extend | $Format)*; +$MidLetterEx = $MidLetter ($Extend | $Format)*; +$MidNumEx = $MidNum ($Extend | $Format)*; +$NumericEx = $Numeric ($Extend | $Format)*; +$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; + +$Ideographic = [\p{Ideographic}]; +$HiraganaEx = $Hiragana ($Extend | $Format)*; +$IdeographicEx = $Ideographic ($Extend | $Format)*; ## ------------------------------------------------- !!forward; -# Rule 3 - don't break grapheme clusters. -# see character breaks - +# Rule 3 - CR x LF +# $CR $LF; -#[^$Control] $Extend*; -#$NumericEx $Extend* {100}; -#$ALetterEx $Extend* {200}; -[^$Control] $Extend+; + +# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning +# of a region of Text. The rule here comes into play when the start of text +# begins with a group of Format chars, or with a "word" consisting of a single +# char that is not in any of the listed word break categories followed by +# format char(s), or is not a CJK dictionary character. +[^$CR $LF $Newline]? ($Extend | $Format)+; + $NumericEx {100}; $ALetterEx {200}; -$KatakanaEx {300}; +$HangulSyllable {200}; +$KatakanaEx {400}; # note: these status values override those from rule 5 +$HiraganaEx {400}; # by virtue of being numerically larger. +$IdeographicEx {400}; # +# # rule 5 - -$ALetterEx $Format* $ALetterEx {200}; +# Do not break between most letters. +# +$ALetterEx $ALetterEx {200}; # rule 6 and 7 -$ALetterEx $Format* $MidLetterEx $Format* $ALetterEx {200}; +$ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200}; # rule 8 -$NumericEx $Format* $NumericEx {100}; +$NumericEx $NumericEx {100}; # rule 9 -$ALetterEx $Format* $NumericEx {200}; +$ALetterEx $NumericEx {200}; # rule 10 -$NumericEx $Format* $ALetterEx {200}; +$NumericEx $ALetterEx {200}; # rule 11 and 12 -$NumericEx $Format* $MidNumEx $Format* $NumericEx {100}; +$NumericEx ($MidNumEx | $MidNumLetEx) $NumericEx {100}; # rule 13 - -$KatakanaEx $Format* $KatakanaEx {300}; -$Hiragana $Extend* {300}; -$Ideographic $Extend* {400}; +# to be consistent with $KanaKanji $KanaKanhi, changed +# from 300 to 400. +# See also TestRuleStatus in intltest/rbbiapts.cpp +$KatakanaEx $KatakanaEx {400}; # rule 13a/b -$ALetterEx $Format* $ExtendNumLetEx {200}; # (13a) -$NumericEx $Format* $ExtendNumLetEx {100}; # (13a) -$KatakanaEx $Format* $ExtendNumLetEx {300}; # (13a) -$ExtendNumLetEx $Format* $ExtendNumLetEx{200}; # (13a) - -$ExtendNumLetEx $Format* $ALetterEx {200}; # (13b) -$ExtendNumLetEx $Format* $NumericEx {100}; # (13b) -$ExtendNumLetEx $Format* $KatakanaEx {300}; # (13b) - +$ALetterEx $ExtendNumLetEx {200}; # (13a) +$NumericEx $ExtendNumLetEx {100}; # (13a) +$KatakanaEx $ExtendNumLetEx {400}; # (13a) +$ExtendNumLetEx $ExtendNumLetEx {200}; # (13a) + +$ExtendNumLetEx $ALetterEx {200}; # (13b) +$ExtendNumLetEx $NumericEx {100}; # (13b) +$ExtendNumLetEx $KatakanaEx {400}; # (13b) + +# rule 13c + +$RI_A ($Extend|$Format)* $RI_A_End ($Extend|$Format)*; +$RI_B ($Extend|$Format)* $RI_B_End ($Extend|$Format)*; +$RI_C ($Extend|$Format)* $RI_C_End ($Extend|$Format)*; +$RI_D ($Extend|$Format)* $RI_D_End ($Extend|$Format)*; +$RI_E ($Extend|$Format)* $RI_E_End ($Extend|$Format)*; +$RI_F ($Extend|$Format)* $RI_F_End ($Extend|$Format)*; +$RI_G ($Extend|$Format)* $RI_G_End ($Extend|$Format)*; +$RI_H ($Extend|$Format)* $RI_H_End ($Extend|$Format)*; +$RI_I ($Extend|$Format)* $RI_I_End ($Extend|$Format)*; +$RI_J ($Extend|$Format)* $RI_J_End ($Extend|$Format)*; +$RI_K ($Extend|$Format)* $RI_K_End ($Extend|$Format)*; +$RI_L ($Extend|$Format)* $RI_L_End ($Extend|$Format)*; +$RI_M ($Extend|$Format)* $RI_M_End ($Extend|$Format)*; +$RI_N ($Extend|$Format)* $RI_N_End ($Extend|$Format)*; +$RI_P ($Extend|$Format)* $RI_P_End ($Extend|$Format)*; +$RI_R ($Extend|$Format)* $RI_R_End ($Extend|$Format)*; +$RI_S ($Extend|$Format)* $RI_S_End ($Extend|$Format)*; +$RI_T ($Extend|$Format)* $RI_T_End ($Extend|$Format)*; +$RI_U ($Extend|$Format)* $RI_U_End ($Extend|$Format)*; +$RI_V ($Extend|$Format)* $RI_V_End ($Extend|$Format)*; + +# special handling for CJK characters: chain for later dictionary segmentation +$HangulSyllable $HangulSyllable {200}; +$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found ## ------------------------------------------------- !!reverse; -$BackALetterEx = $Extend* $ALetter; -$BackNumericEx = $Extend* $Numeric; -$BackMidNumEx = $Extend* $MidNum; -$BackMidLetterEx = $Extend* $MidLetter; -$BackKatakanaEx = $Extend* $Katakana; -$BackExtendNumLetEx= $Extend* $ExtendNumLet; +$BackALetterEx = ($Format | $Extend)* $ALetterPlus; +$BackMidNumLetEx = ($Format | $Extend)* $MidNumLet; +$BackNumericEx = ($Format | $Extend)* $Numeric; +$BackMidNumEx = ($Format | $Extend)* $MidNum; +$BackMidLetterEx = ($Format | $Extend)* $MidLetter; +$BackKatakanaEx = ($Format | $Extend)* $Katakana; +$BackHiraganaEx = ($Format | $Extend)* $Hiragana; +$BackExtendNumLetEx = ($Format | $Extend)* $ExtendNumLet; +# rule 3 $LF $CR; -# see character breaks - -$Extend* [^$Control]; +# rule 4 +($Format | $Extend)* [^$CR $LF $Newline]?; # rule 5 -$BackALetterEx $Format* $BackALetterEx; +$BackALetterEx $BackALetterEx; # rule 6 and 7 -$BackALetterEx $Format* $BackMidLetterEx $Format* $BackALetterEx; +$BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx; # rule 8 -$BackNumericEx $Format* $BackNumericEx; +$BackNumericEx $BackNumericEx; # rule 9 -$BackNumericEx $Format* $BackALetterEx; +$BackNumericEx $BackALetterEx; # rule 10 -$BackALetterEx $Format* $BackNumericEx; +$BackALetterEx $BackNumericEx; # rule 11 and 12 -$BackNumericEx $Format* $BackMidNumEx $Format* $BackNumericEx; +$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx) $BackNumericEx; # rule 13 -$BackKatakanaEx $Format* $BackKatakanaEx; +$BackKatakanaEx $BackKatakanaEx; # rules 13 a/b # -($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx) $Format* $BackExtendNumLetEx; -$BackExtendNumLetEx $Format* ($BackALetterEx | $BackNumericEx | $BackKatakanaEx); +$BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx); +($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; + +# rule 13c + +($Format|$Extend)* $RI_A_End ($Format|$Extend)* $RI_A; +($Format|$Extend)* $RI_B_End ($Format|$Extend)* $RI_B; +($Format|$Extend)* $RI_C_End ($Format|$Extend)* $RI_C; +($Format|$Extend)* $RI_D_End ($Format|$Extend)* $RI_D; +($Format|$Extend)* $RI_E_End ($Format|$Extend)* $RI_E; +($Format|$Extend)* $RI_F_End ($Format|$Extend)* $RI_F; +($Format|$Extend)* $RI_G_End ($Format|$Extend)* $RI_G; +($Format|$Extend)* $RI_H_End ($Format|$Extend)* $RI_H; +($Format|$Extend)* $RI_I_End ($Format|$Extend)* $RI_I; +($Format|$Extend)* $RI_J_End ($Format|$Extend)* $RI_J; +($Format|$Extend)* $RI_K_End ($Format|$Extend)* $RI_K; +($Format|$Extend)* $RI_L_End ($Format|$Extend)* $RI_L; +($Format|$Extend)* $RI_M_End ($Format|$Extend)* $RI_M; +($Format|$Extend)* $RI_N_End ($Format|$Extend)* $RI_N; +($Format|$Extend)* $RI_P_End ($Format|$Extend)* $RI_P; +($Format|$Extend)* $RI_R_End ($Format|$Extend)* $RI_R; +($Format|$Extend)* $RI_S_End ($Format|$Extend)* $RI_S; +($Format|$Extend)* $RI_T_End ($Format|$Extend)* $RI_T; +($Format|$Extend)* $RI_U_End ($Format|$Extend)* $RI_U; +($Format|$Extend)* $RI_V_End ($Format|$Extend)* $RI_V; + +# special handling for CJK characters: chain for later dictionary segmentation +$HangulSyllable $HangulSyllable; +$KanaKanji $KanaKanji; #different rule status if both kanji and kana found ## ------------------------------------------------- !!safe_reverse; # rule 3 -$Extend+ [^$Extend]; -$Extend+; # comes into play when buffer _begins_ with an $Extend+. - -# rule 4 -$Format+ $BackALetterEx; -$Format+ $BackNumericEx; -$Format+ $BackMidLetterEx; -$Format+ $BackMidNumEx; -$Format+ $BackKatakanaEx; -$Format+ $BackExtendNumLetEx; - +($Extend | $Format)+ .?; # rule 6 -$MidLetter $Format* $BackALetterEx; +($MidLetter | $MidNumLet) $BackALetterEx; # rule 11 -$MidNum $Format* $BackNumericEx; +($MidNum | $MidNumLet) $BackNumericEx; + +# For dictionary-based break +$dictionary $dictionary; ## ------------------------------------------------- !!safe_forward; -# rule 3 -$Extend+; - # rule 4 -$Extend* $Format+ $ALetterEx; -$Extend* $Format+ $NumericEx; -$Extend* $Format+ $MidLetterEx; -$Extend* $Format+ $MidNumEx; -$Extend* $Format+ $KatakanaEx; -$Extend* $Format+ $ExtendNumLetEx; - -$Extend+ $Format* $ALetterEx; -$Extend+ $Format* $NumericEx; -$Extend+ $Format* $MidLetterEx; -$Extend+ $Format* $MidNumEx; -$Extend+ $Format* $KatakanaEx; -$Extend+ $Format* $ExtendNumLetEx; +($Extend | $Format)+ .?; # rule 6 -$MidLetterEx $Format* $ALetterEx; +($MidLetterEx | $MidNumLetEx) $ALetterEx; # rule 11 -$MidNumEx $Format* $NumericEx; +($MidNumEx | $MidNumLetEx) $NumericEx; + +# For dictionary-based break +$dictionary $dictionary;