#
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 2002-2016, International Business Machines Corporation
# and others. All Rights Reserved.
#
#
# ICU Word Break Rules, POSIX locale.
# See Unicode Standard Annex #29.
-# These rules are based on UAX #29 Revision 27 for Unicode Version 8.0
-# with additions from L2/16-011R3 for Emoji sequences.
+# These rules are based on UAX #29 Revision 34 for Unicode Version 12.0
#
# Note: Updates to word.txt will usually need to be merged into
# word_POSIX.txt also.
##############################################################################
!!chain;
+!!quoted_literals_only;
#
# Character Class Definitions.
#
-# Apple $EmojiForMods becomes $E_Base here
-$E_Base = [\u261D\u26F9\u270A-\u270D\U0001F385\U0001F3C2-\U0001F3C4\U0001F3C7\U0001F3CA-\U0001F3CC\U0001F442-\U0001F443\U0001F446-\U0001F450\U0001F466-\U0001F478\U0001F47C\U0001F481-\U0001F483\U0001F485-\U0001F487\U0001F4AA\U0001F574-\U0001F575\U0001F57A\U0001F590\U0001F595-\U0001F596\U0001F645-\U0001F647\U0001F64B-\U0001F64F\U0001F6A3\U0001F6B4-\U0001F6B6\U0001F6C0\U0001F6CC\U0001F918-\U0001F91E\U0001F926\U0001F930\U0001F933-\U0001F939\U0001F93C-\U0001F93E];
-# Apple $EmojiMods becomes $E_Modifier here, same chars
-$E_Modifier = [\U0001F3FB-\U0001F3FF];
-$ZWJ = [\u200D];
-# Apple $EmojiForSeqs becomes $GAZ here (only emoji that follow a ZWJ)
-$GAZ = [\u2640\u2642\u2764\u2695-\u2696\u2708\U0001F308\U0001F33E\U0001F373\U0001F393\U0001F3A4\U0001F3A8\U0001F3EB\U0001F3ED\U0001F466-\U0001F469\U0001F48B\U0001F4BB-\U0001F4BC\U0001F527\U0001F52C\U0001F5E8\U0001F680\U0001F692];
+$Han = [:Han:];
$CR = [\p{Word_Break = CR}];
$LF = [\p{Word_Break = LF}];
-$Newline = [\p{Word_Break = Newline} ];
-$Extend = [[\p{Word_Break = Extend}][:Block=Tags:]];
+$Newline = [\p{Word_Break = Newline}];
+$Extend = [\p{Word_Break = Extend}-$Han];
+$ZWJ = [\p{Word_Break = ZWJ}];
$Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
-$Format = [[\p{Word_Break = Format}] - [:Block=Tags:]];
+$Format = [\p{Word_Break = Format}];
$Katakana = [\p{Word_Break = Katakana}];
$Hebrew_Letter = [\p{Word_Break = Hebrew_Letter}];
$ALetter = [\p{Word_Break = ALetter}];
$MidNum = [\p{Word_Break = MidNum} [.]];
$Numeric = [\p{Word_Break = Numeric}];
$ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
+$WSegSpace = [\p{Word_Break = WSegSpace}];
+$Extended_Pict = [\p{Extended_Pictographic}];
-$Han = [:Han:];
$Hiragana = [:Hiragana:];
+$Ideographic = [\p{Ideographic}];
# Dictionary character set, for triggering language-based break engines. Currently
$dictionaryCJK = [$KanaKanji $HangulSyllable];
$dictionary = [$ComplexContext $dictionaryCJK];
+# TODO: check if handling of katakana in dictionary makes rules incorrect/void
+
# leave CJK scripts out of ALetterPlus
$ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
-#
-# Rules 4 Ignore Format and Extend characters,
-# except when they appear at the beginning of a region of text.
-#
-# TODO: check if handling of katakana in dictionary makes rules incorrect/void
-$KatakanaEx = $Katakana ($Extend | $Format | $ZWJ)*;
-$Hebrew_LetterEx = $Hebrew_Letter ($Extend | $Format | $ZWJ)*;
-$ALetterEx = $ALetterPlus ($Extend | $Format | $ZWJ)*;
-$Single_QuoteEx = $Single_Quote ($Extend | $Format | $ZWJ)*;
-$Double_QuoteEx = $Double_Quote ($Extend | $Format | $ZWJ)*;
-$MidNumLetEx = $MidNumLet ($Extend | $Format | $ZWJ)*;
-$MidLetterEx = $MidLetter ($Extend | $Format | $ZWJ)*;
-$MidNumEx = $MidNum ($Extend | $Format | $ZWJ)*;
-$NumericEx = $Numeric ($Extend | $Format | $ZWJ)*;
-$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format | $ZWJ)*;
-$Regional_IndicatorEx = $Regional_Indicator ($Extend | $Format | $ZWJ)*;
-
-$Ideographic = [\p{Ideographic}];
-$HiraganaEx = $Hiragana ($Extend | $Format | $ZWJ)*;
-$IdeographicEx = $Ideographic ($Extend | $Format | $ZWJ)*;
-
## -------------------------------------------------
-!!forward;
-
-
# Rule 3 - CR x LF
#
$CR $LF;
-# Rule 3c ZWJ x GAZ. Preceeds WB4, so no intervening Extend chars allowed.
+# Rule 3c Do not break within emoji zwj sequences.
+# ZWJ × \p{Extended_Pictographic}. Precedes WB4, so no intervening Extend chars allowed.
#
-$ZWJ $GAZ;
+$ZWJ $Extended_Pict;
+# Rule 3d - Keep horizontal whitespace together.
+#
+$WSegSpace $WSegSpace;
# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
-# of a region of Text. The rule here comes into play when the start of text
-# begins with a group of Format chars, or with a "word" consisting of a single
-# char that is not in any of the listed word break categories followed by
-# format char(s), or is not a CJK dictionary character.
-[^$CR $LF $Newline]? ($Extend | $Format | $ZWJ)+;
-
-$NumericEx {100};
-$ALetterEx {200};
-$HangulSyllable {200};
-$Hebrew_LetterEx{200};
-$KatakanaEx {400}; # note: these status values override those from rule 5
-$HiraganaEx {400}; # by virtue of being numerically larger.
-$IdeographicEx {400}; #
+# of a region of Text.
+
+$ExFm = [$Extend $Format $ZWJ];
+
+^$ExFm+; # This rule fires only when there are format or extend characters at the
+ # start of text, or immediately following another boundary. It groups them, in
+ # the event there are more than one.
+
+[^$CR $LF $Newline $ExFm] $ExFm*; # This rule rule attaches trailing format/extends to words,
+ # with no special rule status value.
-$E_Base ($Extend | $Format | $ZWJ)*;
-$E_Modifier ($Extend | $Format | $ZWJ)*;
-$GAZ ($Extend | $Format | $ZWJ)*;
+$Numeric $ExFm* {100}; # This group of rules also attach trailing format/extends, but
+$ALetterPlus $ExFm* {200}; # with rule status set based on the word's final base character.
+$HangulSyllable {200};
+$Hebrew_Letter $ExFm* {200};
+$Katakana $ExFm* {400}; # note: these status values override those from rule 5
+$Hiragana $ExFm* {400}; # by virtue of being numerically larger.
+$Ideographic $ExFm* {400}; #
#
# rule 5
# Do not break between most letters.
#
-($ALetterEx | $Hebrew_LetterEx) ($ALetterEx | $Hebrew_LetterEx) {200};
+($ALetterPlus | $Hebrew_Letter) $ExFm* ($ALetterPlus | $Hebrew_Letter);
# rule 6 and 7
-($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};
+($ALetterPlus | $Hebrew_Letter) $ExFm* ($MidLetter | $MidNumLet | $Single_Quote) $ExFm* ($ALetterPlus | $Hebrew_Letter) {200};
# rule 7a
-$Hebrew_LetterEx $Single_QuoteEx {200};
+$Hebrew_Letter $ExFm* $Single_Quote {200};
# rule 7b and 7c
-$Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
+$Hebrew_Letter $ExFm* $Double_Quote $ExFm* $Hebrew_Letter;
# rule 8
-$NumericEx $NumericEx {100};
+$Numeric $ExFm* $Numeric;
# rule 9
-($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};
+($ALetterPlus | $Hebrew_Letter) $ExFm* $Numeric;
# rule 10
-$NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
+$Numeric $ExFm* ($ALetterPlus | $Hebrew_Letter);
# rule 11 and 12
-$NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
+$Numeric $ExFm* ($MidNum | $MidNumLet | $Single_Quote) $ExFm* $Numeric;
# rule 13
# to be consistent with $KanaKanji $KanaKanhi, changed
# from 300 to 400.
# See also TestRuleStatus in intltest/rbbiapts.cpp
-$KatakanaEx $KatakanaEx {400};
+$Katakana $ExFm* $Katakana {400};
# rule 13a/b
-$ALetterEx $ExtendNumLetEx {200}; # (13a)
-$Hebrew_LetterEx $ExtendNumLetEx {200}; # (13a)
-$NumericEx $ExtendNumLetEx {100}; # (13a)
-$KatakanaEx $ExtendNumLetEx {400}; # (13a)
-$ExtendNumLetEx $ExtendNumLetEx {200}; # (13a)
+$ALetterPlus $ExFm* $ExtendNumLet {200}; # (13a)
+$Hebrew_Letter $ExFm* $ExtendNumLet {200}; # (13a)
+$Numeric $ExFm* $ExtendNumLet {100}; # (13a)
+$Katakana $ExFm* $ExtendNumLet {400}; # (13a)
+$ExtendNumLet $ExFm* $ExtendNumLet {200}; # (13a)
-$ExtendNumLetEx $ALetterEx {200}; # (13b)
-$ExtendNumLetEx $Hebrew_Letter {200}; # (13b)
-$ExtendNumLetEx $NumericEx {100}; # (13b)
-$ExtendNumLetEx $KatakanaEx {400}; # (13b)
+$ExtendNumLet $ExFm* $ALetterPlus {200}; # (13b)
+$ExtendNumLet $ExFm* $Hebrew_Letter {200}; # (13b)
+$ExtendNumLet $ExFm* $Numeric {100}; # (13b)
+$ExtendNumLet $ExFm* $Katakana {400}; # (13b)
-# rule 13c
+# rules 15 - 17
# Pairs of Regional Indicators stay together.
-# With rule chaining disabled by ^, this rule will match exactly two of them.
+# With incoming rule chaining disabled by ^, this rule will match exactly two of them.
# No other rule begins with a Regional_Indicator, so chaining cannot extend the match.
#
-^$Regional_IndicatorEx $Regional_IndicatorEx;
+^$Regional_Indicator $ExFm* $Regional_Indicator;
# special handling for CJK characters: chain for later dictionary segmentation
$HangulSyllable $HangulSyllable {200};
$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
-# rule 13d
-# E_Base x E_Modifier
-#
-($E_Base | $GAZ) ($Format | $Extend | $ZWJ)* $E_Modifier;
-
-
-## -------------------------------------------------
-
-!!reverse;
-
-$BackHebrew_LetterEx = ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
-$BackALetterEx = ($Format | $Extend | $ZWJ)* $ALetterPlus;
-$BackSingle_QuoteEx = ($Format | $Extend | $ZWJ)* $Single_Quote;
-$BackDouble_QuoteEx = ($Format | $Extend | $ZWJ)* $Double_Quote;
-$BackMidNumLetEx = ($Format | $Extend | $ZWJ)* $MidNumLet;
-$BackNumericEx = ($Format | $Extend | $ZWJ)* $Numeric;
-$BackMidNumEx = ($Format | $Extend | $ZWJ)* $MidNum;
-$BackMidLetterEx = ($Format | $Extend | $ZWJ)* $MidLetter;
-$BackKatakanaEx = ($Format | $Extend | $ZWJ)* $Katakana;
-$BackHiraganaEx = ($Format | $Extend | $ZWJ)* $Hiragana;
-$BackExtendNumLetEx = ($Format | $Extend | $ZWJ)* $ExtendNumLet;
-$BackRegional_IndicatorEx = ($Format | $Extend | $ZWJ)* $Regional_Indicator;
-
-# rule 3
-$LF $CR;
-
-# Rule 3c ZWJ x GAZ. Preceeds WB4, so no intervening Extend chars allowed.
-#
-$GAZ $ZWJ;
-
-# rule 4
-($Format | $Extend | $ZWJ)* [^$CR $LF $Newline]?;
-
-# rule 5
-
-($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 6 and 7
-
-($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 7a
-$BackSingle_QuoteEx $BackHebrew_LetterEx;
-
-# Rule 7b and 7c
-$BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
-
-# rule 8
-
-$BackNumericEx $BackNumericEx;
-
-# rule 9
-
-$BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 10
-
-($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
-
-# rule 11 and 12
-
-$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
-
-# rule 13
-
-$BackKatakanaEx $BackKatakanaEx;
-
-# rules 13 a/b
-#
-$BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
-($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
-
-# rule 13c
-
-^$BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-^$BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-
-$GAZ $ZWJ $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-$GAZ $ZWJ $BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-
-# special handling for CJK characters: chain for later dictionary segmentation
-$HangulSyllable $HangulSyllable;
-$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
-
-# rule 13d
-
-$E_Modifier ($Format | $Extend | $ZWJ)* ($E_Base | $GAZ);
-
-
-
-## -------------------------------------------------
-
-!!safe_reverse;
-
-# rule 3
-($Extend | $Format | $ZWJ)+ .?;
-
-# rule 6
-($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 7b
-$Double_Quote $BackHebrew_LetterEx;
-
-
-# rule 11
-($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
-
-# rule 13c
-$BackRegional_IndicatorEx*;
-
-# For dictionary-based break
-$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# rule 4
-($Extend | $Format | $ZWJ)+ .?;
-
-# rule 6
-($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
-
-# rule 7b
-$Double_QuoteEx $Hebrew_LetterEx;
-
-# rule 11
-($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
-
-# rule 13c
-$Regional_IndicatorEx*;
-
-# For dictionary-based break
-$dictionary $dictionary;
+# Rule 999
+# Match a single code point if no other rule applies.
+.;
projects / apple / icu.git / blobdiff