[apple/icu.git] / icuSources / data / brkitr / rules / word.txt

#
# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 2002-2016, International Business Machines Corporation 
# and others. All Rights Reserved.
#
# file:  word.txt
#
# ICU Word Break Rules (modified from standard to remove colon from $MidLetter)
#      See Unicode Standard Annex #29.
#      These rules are based on UAX #29 Revision 29 for Unicode Version 9.0
#      with additions for Emoji Sequences from https://goo.gl/cluFCn
#      Plus additional characters introduces with Emoji 5, http://www.unicode.org/reports/tr51/proposed.html
#
# Note:  Updates to word.txt will usually need to be merged into
#        word_POSIX.txt also.

##############################################################################
#
#  Character class definitions from TR 29
#
##############################################################################

!!chain;


#
#  Character Class Definitions.
#

$CR                 = [\p{Word_Break = CR}];
$LF                 = [\p{Word_Break = LF}];
$Newline            = [\p{Word_Break = Newline} ];
$Extend             = [\p{Word_Break = Extend}];
$ZWJ                = [\p{Word_Break = ZWJ}];
$Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
$Format             = [\p{Word_Break = Format}];
$Katakana           = [\p{Word_Break = Katakana}];
$Hebrew_Letter      = [\p{Word_Break = Hebrew_Letter}];
$ALetter            = [\p{Word_Break = ALetter}];
$Single_Quote       = [\p{Word_Break = Single_Quote}];
$Double_Quote       = [\p{Word_Break = Double_Quote}];
$MidNumLet          = [\p{Word_Break = MidNumLet}];
$MidLetter          = [\p{Word_Break = MidLetter} - [\:]];
$MidNum             = [\p{Word_Break = MidNum}];
$Numeric            = [\p{Word_Break = Numeric}];
$ExtendNumLet       = [\p{Word_Break = ExtendNumLet}];
$E_Base             = [[\p{Word_Break = EB}] \U0001F46A-\U0001F46D\U0001F46F\U0001F91D\U0001F93C];
$E_Modifier         = [\p{Word_Break = EM}];

# Data for Extended Pictographic scraped from CLDR common/properties/ExtendedPictographic.txt, r13267
$Extended_Pict = [\U0001F774-\U0001F77F\U00002700-\U00002701\U00002703-\U00002704\U0000270E\U00002710-\U00002711\U00002765-\U00002767\U0001F030-\U0001F093\U0001F094-\U0001F09F\U0001F10D-\U0001F10F\U0001F12F\U0001F16C-\U0001F16F\U0001F1AD-\U0001F1E5\U0001F260-\U0001F265\U0001F203-\U0001F20F\U0001F23C-\U0001F23F\U0001F249-\U0001F24F\U0001F252-\U0001F25F\U0001F266-\U0001F2FF\U0001F7D5-\U0001F7FF\U0001F000-\U0001F003\U0001F005-\U0001F02B\U0001F02C-\U0001F02F\U0001F322-\U0001F323\U0001F394-\U0001F395\U0001F398\U0001F39C-\U0001F39D\U0001F3F1-\U0001F3F2\U0001F3F6\U0001F4FE\U0001F53E-\U0001F548\U0001F54F\U0001F568-\U0001F56E\U0001F571-\U0001F572\U0001F57B-\U0001F586\U0001F588-\U0001F589\U0001F58E-\U0001F58F\U0001F591-\U0001F594\U0001F597-\U0001F5A3\U0001F5A6-\U0001F5A7\U0001F5A9-\U0001F5B0\U0001F5B3-\U0001F5BB\U0001F5BD-\U0001F5C1\U0001F5C5-\U0001F5D0\U0001F5D4-\U0001F5DB\U0001F5DF-\U0001F5E0\U0001F5E2\U0001F5E4-\U0001F5E7\U0001F5E9-\U0001F5EE\U0001F5F0-\U0001F5F2\U0001F5F4-\U0001F5F9\U00002605\U00002607-\U0000260D\U0000260F-\U00002610\U00002612\U00002616-\U00002617\U00002619-\U0000261C\U0000261E-\U0000261F\U00002621\U00002624-\U00002625\U00002627-\U00002629\U0000262B-\U0000262D\U00002630-\U00002637\U0000263B-\U00002647\U00002654-\U0000265F\U00002661-\U00002662\U00002664\U00002667\U00002669-\U0000267A\U0000267C-\U0000267E\U00002680-\U00002691\U00002695\U00002698\U0000269A\U0000269D-\U0000269F\U000026A2-\U000026A9\U000026AC-\U000026AF\U000026B2-\U000026BC\U000026BF-\U000026C3\U000026C6-\U000026C7\U000026C9-\U000026CD\U000026D0\U000026D2\U000026D5-\U000026E8\U000026EB-\U000026EF\U000026F6\U000026FB-\U000026FC\U000026FE-\U000026FF\U00002388\U0001FA00-\U0001FFFD\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F0AF-\U0001F0B0\U0001F0C0\U0001F0D0\U0001F0F6-\U0001F0FF\U0001F80C-\U0001F80F\U0001F848-\U0001F84F\U0001F85A-\U0001F85F\U0001F888-\U0001F88F\U0001F8AE-\U0001F8FF\U0001F900-\U0001F90B\U0001F91F\U0001F928-\U0001F92F\U0001F931-\U0001F932\U0001F94C\U0001F95F-\U0001F96B\U0001F992-\U0001F997\U0001F9D0-\U0001F9E6\U0001F90C-\U0001F90F\U0001F93F\U0001F94D-\U0001F94F\U0001F96C-\U0001F97F\U0001F998-\U0001F9BF\U0001F9C1-\U0001F9CF\U0001F9E7-\U0001F9FF\U0001F6C6-\U0001F6CA\U0001F6D3-\U0001F6D4\U0001F6E6-\U0001F6E8\U0001F6EA\U0001F6F1-\U0001F6F2\U0001F6F7-\U0001F6F8\U0001F6D5-\U0001F6DF\U0001F6ED-\U0001F6EF\U0001F6F9-\U0001F6FF];
$EBG                = [\p{Word_Break = EBG}];
$EmojiNRK           = [[\p{Emoji}] - [\p{Word_Break = Regional_Indicator}\u002a\u00230-9©®™〰〽]];

$Han                = [:Han:];
$Hiragana           = [:Hiragana:];


#   Dictionary character set, for triggering language-based break engines. Currently
#   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
#   5.0 or later as the definition of Complex_Context was corrected to include all
#   characters requiring dictionary break.

$Control        = [\p{Grapheme_Cluster_Break = Control}]; 
$HangulSyllable = [\uac00-\ud7a3];
$ComplexContext = [:LineBreak = Complex_Context:];
$KanaKanji      = [$Han $Hiragana $Katakana];
$dictionaryCJK  = [$KanaKanji $HangulSyllable];
$dictionary     = [$ComplexContext $dictionaryCJK];

# leave CJK scripts out of ALetterPlus
$ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];


#
#  Rules 4    Ignore Format and Extend characters, 
#             except when they appear at the beginning of a region of text.
#
# TODO: check if handling of katakana in dictionary makes rules incorrect/void
$KatakanaEx           = $Katakana           ($Extend |  $Format | $ZWJ)*;
$Hebrew_LetterEx      = $Hebrew_Letter      ($Extend |  $Format | $ZWJ)*;
$ALetterEx            = $ALetterPlus        ($Extend |  $Format | $ZWJ)*;
$Single_QuoteEx       = $Single_Quote       ($Extend |  $Format | $ZWJ)*;
$Double_QuoteEx       = $Double_Quote       ($Extend |  $Format | $ZWJ)*;
$MidNumLetEx          = $MidNumLet          ($Extend |  $Format | $ZWJ)*;
$MidLetterEx          = $MidLetter          ($Extend |  $Format | $ZWJ)*;
$MidNumEx             = $MidNum             ($Extend |  $Format | $ZWJ)*;
$NumericEx            = $Numeric            ($Extend |  $Format | $ZWJ)*;
$ExtendNumLetEx       = $ExtendNumLet       ($Extend |  $Format | $ZWJ)*;
$Regional_IndicatorEx = $Regional_Indicator ($Extend |  $Format | $ZWJ)*;

$Ideographic    = [\p{Ideographic}];
$HiraganaEx     = $Hiragana     ($Extend |  $Format | $ZWJ)*;
$IdeographicEx  = $Ideographic  ($Extend |  $Format | $ZWJ)*;

## -------------------------------------------------

!!forward;


# Rule 3 - CR x LF
#
$CR $LF;

# Rule 3c   ZWJ x (Extended_Pict | EmojiNRK).  Precedes WB4, so no intervening Extend chars allowed.
#
$ZWJ ($Extended_Pict | $EmojiNRK);


# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
#          of a region of Text.   The rule here comes into play when the start of text
#          begins with a group of Format chars, or with a "word" consisting of a single
#          char that is not in any of the listed word break categories followed by
#          format char(s), or is not a CJK dictionary character.
[^$CR $LF $Newline]? ($Extend |  $Format | $ZWJ)+;

$NumericEx {100};
$ALetterEx {200};
$HangulSyllable {200};
$Hebrew_LetterEx{200};
$KatakanaEx {400};       # note:  these status values override those from rule 5
$HiraganaEx {400};       #        by virtue of being numerically larger.
$IdeographicEx {400};    #

$E_Base ($Extend | $Format | $ZWJ)*;
$E_Modifier ($Extend | $Format | $ZWJ)*;
$Extended_Pict ($Extend | $Format | $ZWJ)*;

#
# rule 5
#    Do not break between most letters.
#
($ALetterEx | $Hebrew_LetterEx)  ($ALetterEx | $Hebrew_LetterEx) {200};

# rule 6 and 7
($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};

# rule 7a
$Hebrew_LetterEx $Single_QuoteEx {200};

# rule 7b and 7c
$Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};

# rule 8

$NumericEx $NumericEx {100};

# rule 9

($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};

# rule 10

$NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};

# rule 11 and 12 

$NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};

# rule 13
# to be consistent with $KanaKanji $KanaKanhi, changed
# from 300 to 400.
# See also TestRuleStatus in intltest/rbbiapts.cpp
$KatakanaEx  $KatakanaEx {400};

# rule 13a/b

$ALetterEx       $ExtendNumLetEx {200};    #  (13a)
$Hebrew_LetterEx $ExtendNumLetEx {200};    #  (13a)
$NumericEx       $ExtendNumLetEx {100};    #  (13a)
$KatakanaEx      $ExtendNumLetEx {400};    #  (13a)
$ExtendNumLetEx  $ExtendNumLetEx {200};    #  (13a)

$ExtendNumLetEx  $ALetterEx      {200};    #  (13b)
$ExtendNumLetEx  $Hebrew_Letter  {200};    #  (13b)
$ExtendNumLetEx  $NumericEx      {100};    #  (13b)
$ExtendNumLetEx  $KatakanaEx     {400};    #  (13b)

# rule 14
#    Do not break within emoji modifier sequences

($E_Base | $EBG) ($Format | $Extend | $ZWJ)* $E_Modifier;

# rules 15 - 17
#    Pairs of Regional Indicators stay together.
#    With rule chaining disabled by ^, this rule will match exactly two of them.
#    No other rule begins with a Regional_Indicator, so chaining cannot extend the match.
#
^$Regional_IndicatorEx $Regional_IndicatorEx;

# special handling for CJK characters: chain for later dictionary segmentation
$HangulSyllable $HangulSyllable {200};
$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found 


## -------------------------------------------------

!!reverse;

$BackHebrew_LetterEx      = ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
$BackALetterEx            = ($Format | $Extend | $ZWJ)* $ALetterPlus;
$BackSingle_QuoteEx       = ($Format | $Extend | $ZWJ)* $Single_Quote;
$BackDouble_QuoteEx       = ($Format | $Extend | $ZWJ)* $Double_Quote;
$BackMidNumLetEx          = ($Format | $Extend | $ZWJ)* $MidNumLet;
$BackNumericEx            = ($Format | $Extend | $ZWJ)* $Numeric;
$BackMidNumEx             = ($Format | $Extend | $ZWJ)* $MidNum;
$BackMidLetterEx          = ($Format | $Extend | $ZWJ)* $MidLetter;
$BackKatakanaEx           = ($Format | $Extend | $ZWJ)* $Katakana;
$BackHiraganaEx           = ($Format | $Extend | $ZWJ)* $Hiragana;
$BackExtendNumLetEx       = ($Format | $Extend | $ZWJ)* $ExtendNumLet;
$BackRegional_IndicatorEx = ($Format | $Extend | $ZWJ)* $Regional_Indicator;

# rule 3
$LF $CR;

# Rule 3c   ZWJ x (Extended_Pict | EmojiNRK).  Precedes WB4, so no intervening Extend chars allowed.
#
($Extended_Pict | $EmojiNRK) $ZWJ;

# rule 4
($Format | $Extend | $ZWJ)*  [^$CR $LF $Newline]?;

# rule 5

($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);

# rule 6 and 7

($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);

# rule 7a
$BackSingle_QuoteEx $BackHebrew_LetterEx;

# Rule 7b and 7c
$BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;

# rule 8

$BackNumericEx $BackNumericEx;

# rule 9

$BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);

# rule 10

($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;

# rule 11 and 12

$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;

# rule 13

$BackKatakanaEx $BackKatakanaEx;

# rules 13 a/b
#
$BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; 

# special handling for CJK characters: chain for later dictionary segmentation
$HangulSyllable $HangulSyllable;
$KanaKanji $KanaKanji; #different rule status if both kanji and kana found

# rule 14

$E_Modifier ($Format | $Extend | $ZWJ)* ($E_Base | $EBG);

# rule 15 - 17
#    Pairs of Regional Indicators stay together.

^$BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)* 
        ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
^$BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)* 
        ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];

($Extended_Pict | $EmojiNRK) $ZWJ $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)* 
        ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
($Extended_Pict | $EmojiNRK) $ZWJ $BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)* 
        ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];


## -------------------------------------------------

!!safe_reverse;

# rule 3
($Extend | $Format | $ZWJ)+ .?;

# rule 6
($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);

# rule 7b
$Double_Quote $BackHebrew_LetterEx;


# rule 11
($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;

# rule 13c
$BackRegional_IndicatorEx*;

# For dictionary-based break
$dictionary $dictionary;

## -------------------------------------------------

!!safe_forward;

# rule 4
($Extend | $Format | $ZWJ)+ .?;

# rule 6
($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);

# rule 7b
$Double_QuoteEx $Hebrew_LetterEx;

# rule 11
($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;

# rule 13c
$Regional_IndicatorEx*;

# For dictionary-based break
$dictionary $dictionary;
Commit	Line	Data
f3c0d7a5 A	1	#
	2	# Copyright (C) 2016 and later: Unicode, Inc. and others.
	3	# License & terms of use: http://www.unicode.org/copyright.html
	4	# Copyright (C) 2002-2016, International Business Machines Corporation
374ca955	5	# and others. All Rights Reserved.
b75a7d8f	6	#
374ca955	7	# file: word.txt
b75a7d8f	8	#
2ca993e8	9	# ICU Word Break Rules (modified from standard to remove colon from $MidLetter)
b75a7d8f	10	# See Unicode Standard Annex #29.
f3c0d7a5 A	11	# These rules are based on UAX #29 Revision 29 for Unicode Version 9.0
	12	# with additions for Emoji Sequences from https://goo.gl/cluFCn
	13	# Plus additional characters introduces with Emoji 5, http://www.unicode.org/reports/tr51/proposed.html
b75a7d8f	14	#
73c04bcf	15	# Note: Updates to word.txt will usually need to be merged into
51004dcb	16	# word_POSIX.txt also.
b75a7d8f	17
374ca955	18	##############################################################################
b75a7d8f A	19	#
	20	# Character class definitions from TR 29
	21	#
374ca955 A	22	##############################################################################
	23
	24	!!chain;
	25
b75a7d8f A	26
	27	#
	28	# Character Class Definitions.
b75a7d8f	29	#
b75a7d8f	30
57a6839d A	31	$CR = [\p{Word_Break = CR}];
57a6839d A	32	$LF = [\p{Word_Break = LF}];
2ca993e8	33	$Newline = [\p{Word_Break = Newline} ];
f3c0d7a5 A	34	$Extend = [\p{Word_Break = Extend}];
f3c0d7a5 A	35	$ZWJ = [\p{Word_Break = ZWJ}];
2ca993e8	36	$Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
f3c0d7a5	37	$Format = [\p{Word_Break = Format}];
57a6839d A	38	$Katakana = [\p{Word_Break = Katakana}];
	39	$Hebrew_Letter = [\p{Word_Break = Hebrew_Letter}];
	40	$ALetter = [\p{Word_Break = ALetter}];
	41	$Single_Quote = [\p{Word_Break = Single_Quote}];
	42	$Double_Quote = [\p{Word_Break = Double_Quote}];
	43	$MidNumLet = [\p{Word_Break = MidNumLet}];
2ca993e8	44	$MidLetter = [\p{Word_Break = MidLetter} - [\:]];
57a6839d A	45	$MidNum = [\p{Word_Break = MidNum}];
	46	$Numeric = [\p{Word_Break = Numeric}];
	47	$ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
f3c0d7a5 A	48	$E_Base = [[\p{Word_Break = EB}] \U0001F46A-\U0001F46D\U0001F46F\U0001F91D\U0001F93C];
	49	$E_Modifier = [\p{Word_Break = EM}];
	50
	51	# Data for Extended Pictographic scraped from CLDR common/properties/ExtendedPictographic.txt, r13267
	52	$Extended_Pict = [\U0001F774-\U0001F77F\U00002700-\U00002701\U00002703-\U00002704\U0000270E\U00002710-\U00002711\U00002765-\U00002767\U0001F030-\U0001F093\U0001F094-\U0001F09F\U0001F10D-\U0001F10F\U0001F12F\U0001F16C-\U0001F16F\U0001F1AD-\U0001F1E5\U0001F260-\U0001F265\U0001F203-\U0001F20F\U0001F23C-\U0001F23F\U0001F249-\U0001F24F\U0001F252-\U0001F25F\U0001F266-\U0001F2FF\U0001F7D5-\U0001F7FF\U0001F000-\U0001F003\U0001F005-\U0001F02B\U0001F02C-\U0001F02F\U0001F322-\U0001F323\U0001F394-\U0001F395\U0001F398\U0001F39C-\U0001F39D\U0001F3F1-\U0001F3F2\U0001F3F6\U0001F4FE\U0001F53E-\U0001F548\U0001F54F\U0001F568-\U0001F56E\U0001F571-\U0001F572\U0001F57B-\U0001F586\U0001F588-\U0001F589\U0001F58E-\U0001F58F\U0001F591-\U0001F594\U0001F597-\U0001F5A3\U0001F5A6-\U0001F5A7\U0001F5A9-\U0001F5B0\U0001F5B3-\U0001F5BB\U0001F5BD-\U0001F5C1\U0001F5C5-\U0001F5D0\U0001F5D4-\U0001F5DB\U0001F5DF-\U0001F5E0\U0001F5E2\U0001F5E4-\U0001F5E7\U0001F5E9-\U0001F5EE\U0001F5F0-\U0001F5F2\U0001F5F4-\U0001F5F9\U00002605\U00002607-\U0000260D\U0000260F-\U00002610\U00002612\U00002616-\U00002617\U00002619-\U0000261C\U0000261E-\U0000261F\U00002621\U00002624-\U00002625\U00002627-\U00002629\U0000262B-\U0000262D\U00002630-\U00002637\U0000263B-\U00002647\U00002654-\U0000265F\U00002661-\U00002662\U00002664\U00002667\U00002669-\U0000267A\U0000267C-\U0000267E\U00002680-\U00002691\U00002695\U00002698\U0000269A\U0000269D-\U0000269F\U000026A2-\U000026A9\U000026AC-\U000026AF\U000026B2-\U000026BC\U000026BF-\U000026C3\U000026C6-\U000026C7\U000026C9-\U000026CD\U000026D0\U000026D2\U000026D5-\U000026E8\U000026EB-\U000026EF\U000026F6\U000026FB-\U000026FC\U000026FE-\U000026FF\U00002388\U0001FA00-\U0001FFFD\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F0AF-\U0001F0B0\U0001F0C0\U0001F0D0\U0001F0F6-\U0001F0FF\U0001F80C-\U0001F80F\U0001F848-\U0001F84F\U0001F85A-\U0001F85F\U0001F888-\U0001F88F\U0001F8AE-\U0001F8FF\U0001F900-\U0001F90B\U0001F91F\U0001F928-\U0001F92F\U0001F931-\U0001F932\U0001F94C\U0001F95F-\U0001F96B\U0001F992-\U0001F997\U0001F9D0-\U0001F9E6\U0001F90C-\U0001F90F\U0001F93F\U0001F94D-\U0001F94F\U0001F96C-\U0001F97F\U0001F998-\U0001F9BF\U0001F9C1-\U0001F9CF\U0001F9E7-\U0001F9FF\U0001F6C6-\U0001F6CA\U0001F6D3-\U0001F6D4\U0001F6E6-\U0001F6E8\U0001F6EA\U0001F6F1-\U0001F6F2\U0001F6F7-\U0001F6F8\U0001F6D5-\U0001F6DF\U0001F6ED-\U0001F6EF\U0001F6F9-\U0001F6FF];
	53	$EBG = [\p{Word_Break = EBG}];
	54	$EmojiNRK = [[\p{Emoji}] - [\p{Word_Break = Regional_Indicator}\u002a\u00230-9©®™〰〽]];
57a6839d A	55
	56	$Han = [:Han:];
	57	$Hiragana = [:Hiragana:];
73c04bcf A	58
73c04bcf A	59
73c04bcf A	60	# Dictionary character set, for triggering language-based break engines. Currently
	61	# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
	62	# 5.0 or later as the definition of Complex_Context was corrected to include all
	63	# characters requiring dictionary break.
	64
f3c0d7a5	65	$Control = [\p{Grapheme_Cluster_Break = Control}];
51004dcb A	66	$HangulSyllable = [\uac00-\ud7a3];
	67	$ComplexContext = [:LineBreak = Complex_Context:];
	68	$KanaKanji = [$Han $Hiragana $Katakana];
	69	$dictionaryCJK = [$KanaKanji $HangulSyllable];
	70	$dictionary = [$ComplexContext $dictionaryCJK];
	71
	72	# leave CJK scripts out of ALetterPlus
	73	$ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
	74
73c04bcf A	75
73c04bcf A	76	#
f3c0d7a5	77	# Rules 4 Ignore Format and Extend characters,
46f4442e	78	# except when they appear at the beginning of a region of text.
73c04bcf	79	#
51004dcb	80	# TODO: check if handling of katakana in dictionary makes rules incorrect/void
2ca993e8 A	81	$KatakanaEx = $Katakana ($Extend \| $Format \| $ZWJ)*;
	82	$Hebrew_LetterEx = $Hebrew_Letter ($Extend \| $Format \| $ZWJ)*;
	83	$ALetterEx = $ALetterPlus ($Extend \| $Format \| $ZWJ)*;
	84	$Single_QuoteEx = $Single_Quote ($Extend \| $Format \| $ZWJ)*;
	85	$Double_QuoteEx = $Double_Quote ($Extend \| $Format \| $ZWJ)*;
	86	$MidNumLetEx = $MidNumLet ($Extend \| $Format \| $ZWJ)*;
	87	$MidLetterEx = $MidLetter ($Extend \| $Format \| $ZWJ)*;
	88	$MidNumEx = $MidNum ($Extend \| $Format \| $ZWJ)*;
	89	$NumericEx = $Numeric ($Extend \| $Format \| $ZWJ)*;
	90	$ExtendNumLetEx = $ExtendNumLet ($Extend \| $Format \| $ZWJ)*;
	91	$Regional_IndicatorEx = $Regional_Indicator ($Extend \| $Format \| $ZWJ)*;
73c04bcf	92
46f4442e	93	$Ideographic = [\p{Ideographic}];
2ca993e8 A	94	$HiraganaEx = $Hiragana ($Extend \| $Format \| $ZWJ)*;
2ca993e8 A	95	$IdeographicEx = $Ideographic ($Extend \| $Format \| $ZWJ)*;
b75a7d8f	96
374ca955	97	## -------------------------------------------------
b75a7d8f	98
374ca955	99	!!forward;
b75a7d8f	100
b75a7d8f	101
73c04bcf	102	# Rule 3 - CR x LF
46f4442e A	103	#
46f4442e A	104	$CR $LF;
73c04bcf	105
f3c0d7a5	106	# Rule 3c ZWJ x (Extended_Pict \| EmojiNRK). Precedes WB4, so no intervening Extend chars allowed.
2ca993e8	107	#
f3c0d7a5	108	$ZWJ ($Extended_Pict \| $EmojiNRK);
2ca993e8 A	109
2ca993e8 A	110
73c04bcf A	111	# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
	112	# of a region of Text. The rule here comes into play when the start of text
	113	# begins with a group of Format chars, or with a "word" consisting of a single
	114	# char that is not in any of the listed word break categories followed by
51004dcb	115	# format char(s), or is not a CJK dictionary character.
2ca993e8	116	[^$CR $LF $Newline]? ($Extend \| $Format \| $ZWJ)+;
b75a7d8f	117
374ca955 A	118	$NumericEx {100};
374ca955 A	119	$ALetterEx {200};
51004dcb	120	$HangulSyllable {200};
57a6839d	121	$Hebrew_LetterEx{200};
51004dcb A	122	$KatakanaEx {400}; # note: these status values override those from rule 5
51004dcb A	123	$HiraganaEx {400}; # by virtue of being numerically larger.
46f4442e	124	$IdeographicEx {400}; #
b75a7d8f	125
2ca993e8 A	126	$E_Base ($Extend \| $Format \| $ZWJ)*;
2ca993e8 A	127	$E_Modifier ($Extend \| $Format \| $ZWJ)*;
f3c0d7a5	128	$Extended_Pict ($Extend \| $Format \| $ZWJ)*;
2ca993e8	129
46f4442e	130	#
374ca955	131	# rule 5
46f4442e A	132	# Do not break between most letters.
46f4442e A	133	#
57a6839d	134	($ALetterEx \| $Hebrew_LetterEx) ($ALetterEx \| $Hebrew_LetterEx) {200};
b75a7d8f	135
374ca955	136	# rule 6 and 7
57a6839d A	137	($ALetterEx \| $Hebrew_LetterEx) ($MidLetterEx \| $MidNumLetEx \| $Single_QuoteEx) ($ALetterEx \| $Hebrew_LetterEx) {200};
	138
	139	# rule 7a
	140	$Hebrew_LetterEx $Single_QuoteEx {200};
	141
	142	# rule 7b and 7c
	143	$Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
b75a7d8f	144
374ca955	145	# rule 8
b75a7d8f	146
73c04bcf	147	$NumericEx $NumericEx {100};
b75a7d8f	148
374ca955	149	# rule 9
b75a7d8f	150
57a6839d	151	($ALetterEx \| $Hebrew_LetterEx) $NumericEx {200};
b75a7d8f	152
374ca955	153	# rule 10
b75a7d8f	154
57a6839d	155	$NumericEx ($ALetterEx \| $Hebrew_LetterEx) {200};
b75a7d8f	156
f3c0d7a5	157	# rule 11 and 12
374ca955	158
57a6839d	159	$NumericEx ($MidNumEx \| $MidNumLetEx \| $Single_QuoteEx) $NumericEx {100};
374ca955 A	160
374ca955 A	161	# rule 13
51004dcb A	162	# to be consistent with $KanaKanji $KanaKanhi, changed
	163	# from 300 to 400.
	164	# See also TestRuleStatus in intltest/rbbiapts.cpp
	165	$KatakanaEx $KatakanaEx {400};
374ca955 A	166
	167	# rule 13a/b
	168
57a6839d A	169	$ALetterEx $ExtendNumLetEx {200}; # (13a)
	170	$Hebrew_LetterEx $ExtendNumLetEx {200}; # (13a)
	171	$NumericEx $ExtendNumLetEx {100}; # (13a)
	172	$KatakanaEx $ExtendNumLetEx {400}; # (13a)
	173	$ExtendNumLetEx $ExtendNumLetEx {200}; # (13a)
374ca955	174
57a6839d A	175	$ExtendNumLetEx $ALetterEx {200}; # (13b)
	176	$ExtendNumLetEx $Hebrew_Letter {200}; # (13b)
	177	$ExtendNumLetEx $NumericEx {100}; # (13b)
	178	$ExtendNumLetEx $KatakanaEx {400}; # (13b)
51004dcb	179
f3c0d7a5 A	180	# rule 14
	181	# Do not break within emoji modifier sequences
	182
	183	($E_Base \| $EBG) ($Format \| $Extend \| $ZWJ)* $E_Modifier;
	184
	185	# rules 15 - 17
2ca993e8 A	186	# Pairs of Regional Indicators stay together.
	187	# With rule chaining disabled by ^, this rule will match exactly two of them.
	188	# No other rule begins with a Regional_Indicator, so chaining cannot extend the match.
	189	#
	190	^$Regional_IndicatorEx $Regional_IndicatorEx;
51004dcb A	191
	192	# special handling for CJK characters: chain for later dictionary segmentation
	193	$HangulSyllable $HangulSyllable {200};
f3c0d7a5	194	$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
374ca955 A	195
	196
	197	## -------------------------------------------------
b75a7d8f	198
374ca955 A	199	!!reverse;
374ca955 A	200
2ca993e8 A	201	$BackHebrew_LetterEx = ($Format \| $Extend \| $ZWJ)* $Hebrew_Letter;
	202	$BackALetterEx = ($Format \| $Extend \| $ZWJ)* $ALetterPlus;
	203	$BackSingle_QuoteEx = ($Format \| $Extend \| $ZWJ)* $Single_Quote;
	204	$BackDouble_QuoteEx = ($Format \| $Extend \| $ZWJ)* $Double_Quote;
	205	$BackMidNumLetEx = ($Format \| $Extend \| $ZWJ)* $MidNumLet;
	206	$BackNumericEx = ($Format \| $Extend \| $ZWJ)* $Numeric;
	207	$BackMidNumEx = ($Format \| $Extend \| $ZWJ)* $MidNum;
	208	$BackMidLetterEx = ($Format \| $Extend \| $ZWJ)* $MidLetter;
	209	$BackKatakanaEx = ($Format \| $Extend \| $ZWJ)* $Katakana;
	210	$BackHiraganaEx = ($Format \| $Extend \| $ZWJ)* $Hiragana;
	211	$BackExtendNumLetEx = ($Format \| $Extend \| $ZWJ)* $ExtendNumLet;
	212	$BackRegional_IndicatorEx = ($Format \| $Extend \| $ZWJ)* $Regional_Indicator;
374ca955	213
73c04bcf	214	# rule 3
46f4442e	215	$LF $CR;
374ca955	216
f3c0d7a5	217	# Rule 3c ZWJ x (Extended_Pict \| EmojiNRK). Precedes WB4, so no intervening Extend chars allowed.
2ca993e8	218	#
f3c0d7a5	219	($Extended_Pict \| $EmojiNRK) $ZWJ;
2ca993e8	220
73c04bcf	221	# rule 4
2ca993e8	222	($Format \| $Extend \| $ZWJ)* [^$CR $LF $Newline]?;
374ca955 A	223
	224	# rule 5
	225
57a6839d	226	($BackALetterEx \| $BackHebrew_LetterEx) ($BackALetterEx \| $BackHebrew_LetterEx);
374ca955 A	227
	228	# rule 6 and 7
	229
57a6839d A	230	($BackALetterEx \| $BackHebrew_LetterEx) ($BackMidLetterEx \| $BackMidNumLetEx \| $BackSingle_QuoteEx) ($BackALetterEx \| $BackHebrew_LetterEx);
	231
	232	# rule 7a
	233	$BackSingle_QuoteEx $BackHebrew_LetterEx;
374ca955	234
57a6839d A	235	# Rule 7b and 7c
57a6839d A	236	$BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
374ca955 A	237
	238	# rule 8
	239
73c04bcf	240	$BackNumericEx $BackNumericEx;
374ca955 A	241
	242	# rule 9
	243
57a6839d	244	$BackNumericEx ($BackALetterEx \| $BackHebrew_LetterEx);
374ca955 A	245
	246	# rule 10
	247
57a6839d	248	($BackALetterEx \| $BackHebrew_LetterEx) $BackNumericEx;
374ca955 A	249
	250	# rule 11 and 12
	251
57a6839d	252	$BackNumericEx ($BackMidNumEx \| $BackMidNumLetEx \| $BackSingle_QuoteEx) $BackNumericEx;
374ca955 A	253
	254	# rule 13
	255
73c04bcf	256	$BackKatakanaEx $BackKatakanaEx;
374ca955 A	257
374ca955 A	258	# rules 13 a/b
b75a7d8f	259	#
57a6839d	260	$BackExtendNumLetEx ($BackALetterEx \| $BackHebrew_LetterEx \| $BackNumericEx \| $BackKatakanaEx \| $BackExtendNumLetEx);
f3c0d7a5	261	($BackALetterEx \| $BackHebrew_LetterEx \| $BackNumericEx \| $BackKatakanaEx) $BackExtendNumLetEx;
374ca955	262
f3c0d7a5 A	263	# special handling for CJK characters: chain for later dictionary segmentation
	264	$HangulSyllable $HangulSyllable;
	265	$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
	266
	267	# rule 14
	268
	269	$E_Modifier ($Format \| $Extend \| $ZWJ)* ($E_Base \| $EBG);
51004dcb	270
f3c0d7a5 A	271	# rule 15 - 17
	272	# Pairs of Regional Indicators stay together.
	273
	274	^$BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
2ca993e8	275	($Format \| $Extend \| $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
f3c0d7a5	276	^$BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
2ca993e8	277	($Format \| $Extend \| $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
08b89b0a	278
f3c0d7a5	279	($Extended_Pict \| $EmojiNRK) $ZWJ $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
2ca993e8	280	($Format \| $Extend \| $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
f3c0d7a5	281	($Extended_Pict \| $EmojiNRK) $ZWJ $BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
2ca993e8	282	($Format \| $Extend \| $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
51004dcb	283
2ca993e8 A	284
2ca993e8 A	285
374ca955 A	286	## -------------------------------------------------
	287
	288	!!safe_reverse;
	289
	290	# rule 3
2ca993e8	291	($Extend \| $Format \| $ZWJ)+ .?;
374ca955 A	292
374ca955 A	293	# rule 6
57a6839d A	294	($MidLetter \| $MidNumLet \| $Single_Quote) ($BackALetterEx \| $BackHebrew_LetterEx);
	295
	296	# rule 7b
	297	$Double_Quote $BackHebrew_LetterEx;
	298
374ca955 A	299
374ca955 A	300	# rule 11
57a6839d	301	($MidNum \| $MidNumLet \| $Single_Quote) $BackNumericEx;
73c04bcf	302
2ca993e8 A	303	# rule 13c
	304	$BackRegional_IndicatorEx*;
	305
73c04bcf A	306	# For dictionary-based break
73c04bcf A	307	$dictionary $dictionary;
374ca955 A	308
	309	## -------------------------------------------------
	310
	311	!!safe_forward;
	312
374ca955	313	# rule 4
2ca993e8	314	($Extend \| $Format \| $ZWJ)+ .?;
374ca955 A	315
374ca955 A	316	# rule 6
57a6839d A	317	($MidLetterEx \| $MidNumLetEx \| $Single_QuoteEx) ($ALetterEx \| $Hebrew_LetterEx);
	318
	319	# rule 7b
	320	$Double_QuoteEx $Hebrew_LetterEx;
b75a7d8f	321
374ca955	322	# rule 11
57a6839d	323	($MidNumEx \| $MidNumLetEx \| $Single_QuoteEx) $NumericEx;
73c04bcf	324
2ca993e8 A	325	# rule 13c
	326	$Regional_IndicatorEx*;
	327
73c04bcf A	328	# For dictionary-based break
73c04bcf A	329	$dictionary $dictionary;