-# Copyright (c) 2002-2006 International Business Machines Corporation and\r
-# others. All Rights Reserved.\r
-#\r
-# file: line.txt\r
-#\r
-# Line Breaking Rules\r
-# Implement default line breaking as defined by Unicode Standard Annex #14 version 5.0.0\r
-# http://www.unicode.org/reports/tr14/\r
-\r
-\r
-\r
-#\r
-# Character Classes defined by TR 14.\r
-#\r
-\r
-!!chain;\r
-!!LBCMNoChain;\r
-\r
-\r
-!!lookAheadHardBreak;\r
-#\r
-# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere\r
-# and only used for the line break rules.\r
-#\r
-# It is used in the implementation of the incredibly annoying rule LB 10\r
-# which says to treat any combining mark that is not attached to a base\r
-# character as if it were of class AL (alphabetic).\r
-#\r
-# The problem occurs in the reverse rules.\r
-#\r
-# Consider a sequence like, with correct breaks as shown\r
-# LF ID CM AL AL\r
-# ^ ^ ^\r
-# Then consider the sequence without the initial ID (ideographic)\r
-# LF CM AL AL\r
-# ^ ^\r
-# Our CM, which in the first example was attached to the ideograph,\r
-# is now unattached, becomes an alpha, and joins in with the other\r
-# alphas.\r
-#\r
-# When iterating forwards, these sequences do not present any problems\r
-# When iterating backwards, we need to look ahead when encountering\r
-# a CM to see whether it attaches to something further on or not.\r
-# (Look-ahead in a reverse rule is looking towards the start)\r
-#\r
-# If the CM is unattached, we need to force a break.\r
-#\r
-# !!lookAheadHardBreak forces the run time state machine to\r
-# stop immediately when a look ahead rule ( '/' operator) matches,\r
-# and set the match position to that of the look-ahead operator,\r
-# no matter what other rules may be in play at the time.\r
-#\r
-# See rule LB 19 for an example.\r
-#\r
-\r
-$AI = [:LineBreak = Ambiguous:];\r
-$AL = [:LineBreak = Alphabetic:];\r
-$BA = [:LineBreak = Break_After:];\r
-$BB = [:LineBreak = Break_Before:];\r
-$BK = [:LineBreak = Mandatory_Break:];\r
-$B2 = [:LineBreak = Break_Both:];\r
-$CB = [:LineBreak = Contingent_Break:];\r
-$CL = [:LineBreak = Close_Punctuation:];\r
-$CM = [:LineBreak = Combining_Mark:];\r
-$CR = [:LineBreak = Carriage_Return:];\r
-$EX = [:LineBreak = Exclamation:];\r
-$GL = [:LineBreak = Glue:];\r
-$HY = [:LineBreak = Hyphen:];\r
-$H2 = [:LineBreak = H2:];\r
-$H3 = [:LineBreak = H3:];\r
-$ID = [:LineBreak = Ideographic:];\r
-$IN = [:LineBreak = Inseperable:];\r
-$IS = [:LineBreak = Infix_Numeric:];\r
-$JL = [:LineBreak = JL:];\r
-$JV = [:LineBreak = JV:];\r
-$JT = [:LineBreak = JT:];\r
-$LF = [:LineBreak = Line_Feed:];\r
-$NL = [:LineBreak = Next_Line:];\r
-$NS = [:LineBreak = Nonstarter:];\r
-$NU = [:LineBreak = Numeric:];\r
-$OP = [:LineBreak = Open_Punctuation:];\r
-$PO = [:LineBreak = Postfix_Numeric:];\r
-$PR = [:LineBreak = Prefix_Numeric:];\r
-$QU = [:LineBreak = Quotation:];\r
-$SA = [:LineBreak = Complex_Context:];\r
-$SG = [:LineBreak = Surrogate:];\r
-$SP = [:LineBreak = Space:];\r
-$SY = [:LineBreak = Break_Symbols:];\r
-$WJ = [:LineBreak = Word_Joiner:];\r
-$XX = [:LineBreak = Unknown:];\r
-$ZW = [:LineBreak = ZWSpace:];\r
-\r
-# Dictionary character set, for triggering language-based break engines. Currently\r
-# limited to LineBreak=Complex_Context. Note that this set only works in Unicode\r
-# 5.0 or later as the definition of Complex_Context was corrected to include all\r
-# characters requiring dictionary break.\r
-\r
-$dictionary = [:LineBreak = Complex_Context:];\r
-\r
-#\r
-# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),\r
-# SA (South East Asian: Thai, Lao, Khmer)\r
-# SG (Unpaired Surrogates)\r
-# XX (Unknown, unassigned)\r
-# as $AL (Alphabetic)\r
-#\r
-$ALPlus = [$AL $AI $SA $SG $XX];\r
-\r
-#\r
-# Combining Marks. X $CM* behaves as if it were X. Rule LB6.\r
-#\r
-$ALcm = $ALPlus $CM*;\r
-$BAcm = $BA $CM*;\r
-$BBcm = $BB $CM*;\r
-$B2cm = $B2 $CM*;\r
-$CLcm = $CL $CM*;\r
-$EXcm = $EX $CM*;\r
-$GLcm = $GL $CM*;\r
-$HYcm = $HY $CM*;\r
-$H2cm = $H2 $CM*;\r
-$H3cm = $H3 $CM*;\r
-$IDcm = $ID $CM*;\r
-$INcm = $IN $CM*;\r
-$IScm = $IS $CM*;\r
-$JLcm = $JL $CM*;\r
-$JVcm = $JV $CM*;\r
-$JTcm = $JT $CM*;\r
-$NScm = $NS $CM*;\r
-$NUcm = $NU $CM*;\r
-$OPcm = $OP $CM*;\r
-$POcm = $PO $CM*;\r
-$PRcm = $PR $CM*;\r
-$QUcm = $QU $CM*;\r
-$SYcm = $SY $CM*;\r
-$WJcm = $WJ $CM*;\r
-\r
-## -------------------------------------------------\r
-\r
-!!forward;\r
-\r
-#\r
-# Each class of character can stand by itself as an unbroken token, with trailing combining stuff\r
-#\r
-$ALPlus $CM+;\r
-$BA $CM+;\r
-$BB $CM+;\r
-$B2 $CM+;\r
-$CL $CM+;\r
-$EX $CM+;\r
-$GL $CM+;\r
-$HY $CM+;\r
-$H2 $CM+;\r
-$H3 $CM+;\r
-$ID $CM+;\r
-$IN $CM+;\r
-$IS $CM+;\r
-$JL $CM+;\r
-$JV $CM+;\r
-$JT $CM+;\r
-$NS $CM+;\r
-$NU $CM+;\r
-$OP $CM+;\r
-$PO $CM+;\r
-$PR $CM+;\r
-$QU $CM+;\r
-$SY $CM+;\r
-$WJ $CM+;\r
-\r
-#\r
-# CAN_CM is the set of characters that may combine with CM combining chars.\r
-# Note that Linebreak UAX 14's concept of a combining char and the rules\r
-# for what they can combine with are _very_ different from the rest of Unicode.\r
-#\r
-# Note that $CM itself is left out of this set. If CM is needed as a base\r
-# it must be listed separately in the rule.\r
-#\r
-$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs\r
-$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs\r
-\r
-#\r
-# AL_FOLLOW set of chars that can unconditionally follow an AL\r
-# Needed in rules where stand-alone $CM s are treated as AL.\r
-# Chaining is disabled with CM because it causes other failures,\r
-# so for this one case we need to manually list out longer sequences.\r
-#\r
-$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];\r
-$AL_FOLLOW_CM = [$CL $EX $IS $SY $WJ $GL $QU $BA $HY $NS $IN $NU $ALPlus $OP];\r
-$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];\r
-\r
-\r
-#\r
-# Rule LB 4, 5 Mandatory (Hard) breaks.\r
-#\r
-$LB4Breaks = [$BK $CR $LF $NL];\r
-$LB4NonBreaks = [^$BK $CR $LF $NL];\r
-$CR $LF {100};\r
-\r
-#\r
-# LB 6 Do not break before hard line breaks.\r
-#\r
-$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.\r
-$CAN_CM $CM* $LB4Breaks {100};\r
-$CM+ $LB4Breaks {100};\r
-\r
-# LB 7 x SP\r
-# x ZW\r
-$LB4NonBreaks [$SP $ZW];\r
-$CAN_CM $CM* [$SP $ZW];\r
-$CM+ [$SP $ZW];\r
-\r
-#\r
-# LB 8 Break after zero width space\r
-#\r
-$LB8Breaks = [$LB4Breaks $ZW];\r
-$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];\r
-\r
-\r
-# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL \r
-# $CM not covered by the above needs to behave like $AL \r
-# See definition of $CAN_CM.\r
-\r
-$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.\r
-$CM+;\r
-\r
-#\r
-# LB 11 Do not break before or after WORD JOINER & related characters.\r
-#\r
-$CAN_CM $CM* $WJcm;\r
-$LB8NonBreaks $WJcm;\r
-$CM+ $WJcm;\r
-\r
-$WJcm [^$CAN_CM];\r
-$WJcm $CAN_CM $CM*;\r
-\r
-#\r
-# LB 12 Do not break before or after NBSP and related characters.\r
-#\r
-# (!SP) x GL\r
-[$LB8NonBreaks-$SP] $CM* $GLcm;\r
-$CM+ $GLcm;\r
-\r
-# GL x\r
-$GLcm ($LB8Breaks | $SP);\r
-$GLcm [$LB8NonBreaks-$SP] $CM*; # Don't let a combining mark go onto $CR, $BK, etc.\r
- # TODO: I don't think we need this rule.\r
- # All but $CM will chain off of preceding rule.\r
- # $GLcm will pick up the CM case by itself.\r
-\r
-\r
-\r
-\r
-#\r
-# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.\r
-#\r
-$LB8NonBreaks $CL;\r
-$CAN_CM $CM* $CL;\r
-$CM+ $CL; # by rule 10, stand-alone CM behaves as AL\r
-\r
-$LB8NonBreaks $EX;\r
-$CAN_CM $CM* $EX;\r
-$CM+ $EX; # by rule 10, stand-alone CM behaves as AL\r
-\r
-$LB8NonBreaks $IS;\r
-$CAN_CM $CM* $IS;\r
-$CM+ $IS; # by rule 10, stand-alone CM behaves as AL\r
-\r
-$LB8NonBreaks $SY;\r
-$CAN_CM $CM* $SY;\r
-$CM+ $SY; # by rule 10, stand-alone CM behaves as AL\r
-\r
-\r
-#\r
-# LB 14 Do not break after OP, even after spaced\r
-#\r
-$OPcm $SP* $CAN_CM $CM*;\r
-$OPcm $SP* $CANT_CM;\r
-\r
-$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL\r
-\r
-# LB 15\r
-$QUcm $SP* $OPcm;\r
-\r
-# LB 16\r
-$CLcm $SP* $NScm;\r
-\r
-# LB 17\r
-$B2cm $SP* $B2cm;\r
-\r
-#\r
-# LB 18 Break after spaces.\r
-#\r
-$LB18NonBreaks = [$LB8NonBreaks - [$SP]];\r
-$LB18Breaks = [$LB8Breaks $SP];\r
-\r
-\r
-# LB 19\r
-# x QU\r
-$LB18NonBreaks $CM* $QUcm;\r
-$CM+ $QUcm;\r
-\r
-# QU x\r
-$QUcm .?;\r
-$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.\r
- # TODO: I don't think this rule is needed.\r
-\r
-\r
-# LB 20\r
-# <break> $CB\r
-# $CB <break>\r
-\r
-$LB20NonBreaks = [$LB18NonBreaks - $CB];\r
-\r
-# LB 21 x (BA | HY | NS)\r
-# BB x\r
-#\r
-$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm); \r
-\r
-$BBcm [^$CB]; # $BB x\r
-$BBcm $LB20NonBreaks $CM*;\r
-\r
-# LB 22\r
-$ALcm $INcm;\r
-$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL\r
-$IDcm $INcm;\r
-$INcm $INcm;\r
-$NUcm $INcm;\r
-\r
-\r
-# $LB 23\r
-$IDcm $POcm;\r
-$ALcm $NUcm; # includes $LB19\r
-$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL\r
-$NUcm $ALcm;\r
-\r
-#\r
-# LB 24\r
-#\r
-$PRcm $IDcm;\r
-$PRcm $ALcm;\r
-$POcm $ALcm;\r
-\r
-#\r
-# LB 25 Numbers.\r
-#\r
-($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* $CLcm? ($PRcm | $POcm)?;\r
-\r
-# LB 26 Do not break a Korean syllable\r
-#\r
-$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);\r
-($JVcm | $H2cm) ($JVcm | $JTcm);\r
-($JTcm | $H3cm) $JTcm;\r
-\r
-# LB 27 Treat korean Syllable Block the same as ID (don't break it)\r
-($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;\r
-($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;\r
-$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);\r
-\r
-\r
-# LB 28 Do not break between alphabetics\r
-#\r
-$ALcm $ALcm;\r
-$CM+ $ALcm; # The $CM+ is from rule 10, and unattached CM is treated as AL\r
-\r
-# LB 29\r
-$IScm $ALcm;\r
-\r
-#\r
-# Rule 30 Do not break between letters, numbers or ordinary symbols\r
-# and opening or closing punctuation\r
-#\r
-($ALcm | $NUcm) $OPcm;\r
-$CM+ $OPcm;\r
-$CLcm ($ALcm | $NUcm);\r
-\r
-\r
-\r
-#\r
-# Reverse Rules.\r
-#\r
-## -------------------------------------------------\r
-\r
-!!reverse;\r
-\r
-$CM+ $ALPlus;\r
-$CM+ $BA;\r
-$CM+ $BB;\r
-$CM+ $B2;\r
-$CM+ $CL;\r
-$CM+ $EX;\r
-$CM+ $GL;\r
-$CM+ $HY;\r
-$CM+ $H2;\r
-$CM+ $H3;\r
-$CM+ $ID;\r
-$CM+ $IN;\r
-$CM+ $IS;\r
-$CM+ $JL;\r
-$CM+ $JV;\r
-$CM+ $JT;\r
-$CM+ $NS;\r
-$CM+ $NU;\r
-$CM+ $OP;\r
-$CM+ $PO;\r
-$CM+ $PR;\r
-$CM+ $QU;\r
-$CM+ $SY;\r
-$CM+ $WJ;\r
-$CM+;\r
-\r
-\r
-#\r
-# Sequences of the form (shown forwards)\r
-# [CANT_CM] <break> [CM] [whatever]\r
-# The CM needs to behave as an AL\r
-#\r
-$AL_FOLLOW $CM+ / (\r
- [$BK $CR $LF $NL $ZW {eof}] |\r
- $SP+ $CM+ $SP |\r
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.\r
- # LB14 says OP SP* x . \r
- # becomes OP SP* x AL\r
- # becomes OP SP* x CM+ AL_FOLLOW\r
- #\r
- # Further note: the $AL in [$AL {eof}] is only to work around\r
- # a rule compiler bug which complains about\r
- # empty sets otherwise.\r
- \r
-#\r
-# Sequences of the form (shown forwards)\r
-# [CANT_CM] <break> [CM] <break> [PR]\r
-# The CM needs to behave as an AL\r
-# This rule is concerned about getting the second of the two <breaks> in place.\r
-#\r
-\r
-[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];\r
-\r
-\r
-\r
-# LB 4, 5, 5\r
-\r
-$LB4Breaks [$LB4NonBreaks-$CM];\r
-$LB4Breaks $CM+ $CAN_CM;\r
-$LF $CR;\r
-\r
-\r
-# LB 7 x SP\r
-# x ZW\r
-[$SP $ZW] [$LB4NonBreaks-$CM];\r
-[$SP $ZW] $CM+ $CAN_CM;\r
-\r
-# LB 8 Break after zero width space\r
-\r
-\r
-# LB 9,10 Combining marks.\r
-# X $CM needs to behave like X, where X is not $SP or controls.\r
-# $CM not covered by the above needs to behave like $AL\r
-# Stick together any combining sequences that don't match other rules.\r
-$CM+ $CAN_CM;\r
-\r
-\r
-# LB 11\r
-$CM* $WJ $CM* $CAN_CM;\r
-$CM* $WJ [$LB8NonBreaks-$CM];\r
-\r
- $CANT_CM $CM* $WJ;\r
-$CM* $CAN_CM $CM* $WJ;\r
-\r
-# LB 12\r
-# x GL\r
-#\r
-$CM* $GL $CM* [$LB8NonBreaks-$CM-$SP];\r
-\r
-#\r
-# GL x\r
-#\r
-$CANT_CM $CM* $GL;\r
-$CM* $CAN_CM $CM* $GL;\r
-\r
-\r
-# LB 13\r
-$CL $CM+ $CAN_CM;\r
-$EX $CM+ $CAN_CM;\r
-$IS $CM+ $CAN_CM;\r
-$SY $CM+ $CAN_CM;\r
-\r
-$CL [$LB8NonBreaks-$CM];\r
-$EX [$LB8NonBreaks-$CM];\r
-$IS [$LB8NonBreaks-$CM];\r
-$SY [$LB8NonBreaks-$CM];\r
-\r
-# Rule 13 & 14 taken together for an edge case.\r
-# Match this, shown forward\r
-# OP SP+ ($CM+ behaving as $AL) (CL | EX | IS | IY)\r
-# This really wants to chain at the $CM+ (which is acting as an $AL)\r
-# except for $CM chaining being disabled.\r
-[$CL $EX $IS $SY] $CM+ $SP+ $CM* $OP; \r
-\r
-# LB 14 OP SP* x\r
-#\r
-$CM* $CAN_CM $SP* $CM* $OP;\r
- $CANT_CM $SP* $CM* $OP;\r
-$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP\r
- \r
- $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;\r
-$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;\r
-$SY $CM $SP+ $OP; # TODO: Experiment. Remove.\r
-\r
-\r
-\r
-# LB 15\r
-$CM* $OP $SP* $CM* $QU;\r
-\r
-# LB 16\r
-$CM* $NS $SP* $CM* $CL;\r
-\r
-# LB 17\r
-$CM* $B2 $SP* $CM* $B2;\r
-\r
-# LB 18 break after spaces\r
-# Nothing explicit needed here.\r
-\r
-\r
-#\r
-# LB 19\r
-#\r
-$CM* $QU $CM* $CAN_CM; # . x QU\r
-$CM* $QU $LB18NonBreaks;\r
-\r
-\r
-$CM* $CAN_CM $CM* $QU; # QU x .\r
- $CANT_CM $CM* $QU;\r
- \r
-#\r
-# LB 20 Break before and after CB.\r
-# nothing needed here.\r
-#\r
-\r
-# LB 21\r
-$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)\r
-\r
-$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .\r
-[^$CB] $CM* $BB; # \r
-\r
-\r
-\r
-# LB 22\r
-$CM* $IN $CM* $ALPlus;\r
-$CM* $IN $CM* $ID;\r
-$CM* $IN $CM* $IN;\r
-$CM* $IN $CM* $NU;\r
-\r
-# LB 23\r
-$CM* $PO $CM* $ID;\r
-$CM* $NU $CM* $ALPlus;\r
-$CM* $ALPlus $CM* $NU;\r
-\r
-# LB 24\r
-$CM* $ID $CM* $PR;\r
-$CM* $ALPlus $CM* $PR;\r
-$CM* $ALPlus $CM* $PO;\r
-\r
-\r
-# LB 25\r
-($CM* ($PR | $PO))? ($CM* $CL)? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;\r
-\r
-# LB 26\r
-$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;\r
-$CM* ($JT | $JV) $CM* ($H2 | $JV);\r
-$CM* $JT $CM* ($H3 | $JT);\r
-\r
-# LB 27\r
-$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);\r
-$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);\r
-$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;\r
-\r
-# LB 28\r
-$CM* $ALPlus $CM* $ALPlus;\r
-\r
-\r
-# LB 29\r
-$CM* $ALPlus $CM* $IS;\r
-\r
-# LB 30\r
-$CM* $OP $CM* ($NU | $ALPlus);\r
-$CM* ($NU | $ALPlus) $CM* $CL;\r
-\r
-\r
-## -------------------------------------------------\r
-\r
-!!safe_reverse;\r
-\r
-# LB 7\r
-$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];\r
-$CM+ $SP / .;\r
-\r
-# LB 9\r
-$SP+ $CM* $OP;\r
-\r
-# LB 10\r
-$SP+ $CM* $QU;\r
-\r
-# LB 11\r
-$SP+ $CM* $CL;\r
-$SP+ $CM* $B2;\r
-\r
-# LB 18\r
-($CM* ($IS | $SY))+ $CM* $NU;\r
-$CL $CM* ($NU | $IS | $SY);\r
-\r
-# For dictionary-based break\r
-$dictionary $dictionary;\r
-\r
-## -------------------------------------------------\r
-\r
-!!safe_forward;\r
-\r
-# Skip forward over all character classes that are involved in\r
-# rules containing patterns with possibly more than one char\r
-# of context.\r
-#\r
-# It might be slightly more efficient to have specific rules\r
-# instead of one generic one, but only if we could\r
-# turn off rule chaining. We don't want to move more\r
-# than necessary.\r
-#\r
-[$CM $OP $QU $CL $B2 $PR $HY $SP $dictionary]+ [^$CM $OP $QU $CL $B2 $PR $HY $dictionary];\r
-$dictionary $dictionary;\r
-\r
+# Copyright (c) 2002-2008 International Business Machines Corporation and
+# others. All Rights Reserved.
+#
+# file: line.txt
+#
+# Line Breaking Rules
+# Implement default line breaking as defined by
+# Unicode Standard Annex #14 Revision 21 for Unicode 5.1
+# http://www.unicode.org/reports/tr14/
+
+
+
+#
+# Character Classes defined by TR 14.
+#
+
+!!chain;
+!!LBCMNoChain;
+
+
+!!lookAheadHardBreak;
+#
+# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
+# and only used for the line break rules.
+#
+# It is used in the implementation of the incredibly annoying rule LB 10
+# which says to treat any combining mark that is not attached to a base
+# character as if it were of class AL (alphabetic).
+#
+# The problem occurs in the reverse rules.
+#
+# Consider a sequence like, with correct breaks as shown
+# LF ID CM AL AL
+# ^ ^ ^
+# Then consider the sequence without the initial ID (ideographic)
+# LF CM AL AL
+# ^ ^
+# Our CM, which in the first example was attached to the ideograph,
+# is now unattached, becomes an alpha, and joins in with the other
+# alphas.
+#
+# When iterating forwards, these sequences do not present any problems
+# When iterating backwards, we need to look ahead when encountering
+# a CM to see whether it attaches to something further on or not.
+# (Look-ahead in a reverse rule is looking towards the start)
+#
+# If the CM is unattached, we need to force a break.
+#
+# !!lookAheadHardBreak forces the run time state machine to
+# stop immediately when a look ahead rule ( '/' operator) matches,
+# and set the match position to that of the look-ahead operator,
+# no matter what other rules may be in play at the time.
+#
+# See rule LB 19 for an example.
+#
+
+$AI = [:LineBreak = Ambiguous:];
+$AL = [:LineBreak = Alphabetic:];
+$BA = [:LineBreak = Break_After:];
+$BB = [:LineBreak = Break_Before:];
+$BK = [:LineBreak = Mandatory_Break:];
+$B2 = [:LineBreak = Break_Both:];
+$CB = [:LineBreak = Contingent_Break:];
+$CL = [:LineBreak = Close_Punctuation:];
+$CM = [:LineBreak = Combining_Mark:];
+$CR = [:LineBreak = Carriage_Return:];
+$EX = [:LineBreak = Exclamation:];
+$GL = [:LineBreak = Glue:];
+$HY = [:LineBreak = Hyphen:];
+$H2 = [:LineBreak = H2:];
+$H3 = [:LineBreak = H3:];
+$ID = [:LineBreak = Ideographic:];
+$IN = [:LineBreak = Inseperable:];
+$IS = [:LineBreak = Infix_Numeric:];
+$JL = [:LineBreak = JL:];
+$JV = [:LineBreak = JV:];
+$JT = [:LineBreak = JT:];
+$LF = [:LineBreak = Line_Feed:];
+$NL = [:LineBreak = Next_Line:];
+$NS = [:LineBreak = Nonstarter:];
+$NU = [:LineBreak = Numeric:];
+$OP = [:LineBreak = Open_Punctuation:];
+$PO = [:LineBreak = Postfix_Numeric:];
+$PR = [:LineBreak = Prefix_Numeric:];
+$QU = [:LineBreak = Quotation:];
+$SA = [:LineBreak = Complex_Context:];
+$SG = [:LineBreak = Surrogate:];
+$SP = [:LineBreak = Space:];
+$SY = [:LineBreak = Break_Symbols:];
+$WJ = [:LineBreak = Word_Joiner:];
+$XX = [:LineBreak = Unknown:];
+$ZW = [:LineBreak = ZWSpace:];
+
+# Dictionary character set, for triggering language-based break engines. Currently
+# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
+# 5.0 or later as the definition of Complex_Context was corrected to include all
+# characters requiring dictionary break.
+
+$dictionary = [:LineBreak = Complex_Context:];
+
+#
+# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
+# SA (South East Asian: Thai, Lao, Khmer)
+# SG (Unpaired Surrogates)
+# XX (Unknown, unassigned)
+# as $AL (Alphabetic)
+#
+$ALPlus = [$AL $AI $SA $SG $XX];
+
+#
+# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
+#
+$ALcm = $ALPlus $CM*;
+$BAcm = $BA $CM*;
+$BBcm = $BB $CM*;
+$B2cm = $B2 $CM*;
+$CLcm = $CL $CM*;
+$EXcm = $EX $CM*;
+$GLcm = $GL $CM*;
+$HYcm = $HY $CM*;
+$H2cm = $H2 $CM*;
+$H3cm = $H3 $CM*;
+$IDcm = $ID $CM*;
+$INcm = $IN $CM*;
+$IScm = $IS $CM*;
+$JLcm = $JL $CM*;
+$JVcm = $JV $CM*;
+$JTcm = $JT $CM*;
+$NScm = $NS $CM*;
+$NUcm = $NU $CM*;
+$OPcm = $OP $CM*;
+$POcm = $PO $CM*;
+$PRcm = $PR $CM*;
+$QUcm = $QU $CM*;
+$SYcm = $SY $CM*;
+$WJcm = $WJ $CM*;
+
+## -------------------------------------------------
+
+!!forward;
+
+#
+# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
+#
+$ALPlus $CM+;
+$BA $CM+;
+$BB $CM+;
+$B2 $CM+;
+$CL $CM+;
+$EX $CM+;
+$GL $CM+;
+$HY $CM+;
+$H2 $CM+;
+$H3 $CM+;
+$ID $CM+;
+$IN $CM+;
+$IS $CM+;
+$JL $CM+;
+$JV $CM+;
+$JT $CM+;
+$NS $CM+;
+$NU $CM+;
+$OP $CM+;
+$PO $CM+;
+$PR $CM+;
+$QU $CM+;
+$SY $CM+;
+$WJ $CM+;
+
+#
+# CAN_CM is the set of characters that may combine with CM combining chars.
+# Note that Linebreak UAX 14's concept of a combining char and the rules
+# for what they can combine with are _very_ different from the rest of Unicode.
+#
+# Note that $CM itself is left out of this set. If CM is needed as a base
+# it must be listed separately in the rule.
+#
+$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
+$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
+
+#
+# AL_FOLLOW set of chars that can unconditionally follow an AL
+# Needed in rules where stand-alone $CM s are treated as AL.
+# Chaining is disabled with CM because it causes other failures,
+# so for this one case we need to manually list out longer sequences.
+#
+$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
+$AL_FOLLOW_CM = [$CL $EX $IS $SY $WJ $GL $QU $BA $HY $NS $IN $NU $ALPlus];
+$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
+
+
+#
+# Rule LB 4, 5 Mandatory (Hard) breaks.
+#
+$LB4Breaks = [$BK $CR $LF $NL];
+$LB4NonBreaks = [^$BK $CR $LF $NL];
+$CR $LF {100};
+
+#
+# LB 6 Do not break before hard line breaks.
+#
+$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
+$CAN_CM $CM* $LB4Breaks {100};
+$CM+ $LB4Breaks {100};
+
+# LB 7 x SP
+# x ZW
+$LB4NonBreaks [$SP $ZW];
+$CAN_CM $CM* [$SP $ZW];
+$CM+ [$SP $ZW];
+
+#
+# LB 8 Break after zero width space
+#
+$LB8Breaks = [$LB4Breaks $ZW];
+$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
+
+
+# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
+# $CM not covered by the above needs to behave like $AL
+# See definition of $CAN_CM.
+
+$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
+$CM+;
+
+#
+# LB 11 Do not break before or after WORD JOINER & related characters.
+#
+$CAN_CM $CM* $WJcm;
+$LB8NonBreaks $WJcm;
+$CM+ $WJcm;
+
+$WJcm $CANT_CM;
+$WJcm $CAN_CM $CM*;
+
+#
+# LB 12 Do not break after NBSP and related characters.
+# GL x
+#
+$GLcm $CAN_CM $CM*;
+$GLcm $CANT_CM;
+
+#
+# LB 12a Do not break before NBSP and related characters ...
+# [^SP BA HY] x GL
+#
+[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
+$CM+ GLcm;
+
+
+
+#
+# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
+#
+$LB8NonBreaks $CL;
+$CAN_CM $CM* $CL;
+$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
+
+$LB8NonBreaks $EX;
+$CAN_CM $CM* $EX;
+$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
+
+$LB8NonBreaks $IS;
+$CAN_CM $CM* $IS;
+$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
+
+$LB8NonBreaks $SY;
+$CAN_CM $CM* $SY;
+$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
+
+
+#
+# LB 14 Do not break after OP, even after spaces
+#
+$OPcm $SP* $CAN_CM $CM*;
+$OPcm $SP* $CANT_CM;
+
+$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
+
+# LB 15
+$QUcm $SP* $OPcm;
+
+# LB 16
+$CLcm $SP* $NScm;
+
+# LB 17
+$B2cm $SP* $B2cm;
+
+#
+# LB 18 Break after spaces.
+#
+$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
+$LB18Breaks = [$LB8Breaks $SP];
+
+
+# LB 19
+# x QU
+$LB18NonBreaks $CM* $QUcm;
+$CM+ $QUcm;
+
+# QU x
+$QUcm .?;
+$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
+ # TODO: I don't think this rule is needed.
+
+
+# LB 20
+# <break> $CB
+# $CB <break>
+
+$LB20NonBreaks = [$LB18NonBreaks - $CB];
+
+# LB 21 x (BA | HY | NS)
+# BB x
+#
+$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
+
+$BBcm [^$CB]; # $BB x
+$BBcm $LB20NonBreaks $CM*;
+
+# LB 22
+$ALcm $INcm;
+$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
+$IDcm $INcm;
+$INcm $INcm;
+$NUcm $INcm;
+
+
+# $LB 23
+$IDcm $POcm;
+$ALcm $NUcm; # includes $LB19
+$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
+$NUcm $ALcm;
+
+#
+# LB 24
+#
+$PRcm $IDcm;
+$PRcm $ALcm;
+$POcm $ALcm;
+
+#
+# LB 25 Numbers.
+#
+($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* $CLcm? ($PRcm | $POcm)?;
+
+# LB 26 Do not break a Korean syllable
+#
+$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
+($JVcm | $H2cm) ($JVcm | $JTcm);
+($JTcm | $H3cm) $JTcm;
+
+# LB 27 Treat korean Syllable Block the same as ID (don't break it)
+($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
+($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
+$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
+
+
+# LB 28 Do not break between alphabetics
+#
+$ALcm $ALcm;
+$CM+ $ALcm; # The $CM+ is from rule 10, and unattached CM is treated as AL
+
+# LB 29
+$IScm $ALcm;
+
+
+#
+# Reverse Rules.
+#
+## -------------------------------------------------
+
+!!reverse;
+
+$CM+ $ALPlus;
+$CM+ $BA;
+$CM+ $BB;
+$CM+ $B2;
+$CM+ $CL;
+$CM+ $EX;
+$CM+ $GL;
+$CM+ $HY;
+$CM+ $H2;
+$CM+ $H3;
+$CM+ $ID;
+$CM+ $IN;
+$CM+ $IS;
+$CM+ $JL;
+$CM+ $JV;
+$CM+ $JT;
+$CM+ $NS;
+$CM+ $NU;
+$CM+ $OP;
+$CM+ $PO;
+$CM+ $PR;
+$CM+ $QU;
+$CM+ $SY;
+$CM+ $WJ;
+$CM+;
+
+
+#
+# Sequences of the form (shown forwards)
+# [CANT_CM] <break> [CM] [whatever]
+# The CM needs to behave as an AL
+#
+$AL_FOLLOW $CM+ / (
+ [$BK $CR $LF $NL $ZW {eof}] |
+ $SP+ $CM+ $SP |
+ $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
+ # LB14 says OP SP* x .
+ # becomes OP SP* x AL
+ # becomes OP SP* x CM+ AL_FOLLOW
+ #
+ # Further note: the $AL in [$AL {eof}] is only to work around
+ # a rule compiler bug which complains about
+ # empty sets otherwise.
+
+#
+# Sequences of the form (shown forwards)
+# [CANT_CM] <break> [CM] <break> [PR]
+# The CM needs to behave as an AL
+# This rule is concerned about getting the second of the two <breaks> in place.
+#
+
+[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
+
+
+
+# LB 4, 5, 5
+
+$LB4Breaks [$LB4NonBreaks-$CM];
+$LB4Breaks $CM+ $CAN_CM;
+$LF $CR;
+
+
+# LB 7 x SP
+# x ZW
+[$SP $ZW] [$LB4NonBreaks-$CM];
+[$SP $ZW] $CM+ $CAN_CM;
+
+# LB 8 Break after zero width space
+
+
+# LB 9,10 Combining marks.
+# X $CM needs to behave like X, where X is not $SP or controls.
+# $CM not covered by the above needs to behave like $AL
+# Stick together any combining sequences that don't match other rules.
+$CM+ $CAN_CM;
+
+
+# LB 11
+$CM* $WJ $CM* $CAN_CM;
+$CM* $WJ [$LB8NonBreaks-$CM];
+
+ $CANT_CM $CM* $WJ;
+$CM* $CAN_CM $CM* $WJ;
+
+# LB 12a
+# [^SP BA HY] x GL
+#
+$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
+
+# LB 12
+# GL x
+#
+$CANT_CM $CM* $GL;
+$CM* $CAN_CM $CM* $GL;
+
+
+# LB 13
+$CL $CM+ $CAN_CM;
+$EX $CM+ $CAN_CM;
+$IS $CM+ $CAN_CM;
+$SY $CM+ $CAN_CM;
+
+$CL [$LB8NonBreaks-$CM];
+$EX [$LB8NonBreaks-$CM];
+$IS [$LB8NonBreaks-$CM];
+$SY [$LB8NonBreaks-$CM];
+
+# Rule 13 & 14 taken together for an edge case.
+# Match this, shown forward
+# OP SP+ ($CM+ behaving as $AL) (CL | EX | IS | IY)
+# This really wants to chain at the $CM+ (which is acting as an $AL)
+# except for $CM chaining being disabled.
+[$CL $EX $IS $SY] $CM+ $SP+ $CM* $OP;
+
+# LB 14 OP SP* x
+#
+$CM* $CAN_CM $SP* $CM* $OP;
+ $CANT_CM $SP* $CM* $OP;
+$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
+
+ $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
+$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
+$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
+
+
+
+# LB 15
+$CM* $OP $SP* $CM* $QU;
+
+# LB 16
+$CM* $NS $SP* $CM* $CL;
+
+# LB 17
+$CM* $B2 $SP* $CM* $B2;
+
+# LB 18 break after spaces
+# Nothing explicit needed here.
+
+
+#
+# LB 19
+#
+$CM* $QU $CM* $CAN_CM; # . x QU
+$CM* $QU $LB18NonBreaks;
+
+
+$CM* $CAN_CM $CM* $QU; # QU x .
+ $CANT_CM $CM* $QU;
+
+#
+# LB 20 Break before and after CB.
+# nothing needed here.
+#
+
+# LB 21
+$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
+
+$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
+[^$CB] $CM* $BB; #
+
+
+
+# LB 22
+$CM* $IN $CM* $ALPlus;
+$CM* $IN $CM* $ID;
+$CM* $IN $CM* $IN;
+$CM* $IN $CM* $NU;
+
+# LB 23
+$CM* $PO $CM* $ID;
+$CM* $NU $CM* $ALPlus;
+$CM* $ALPlus $CM* $NU;
+
+# LB 24
+$CM* $ID $CM* $PR;
+$CM* $ALPlus $CM* $PR;
+$CM* $ALPlus $CM* $PO;
+
+
+# LB 25
+($CM* ($PR | $PO))? ($CM* $CL)? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
+
+# LB 26
+$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
+$CM* ($JT | $JV) $CM* ($H2 | $JV);
+$CM* $JT $CM* ($H3 | $JT);
+
+# LB 27
+$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
+$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
+$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
+
+# LB 28
+$CM* $ALPlus $CM* $ALPlus;
+
+
+# LB 29
+$CM* $ALPlus $CM* $IS;
+
+
+## -------------------------------------------------
+
+!!safe_reverse;
+
+# LB 9
+$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
+$CM+ $SP / .;
+
+# LB 14
+$SP+ $CM* $OP;
+
+# LB 15
+$SP+ $CM* $QU;
+
+# LB 16
+$SP+ $CM* $CL;
+
+# LB 17
+$SP+ $CM* $B2;
+
+# LB 25
+($CM* ($IS | $SY))+ $CM* $NU;
+$CL $CM* ($NU | $IS | $SY);
+
+# For dictionary-based break
+$dictionary $dictionary;
+
+## -------------------------------------------------
+
+!!safe_forward;
+
+# Skip forward over all character classes that are involved in
+# rules containing patterns with possibly more than one char
+# of context.
+#
+# It might be slightly more efficient to have specific rules
+# instead of one generic one, but only if we could
+# turn off rule chaining. We don't want to move more
+# than necessary.
+#
+[$CM $OP $QU $CL $B2 $PR $HY $SP $dictionary]+ [^$CM $OP $QU $CL $B2 $PR $HY $dictionary];
+$dictionary $dictionary;
+
projects / apple / icu.git / blobdiff