X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/2ca993e82fb37b597a3c73ecd1586a139a6579c5..340931cb2e044a2141d11567dd0f782524e32994:/icuSources/data/brkitr/rules/line_loose.txt diff --git a/icuSources/data/brkitr/rules/line_loose.txt b/icuSources/data/brkitr/rules/line_loose.txt index d33ad061..2794554e 100644 --- a/icuSources/data/brkitr/rules/line_loose.txt +++ b/icuSources/data/brkitr/rules/line_loose.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2016 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html +# # Copyright (c) 2002-2016 International Business Machines Corporation and # others. All Rights Reserved. # @@ -5,20 +8,14 @@ # # Line Breaking Rules # Implement default line breaking as defined by -# Unicode Standard Annex #14 Revision 35 for Unicode 8.0 -# http://www.unicode.org/reports/tr14/ -# -# Includes the Emoji breaking proposals from Unicode L2/16-011R3. -# http://www.unicode.org/L2/L2016/16011r3-break-prop-emoji.pdf -# -# tailored as noted in 2nd paragraph below. +# Unicode Standard Annex #14 Revision 44 for Unicode 13.0 +# http://www.unicode.org/reports/tr14/, with the following modification: # -# TODO: Rule LB 8 remains as it was in Unicode 5.2 -# This is only because of a limitation of ICU break engine implementation, -# not because the older behavior is desirable. +# Boundaries between hyphens and following letters are suppressed when +# there is a boundary preceding the hyphen. See rule 20.9 # # This tailors the line break behavior to correspond to CSS -# line-break=loose (BCP47 -u-lb-loose) as defined for languages other than +# line-break=loose (BCP47 -u-lb-loose) as defined for languages other than # Chinese & Japanese. # It sets characters of class CJ to behave like ID. # In addition, it allows breaks: @@ -30,67 +27,31 @@ # !!chain; +!!quoted_literals_only; -!!lookAheadHardBreak; -# -# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere -# and only used for the line break rules. -# -# It is used in the implementation of rule LB 10 -# which says to treat any combining mark that is not attached to a base -# character as if it were of class AL (alphabetic). -# -# The problem occurs in the reverse rules. -# -# Consider a sequence like, with correct breaks as shown -# LF ID CM AL AL -# ^ ^ ^ -# Then consider the sequence without the initial ID (ideographic) -# LF CM AL AL -# ^ ^ -# Our CM, which in the first example was attached to the ideograph, -# is now unattached, becomes an alpha, and joins in with the other -# alphas. -# -# When iterating forwards, these sequences do not present any problems -# When iterating backwards, we need to look ahead when encountering -# a CM to see whether it attaches to something further on or not. -# (Look-ahead in a reverse rule is looking towards the start) -# -# If the CM is unattached, we need to force a break. -# -# !!lookAheadHardBreak forces the run time state machine to -# stop immediately when a look ahead rule ( '/' operator) matches, -# and set the match position to that of the look-ahead operator, -# no matter what other rules may be in play at the time. -# -# See rule LB 19 for an example. -# - -# Temporary definitions of Emoji Base and Emoji Modifiers, until properties are available. - -$EB = [\u261D\u26F9\u270A-\u270D\U0001F385\U0001F3C3-\U0001F3C4\U0001F3CA-\U0001F3CB\U0001F442-\U0001F443\U0001F446-\U0001F450\U0001F466-\U0001F469\U0001F46E\U0001F470-\U0001F478\U0001F47C\U0001F481-\U0001F483\U0001F485-\U0001F487\U0001F4AA\U0001F575\U0001F590\U0001F595-\U0001F596\U0001F645-\U0001F647\U0001F64B-\U0001F64F\U0001F6A3\U0001F6B4-\U0001F6B6\U0001F6C0\U0001F918]; -$EM = [\U0001F3FB-\U0001F3FF]; - -$AI = [[:LineBreak = Ambiguous:] - [\u2640\u2642]]; -$AL = [[:LineBreak = Alphabetic:] - [$EM\u2764]]; +$AI = [:LineBreak = Ambiguous:]; +$AL = [:LineBreak = Alphabetic:]; $BA = [:LineBreak = Break_After:]; +$HH = [\u2010]; # \u2010 is HYPHEN, default line break is BA. $BB = [:LineBreak = Break_Before:]; $BK = [:LineBreak = Mandatory_Break:]; $B2 = [:LineBreak = Break_Both:]; $CB = [:LineBreak = Contingent_Break:]; $CJ = [:LineBreak = Conditional_Japanese_Starter:]; $CL = [:LineBreak = Close_Punctuation:]; -$CM = [[:LineBreak = Combining_Mark:] \u200d]; +# $CM = [:LineBreak = Combining_Mark:]; $CP = [:LineBreak = Close_Parenthesis:]; $CR = [:LineBreak = Carriage_Return:]; +$EB = [:LineBreak = EB:]; +$EM = [:LineBreak = EM:]; $EX = [:LineBreak = Exclamation:]; $GL = [:LineBreak = Glue:]; $HL = [:LineBreak = Hebrew_Letter:]; $HY = [:LineBreak = Hyphen:]; $H2 = [:LineBreak = H2:]; $H3 = [:LineBreak = H3:]; -$ID = [[:LineBreak = Ideographic:]$CJ[\u2640\u2642\u2764] - $EB]; +# CSS Loose tailoring: CJ resolves to ID +$ID = [[:LineBreak = Ideographic:] $CJ]; $IN = [:LineBreak = Inseperable:]; $IS = [:LineBreak = Infix_Numeric:]; $JL = [:LineBreak = JL:]; @@ -113,92 +74,38 @@ $SY = [:LineBreak = Break_Symbols:]; $WJ = [:LineBreak = Word_Joiner:]; $XX = [:LineBreak = Unknown:]; $ZW = [:LineBreak = ZWSpace:]; -$ZWJ = [\u200d]; +$ZWJ = [:LineBreak = ZWJ:]; + +# OP30 and CP30 are variants of OP and CP that appear in-line in rule LB30 from UAX 14, +# without a formal name. Because ICU rules require multiple uses of the expressions, +# give them a single definition with a name + +$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]]; +$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]]; + +# By LB9, a ZWJ also behaves as a CM. Including it in the definition of CM avoids having to explicitly +# list it in the numerous rules that use CM. +# By LB1, SA characters with general categor of Mn or Mc also resolve to CM. + +$CM = [[:LineBreak = Combining_Mark:] $ZWJ [$SA & [[:Mn:][:Mc:]]]]; +$CMX = [[$CM] - [$ZWJ]]; # Dictionary character set, for triggering language-based break engines. Currently -# limited to LineBreak=Complex_Context. Note that this set only works in Unicode -# 5.0 or later as the definition of Complex_Context was corrected to include all -# characters requiring dictionary break. +# limited to LineBreak=Complex_Context (SA). -$dictionary = [:LineBreak = Complex_Context:]; +$dictionary = [$SA]; # # Rule LB1. By default, treat AI (characters with ambiguous east Asian width), -# SA (South East Asian: Thai, Lao, Khmer) +# SA (Dictionary chars, excluding Mn and Mc) # SG (Unpaired Surrogates) # XX (Unknown, unassigned) # as $AL (Alphabetic) # -$ALPlus = [$AL $AI $SA $SG $XX]; - -# -# Combining Marks. X $CM* behaves as if it were X. Rule LB6. -# -$ALcm = $ALPlus $CM*; -$BAcm = $BA $CM*; -$BBcm = $BB $CM*; -$B2cm = $B2 $CM*; -$CLcm = $CL $CM*; -$CPcm = $CP $CM*; -$EXcm = $EX $CM*; -$GLcm = $GL $CM*; -$HLcm = $HL $CM*; -$HYcm = $HY $CM*; -$H2cm = $H2 $CM*; -$H3cm = $H3 $CM*; -$INcm = $IN $CM*; -$IScm = $IS $CM*; -$JLcm = $JL $CM*; -$JVcm = $JV $CM*; -$JTcm = $JT $CM*; -$NScm = $NS $CM*; -$NSXcm = $NSX $CM*; -$NUcm = $NU $CM*; -$OPcm = $OP $CM*; -$POcm = $PO $CM*; -$PRcm = $PR $CM*; -$QUcm = $QU $CM*; -$RIcm = $RI $CM*; -$SYcm = $SY $CM*; -$WJcm = $WJ $CM*; +$ALPlus = [$AL $AI $SG $XX [$SA-[[:Mn:][:Mc:]]]]; -## ------------------------------------------------- -!!forward; - -# -# Each class of character can stand by itself as an unbroken token, with trailing combining stuff -# -$ALPlus $CM+; -$BA $CM+; -$BB $CM+; -$B2 $CM+; -$CL $CM+; -$CP $CM+; -$EB $CM+; -$EM $CM+; -$EX $CM+; -$GL $CM+; -$HL $CM+; -$HY $CM+; -$H2 $CM+; -$H3 $CM+; -$ID $CM+; -$IN $CM+; -$IS $CM+; -$JL $CM+; -$JV $CM+; -$JT $CM+; -$NS $CM+; -$NSX $CM+; -$NU $CM+; -$OP $CM+; -$PO $CM+; -$PR $CM+; -$QU $CM+; -$RI $CM+; -$SY $CM+; -$WJ $CM+; +## ------------------------------------------------- # # CAN_CM is the set of characters that may combine with CM combining chars. @@ -214,12 +121,8 @@ $CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs # # AL_FOLLOW set of chars that can unconditionally follow an AL # Needed in rules where stand-alone $CM s are treated as AL. -# Chaining is disabled with CM because it causes other failures, -# so for this one case we need to manually list out longer sequences. # -$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP]; -$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus]; -$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM]; +$AL_FOLLOW = [$BK $CR $LF $NL $ZW $SP $CL $CP $EX $HL $IS $SY $WJ $GL $OP30 $QU $BA $HY $NS $IN $NU $PR $PO $ALPlus]; # @@ -244,16 +147,15 @@ $CAN_CM $CM* [$SP $ZW]; # # LB 8 Break after zero width space -# TODO: ZW SP* -# An engine change is required to write the reverse rule for this. -# For now, leave the Unicode 5.2 rule, ZW +# ZW SP* ÷ # $LB8Breaks = [$LB4Breaks $ZW]; $LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]]; +$ZW $SP* / [^$SP $ZW $LB4Breaks]; -# LB 8a ZWJ x ID Emoji proposal. +# LB 8a ZWJ x Do not break Emoji ZWJ sequences. # -$ZWJ ($ID | $EB | $EM); +$ZWJ [^$CM]; # LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL # $CM not covered by the above needs to behave like $AL @@ -265,31 +167,29 @@ $CAN_CM $CM+; # Stick together any combining sequences that d # # LB 11 Do not break before or after WORD JOINER & related characters. # -$CAN_CM $CM* $WJcm; -$LB8NonBreaks $WJcm; -^$CM+ $WJcm; +$CAN_CM $CM* $WJ; +$LB8NonBreaks $WJ; +^$CM+ $WJ; -$WJcm $CANT_CM; -$WJcm $CAN_CM $CM*; +$WJ $CM* .; # # LB 12 Do not break after NBSP and related characters. # GL x # -$GLcm $CAN_CM $CM*; -$GLcm $CANT_CM; +$GL $CM* .; # # LB 12a Do not break before NBSP and related characters ... # [^SP BA HY] x GL # -[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm; -^$CM+ $GLcm; +[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GL; +^$CM+ $GL; -# -# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces. + +# LB 13 Don't break before ']' or '!' or or '/', even after spaces. # $LB8NonBreaks $CL; $CAN_CM $CM* $CL; @@ -303,10 +203,6 @@ $LB8NonBreaks $EX; $CAN_CM $CM* $EX; ^$CM+ $EX; # by rule 10, stand-alone CM behaves as AL -$LB8NonBreaks $IS; -$CAN_CM $CM* $IS; -^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL - $LB8NonBreaks $SY; $CAN_CM $CM* $SY; ^$CM+ $SY; # by rule 10, stand-alone CM behaves as AL @@ -314,22 +210,46 @@ $CAN_CM $CM* $SY; # # LB 14 Do not break after OP, even after spaces +# Note subtle interaction with "SP IS /" rules in LB14a. +# This rule consumes the SP, chaining happens on the IS, effectivley overriding the SP IS rules, +# which is the desired behavior. +# +$OP $CM* $SP* .; + +$OP $CM* $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL + # by rule 8, CM following a SP is stand-alone. + + +# LB 14a Force a break before start of a number with a leading decimal pt, e.g. " .23" +# Note: would be simpler to express as "$SP / $IS $CM* $NU;", but ICU rules have limitations. +# See issue ICU-20303 + + +$CanFollowIS = [$BK $CR $LF $NL $SP $ZW $WJ $GL $CL $CP $EX $IS $SY $QU $BA $HY $NS $ALPlus $HL $IN]; +$SP $IS / [^ $CanFollowIS $NU $CM]; +$SP $IS $CM* $CMX / [^ $CanFollowIS $NU $CM]; + # -$OPcm $SP* $CAN_CM $CM*; -$OPcm $SP* $CANT_CM; +# LB 14b Do not break before numeric separators (IS), even after spaces. + +[$LB8NonBreaks - $SP] $IS; +$SP $IS $CM* [$CanFollowIS {eof}]; +$SP $IS $CM* $ZWJ [^$CM $NU]; + +$CAN_CM $CM* $IS; +^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL -$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL # LB 15 -$QUcm $SP* $OPcm; +$QU $CM* $SP* $OP; # LB 16 # Do not break between closing punctuation and $NS, even with intervening spaces # But DO allow a break between closing punctuation and $NSX, don't include it here -($CLcm | $CPcm) $SP* $NScm; +($CL | $CP) $CM* $SP* $NS; # LB 17 -$B2cm $SP* $B2cm; +$B2 $CM* $SP* $B2; # # LB 18 Break after spaces. @@ -340,408 +260,114 @@ $LB18Breaks = [$LB8Breaks $SP]; # LB 19 # x QU -$LB18NonBreaks $CM* $QUcm; -^$CM+ $QUcm; +$LB18NonBreaks $CM* $QU; +^$CM+ $QU; # QU x -$QUcm .?; - +$QU $CM* .; # LB 20 # $CB # $CB - +# $LB20NonBreaks = [$LB18NonBreaks - $CB]; +# LB 20.09 Don't break between Hyphens and Letters when there is a break preceding the hyphen. +# Originally added as a Finnish tailoring, now promoted to default ICU behavior. +# Note: this is not default UAX-14 behaviour. See issue ICU-8151. +# +^($HY | $HH) $CM* $ALPlus; + # LB 21 x (BA | HY | NS) # BB x # -# DO allow breaks here before NSXcm, so don't include it -$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm); -^$CM+ ($BAcm | $HYcm | $NScm); +# DO allow breaks here before NSX, so don't include it +$LB20NonBreaks $CM* ($BA | $HY | $NS); + -$BBcm [^$CB]; # $BB x -$BBcm $LB20NonBreaks $CM*; +^$CM+ ($BA | $HY | $NS); + +$BB $CM* [^$CB]; # $BB x +$BB $CM* $LB20NonBreaks; # LB 21a Don't break after Hebrew + Hyphen # HL (HY | BA) x # -$HLcm ($HYcm | $BAcm) [^$CB]?; +$HL $CM* ($HY | $BA) $CM* [^$CB]?; # LB 21b (forward) Don't break between SY and HL # (break between HL and SY already disallowed by LB 13 above) -$SYcm $HLcm; - -# LB 22 -($ALcm | $HLcm) $INcm; -^$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL -$EXcm $INcm; -($ID | $EB | $EM) $CM* $INcm; -# $INcm $INcm; # delete this rule for CSS loose -$NUcm $INcm; - +$SY $CM* $HL; -# $LB 23 -($ID | $EB | $EM) $CM* $POcm; -$ALcm $NUcm; # includes $LB19 -$HLcm $NUcm; -^$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL -$NUcm $ALcm; -$NUcm $HLcm; +# LB 22 Do not break before ellipses # -# LB 24 -# -$PRcm ($ID | $EB | $EM); -$PRcm ($ALcm | $HLcm); -$POcm ($ALcm | $HLcm); -($ALcm | $HLcm) ($PRcm | $POcm); -^$CM+ ($PRcm | $POcm); # Rule 10, any otherwise unattached CM behaves as AL - -# -# LB 25 Numbers. -# -($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?; - -# LB 26 Do not break a Korean syllable -# -$JLcm ($JLcm | $JVcm | $H2cm | $H3cm); -($JVcm | $H2cm) ($JVcm | $JTcm); -($JTcm | $H3cm) $JTcm; - -# LB 27 Treat korean Syllable Block the same as ID (don't break it) -($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm; -($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm; -$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm); - - -# LB 28 Do not break between alphabetics -# -($ALcm | $HLcm) ($ALcm | $HLcm); -^$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL - -# LB 29 -$IScm ($ALcm | $HLcm); - -# LB 30 -($ALcm | $HLcm | $NUcm) $OPcm; -^$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL. -$CPcm ($ALcm | $HLcm | $NUcm); - -# LB 30a Do not break between regional indicators. Break after pairs of them. -# Tricky interaction with LB8a: ZWJ x ID -$RIcm $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM] {eof}]; -$RIcm $RI $CM* $ZWJ / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM $ID $EB $EM] {eof}]; -$RIcm $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM] {eof}]; - -$RIcm $RIcm [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS {eof}]; -$RIcm $RIcm $ZWJ ($ID | $EB | $EM); - -# LB 30b Do not break between an Emoji Base and an Emoji Modifier -$EB $CM* $EM; - -# -# Reverse Rules. -# -## ------------------------------------------------- - -!!reverse; - -^$CM+ $ALPlus; -^$CM+ $BA; -^$CM+ $BB; -^$CM+ $B2; -^$CM+ $CL; -^$CM+ $CP; -^$CM+ $EB; -^$CM+ $EM; -^$CM+ $EX; -^$CM+ $GL; -^$CM+ $HL; -^$CM+ $HY; -^$CM+ $H2; -^$CM+ $H3; -^$CM+ $ID; +[$LB20NonBreaks - $IN] $CM* $IN; # line_loose tailoring ^$CM+ $IN; -^$CM+ $IS; -^$CM+ $JL; -^$CM+ $JV; -^$CM+ $JT; -^$CM+ $NS; -^$CM+ $NSX; -^$CM+ $NU; -^$CM+ $OP; -^$CM+ $PO; -^$CM+ $PR; -^$CM+ $QU; -^$CM+ $RI; -^$CM+ $SY; -^$CM+ $WJ; -^$CM+; - -# -# Sequences of the form (shown forwards) -# [CANT_CM] [CM] [whatever] -# The CM needs to behave as an AL -# -$AL_FOLLOW $CM+ / ( - [$BK $CR $LF $NL $ZW {eof}] | - $SP+ $CM+ $SP | - $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break. - # LB14 says OP SP* x . - # becomes OP SP* x AL - # becomes OP SP* x CM+ AL_FOLLOW - # - # Further note: the $AL in [$AL {eof}] is only to work around - # a rule compiler bug which complains about - # empty sets otherwise. +# LB 23 # -# Sequences of the form (shown forwards) -# [CANT_CM] [CM] [PR] -# The CM needs to behave as an AL -# This rule is concerned about getting the second of the two in place. -# - -# Apple early addition, remove this, superseded by LB24 -# [$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}]; - - - -# LB 4, 5, 6 - -$LB4Breaks [$LB4NonBreaks-$CM]; -$LB4Breaks $CM+ $CAN_CM; -$LF $CR; - - -# LB 7 x SP -# x ZW -[$SP $ZW] [$LB4NonBreaks-$CM]; -[$SP $ZW] $CM+ $CAN_CM; - -# LB 8 ZW SP* -# TODO: to implement this, we need more than one look-ahead hard break in play at a time. -# Requires an engine enhancement. -# / $SP* $ZW +($ALPlus | $HL) $CM* $NU; +^$CM+ $NU; # Rule 10, any otherwise unattached CM behaves as AL +$NU $CM* ($ALPlus | $HL); -# LB 8a ZWJ x ID Unicode Emoji proposal L2/16-011R3 -# The ZWJ will look like a CM to whatever precedes it. +# LB 23a # -($ID | $EB | $EM) $ZWJ $CM* $CAN_CM?; - - -# LB 9,10 Combining marks. -# X $CM needs to behave like X, where X is not $SP or controls. -# $CM not covered by the above needs to behave like $AL -# Stick together any combining sequences that don't match other rules. -^$CM+ $CAN_CM; +$PR $CM* ($ID | $EB | $EM); +($ID | $EB | $EM) $CM* $PO; -# LB 11 # -$WJ $CM* $CAN_CM; -$WJ [$LB8NonBreaks-$CM]; - - $CANT_CM $CM* $WJ; -$CAN_CM $CM* $WJ; - -# LB 12a -# [^SP BA HY] x GL +# LB 24 # -$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]]; +($PR | $PO) $CM* ($ALPlus | $HL); +($ALPlus | $HL) $CM* ($PR | $PO); +^$CM+ ($PR | $PO); # Rule 10, any otherwise unattached CM behaves as AL -# LB 12 -# GL x # -$CANT_CM $CM* $GL; -$CAN_CM $CM* $GL; - - -# LB 13 -$CL $CM+ $CAN_CM; -$CP $CM+ $CAN_CM; -$EX $CM+ $CAN_CM; -$IS $CM+ $CAN_CM; -$SY $CM+ $CAN_CM; - -$CL [$LB8NonBreaks-$CM]; -$CP [$LB8NonBreaks-$CM]; -$EX [$LB8NonBreaks-$CM]; -$IS [$LB8NonBreaks-$CM]; -$SY [$LB8NonBreaks-$CM]; - -# Rule 13 & 14 taken together for an edge case. -# Match this, shown forward -# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY) -# This really wants to chain at the $CM+ (which is acting as an $AL) -# except for $CM chaining being disabled. -[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP; - -# LB 14 OP SP* x +# LB 25 Numbers. # -$CAN_CM $SP* $CM* $OP; - $CANT_CM $SP* $CM* $OP; -$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP - - $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP; -$AL_FOLLOW_CM $CM+ $SP+ $CM* $OP; - - -# LB 15 -$OP $SP* $CM* $QU; - -# LB 16 -# Don't include $NSX here -$NS $SP* $CM* ($CL | $CP); +(($PR | $PO) $CM*)? (($OP | $HY) $CM*)? ($IS $CM*)? $NU ($CM* ($NU | $SY | $IS))* + ($CM* ($CL | $CP))? ($CM* ($PR | $PO))?; -# LB 17 -$B2 $SP* $CM* $B2; - -# LB 18 break after spaces -# Nothing explicit needed here. - - -# -# LB 19 +# LB 26 Do not break a Korean syllable # -$QU $CM* $CAN_CM; # . x QU -$QU $LB18NonBreaks; +$JL $CM* ($JL | $JV | $H2 | $H3); +($JV | $H2) $CM* ($JV | $JT); +($JT | $H3) $CM* $JT; +# LB 27 Treat korean Syllable Block the same as ID (don't break it) +($JL | $JV | $JT | $H2 | $H3) $CM* $IN; +($JL | $JV | $JT | $H2 | $H3) $CM* $PO; +$PR $CM* ($JL | $JV | $JT | $H2 | $H3); -$CAN_CM $CM* $QU; # QU x . - $CANT_CM $CM* $QU; +# LB 28 Do not break between alphabetics # -# LB 20 Break before and after CB. -# nothing needed here. -# - -# LB 21 -# Don't include $NSX here -($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS) - -[$LB20NonBreaks-$CM] $CM* $BB; # BB x . -[^$CB] $CM* $BB; # - -# LB21a -[^$CB] $CM* ($HY | $BA) $CM* $HL; - -# LB21b (reverse) -$HL $CM* $SY; - -# LB 22 -$IN $CM* ($ALPlus | $HL); -$IN $CM* $EX; -$IN $CM* ($ID | $EB | $EM); -# $IN $CM* $IN; # delete this rule for CSS loose -$IN $CM* $NU; - -# LB 23 -$PO $CM* ($ID | $EB | $EM); -$NU $CM* ($ALPlus | $HL); -($ALPlus | $HL) $CM* $NU; - -# LB 24 -($ID | $EB | $EM) $CM* $PR; -($ALPlus | $HL) $CM* $PR; -($ALPlus | $HL) $CM* $PO; -$CM* ($PR | $PO) $CM* ($ALPlus | $HL); -$CM* ($PR | $PO) $CM+ / [$BK $CR $LF $NL $ZW $SP {eof}]; - -# LB 25 -($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?; - -# LB 26 -($H3 | $H2 | $JV | $JL) $CM* $JL; -($JT | $JV) $CM* ($H2 | $JV); -$JT $CM* ($H3 | $JT); - -# LB 27 -$IN $CM* ($H3 | $H2 | $JT | $JV | $JL); -$PO $CM* ($H3 | $H2 | $JT | $JV | $JL); - ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR; - -# LB 28 ($ALPlus | $HL) $CM* ($ALPlus | $HL); - +^$CM+ ($ALPlus | $HL); # The $CM+ is from rule 10, an unattached CM is treated as AL # LB 29 -($ALPlus | $HL) $CM* $IS; +$IS $CM* ($ALPlus | $HL); # LB 30 -$OP $CM* ($ALPlus | $HL | $NU); -($ALPlus | $HL | $NU) $CM* $CP; - -# LB 30a -# Pairs of Regional Indicators. -# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs, -# the second with an even number. Stripping away the cruft they look like -# [^RI] RI / (RI RI)+ ^RI; -# [^RI] RI RI / (RI RI)+ ^RI; -# -# Line Loose tailoring: Don't include NSX here. -[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]]; -[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]]; - -# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs. -$RI $CM* $RI; - -# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI". -$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL)); +($ALPlus | $HL | $NU) $CM* $OP30; +^$CM+ $OP30; # The $CM+ is from rule 10, an unattached CM is treated as AL. +$CP30 $CM* ($ALPlus | $HL | $NU); +# LB 30a Do not break between regional indicators. Break after pairs of them. +# Tricky interaction with LB8a: ZWJ x . together with ZWJ acting like a CM. +$RI $CM* $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $CM]]; +$RI $CM* $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $CM]]; +$RI $CM* $RI $CM* [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $ZWJ {eof}]; +# note: the preceding rule includes {eof} rather than having the last [set] term qualified with '?' +# because of the chain-out behavior difference. The rule must chain out only from the [set characters], +# not from the preceding $RI or $CM, which it would be able to do if the set were optional. # LB 30b Do not break between an Emoji Base and an Emoji Modifier -$EM $CM* $EB; - - -## ------------------------------------------------- - -!!safe_reverse; - -# LB 9 -^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP]; -^$CM+ $SP / .; - -# LB 14 -$SP+ $CM* $OP; - -# LB 15 -$SP+ $CM* $QU; - -# LB 16 -$SP+ $CM* ($CL | $CP); - -# LB 17 -$SP+ $CM* $B2; - -# LB 21 -$CM* ($HY | $BA) $CM* $HL; - -# LB 25 -($CM* ($IS | $SY))+ $CM* $NU; -($CL | $CP) $CM* ($NU | $IS | $SY); - -# LB 30 -($CM* $RI)+; - -# For dictionary-based break -$dictionary $dictionary; - -## ------------------------------------------------- - -!!safe_forward; - -# Skip forward over all character classes that are involved in -# rules containing patterns with possibly more than one char -# of context. -# -# It might be slightly more efficient to have specific rules -# instead of one generic one, but only if we could -# turn off rule chaining. We don't want to move more -# than necessary. -# -^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary]; -$dictionary $dictionary; +$EB $CM* $EM; +# LB 31 Break everywhere else. +# Match a single code point if no other rule applies. +.;