ICU-461.12.tar.gz

[apple/icu.git] / icuSources / data / brkitr / line.txt

diff --git a/icuSources/data/brkitr/line.txt b/icuSources/data/brkitr/line.txt
index 262ed0d3c22818f4c86a29e3c806f2f0605a63d6..bd27b5e004798286e89c77f1abdbcf436b29d675 100644 (file)
--- a/icuSources/data/brkitr/line.txt
+++ b/icuSources/data/brkitr/line.txt
@@ -1,14 +1,16 @@
-# Copyright (c) 2002-2008  International Business Machines Corporation and
+# Copyright (c) 2002-2010  International Business Machines Corporation and
 # others. All Rights Reserved.
 #
 #  file:  line.txt
 #
 #         Line Breaking Rules
 #         Implement default line breaking as defined by 
-#         Unicode Standard Annex #14 Revision 21 for Unicode 5.1
+#         Unicode Standard Annex #14 Revision 24 for Unicode 6.0
 #         http://www.unicode.org/reports/tr14/
-
-
+#
+#         TODO:  Rule LB 8 remains as it was in Unicode 5.2
+#         This is only because of a limitation of ICU break engine implementation,
+#         not because the older behavior is desirable.
 
 #
 #  Character Classes defined by TR 14.
@@ -23,7 +25,7 @@
 #  !!lookAheadHardBreak    Described here because it is (as yet) undocumented elsewhere
 #                          and only used for the line break rules.
 #
-#           It is used in the implementation of the incredibly annoying rule LB 10
+#           It is used in the implementation of rule LB 10
 #           which says to treat any combining mark that is not attached to a base
 #           character as if it were of class AL  (alphabetic).
 #
@@ -63,6 +65,7 @@ $B2 = [:LineBreak =  Break_Both:];
 $CB = [:LineBreak =  Contingent_Break:];
 $CL = [:LineBreak =  Close_Punctuation:];
 $CM = [:LineBreak =  Combining_Mark:];
+$CP = [:LineBreak =  Close_Parenthesis:];
 $CR = [:LineBreak =  Carriage_Return:];
 $EX = [:LineBreak =  Exclamation:];
 $GL = [:LineBreak =  Glue:];
@@ -115,6 +118,7 @@ $BAcm = $BA $CM*;
 $BBcm = $BB $CM*;
 $B2cm = $B2 $CM*;
 $CLcm = $CL $CM*;
+$CPcm = $CP $CM*;
 $EXcm = $EX $CM*;
 $GLcm = $GL $CM*;
 $HYcm = $HY $CM*;
@@ -147,6 +151,7 @@ $BA $CM+;
 $BB $CM+;
 $B2 $CM+;
 $CL $CM+;
+$CP $CM+;
 $EX $CM+;
 $GL $CM+;
 $HY $CM+;
@@ -185,7 +190,7 @@ $CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM];       # Bases that can't take CMs
 #            so for this one case we need to manually list out longer sequences.
 #
 $AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
-$AL_FOLLOW_CM   = [$CL $EX $IS $SY $WJ $GL $QU $BA $HY $NS $IN $NU $ALPlus];
+$AL_FOLLOW_CM   = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus];
 $AL_FOLLOW      = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
 
 
@@ -211,6 +216,9 @@ $CM+          [$SP $ZW];
 
 #
 # LB 8         Break after zero width space
+#              TODO:  ZW SP* <break>
+#              An engine change is required to write the reverse rule for this.
+#              For now, leave the Unicode 5.2 rule, ZW <break>
 #
 $LB8Breaks    = [$LB4Breaks $ZW];
 $LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
@@ -256,6 +264,10 @@ $LB8NonBreaks $CL;
 $CAN_CM $CM*  $CL;
 $CM+          $CL;              # by rule 10, stand-alone CM behaves as AL
 
+$LB8NonBreaks $CP;
+$CAN_CM $CM*  $CP;
+$CM+          $CP;              # by rule 10, stand-alone CM behaves as AL
+
 $LB8NonBreaks $EX;
 $CAN_CM $CM*  $EX;
 $CM+          $EX;              # by rule 10, stand-alone CM behaves as AL
@@ -281,7 +293,7 @@ $OPcm $SP+ $CM+ $AL_FOLLOW?;    # by rule 10, stand-alone CM behaves as AL
 $QUcm $SP* $OPcm;
 
 # LB 16
-$CLcm $SP* $NScm;
+($CLcm | $CPcm) $SP* $NScm;
 
 # LB 17
 $B2cm $SP* $B2cm;
@@ -342,7 +354,7 @@ $POcm $ALcm;
 #
 # LB 25   Numbers.
 #
-($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* $CLcm? ($PRcm | $POcm)?;
+($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
 
 # LB 26  Do not break a Korean syllable
 #
@@ -359,11 +371,16 @@ $PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
 # LB 28   Do not break between alphabetics
 #
 $ALcm $ALcm;
-$CM+ $ALcm;      # The $CM+ is from rule 10, and unattached CM is treated as AL
+$CM+ $ALcm;      # The $CM+ is from rule 10, an unattached CM is treated as AL
 
 # LB 29
 $IScm $ALcm;
 
+# LB 30
+($ALcm | $NUcm) $OPcm;
+$CM+ $OPcm;         # The $CM+ is from rule 10, an unattached CM is treated as AL.          
+$CPcm ($ALcm | $NUcm);
+
 
 #
 #  Reverse Rules.
@@ -377,6 +394,7 @@ $CM+ $BA;
 $CM+ $BB;
 $CM+ $B2;
 $CM+ $CL;
+$CM+ $CP;
 $CM+ $EX;
 $CM+ $GL;
 $CM+ $HY;
@@ -439,8 +457,10 @@ $LF $CR;
 [$SP $ZW] [$LB4NonBreaks-$CM];
 [$SP $ZW] $CM+ $CAN_CM;
 
-# LB 8 Break after zero width space
-
+# LB 8 ZW SP* <break>
+#     TODO: to implement this, we need more than one look-ahead hard break in play at a time.
+#           Requires an engine enhancement.
+#   / $SP* $ZW
 
 # LB 9,10  Combining marks.
 #    X   $CM needs to behave like X, where X is not $SP or controls.
@@ -470,21 +490,23 @@ $CM* $CAN_CM $CM* $GL;
 
 # LB 13
 $CL $CM+ $CAN_CM;
+$CP $CM+ $CAN_CM;
 $EX $CM+ $CAN_CM;
 $IS $CM+ $CAN_CM;
 $SY $CM+ $CAN_CM;
 
 $CL [$LB8NonBreaks-$CM];
+$CP [$LB8NonBreaks-$CM];
 $EX [$LB8NonBreaks-$CM];
 $IS [$LB8NonBreaks-$CM];
 $SY [$LB8NonBreaks-$CM];
 
 # Rule 13 & 14 taken together for an edge case.
 #   Match this, shown forward
-#     OP SP+  ($CM+ behaving as $AL) (CL | EX | IS | IY)
+#     OP SP+  ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
 #   This really wants to chain at the $CM+ (which is acting as an $AL)
 #   except for $CM chaining being disabled.
-[$CL $EX $IS $SY] $CM+ $SP+ $CM* $OP;  
+[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;  
 
 # LB 14    OP SP* x
 #
@@ -502,7 +524,7 @@ $SY $CM $SP+ $OP;   # TODO:  Experiment.  Remove.
 $CM* $OP $SP* $CM* $QU;
 
 # LB 16
-$CM* $NS $SP* $CM* $CL;
+$CM* $NS $SP* $CM* ($CL | $CP);
 
 # LB 17
 $CM* $B2 $SP* $CM* $B2;
@@ -552,7 +574,7 @@ $CM* $ALPlus $CM* $PO;
 
 
 # LB 25
-($CM* ($PR | $PO))? ($CM* $CL)? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
+($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
 
 # LB 26
 $CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
@@ -571,6 +593,10 @@ $CM* $ALPlus $CM* $ALPlus;
 # LB 29
 $CM* $ALPlus $CM* $IS;
 
+# LB 30
+$CM* $OP $CM* ($ALPlus | $NU);
+$CM* ($ALPlus | $NU) $CM* $CP;
+
 
 ## -------------------------------------------------
 
@@ -587,14 +613,14 @@ $SP+ $CM* $OP;
 $SP+ $CM* $QU;
 
 # LB 16
-$SP+ $CM* $CL;
+$SP+ $CM* ($CL | $CP);
 
 # LB 17
 $SP+ $CM* $B2;
 
 # LB 25
 ($CM* ($IS | $SY))+ $CM* $NU;
-$CL $CM* ($NU | $IS | $SY);
+($CL | $CP) $CM* ($NU | $IS | $SY);
 
 # For dictionary-based break
 $dictionary $dictionary;
@@ -612,6 +638,6 @@ $dictionary $dictionary;
 #  turn off rule chaining.  We don't want to move more
 #  than necessary.
 #
-[$CM $OP $QU $CL $B2 $PR $HY $SP $dictionary]+ [^$CM $OP $QU $CL $B2 $PR $HY $dictionary];
+[$CM $OP $QU $CL $CP $B2 $PR $HY $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $dictionary];
 $dictionary $dictionary;