]>
Commit | Line | Data |
---|---|---|
0f5d89e8 | 1 | # Copyright (C) 2016 and later: Unicode, Inc. and others. |
f3c0d7a5 | 2 | # License & terms of use: http://www.unicode.org/copyright.html |
2ca993e8 | 3 | # Copyright (c) 2002-2016 International Business Machines Corporation and |
729e4ab9 A |
4 | # others. All Rights Reserved. |
5 | # | |
3d1f044b | 6 | # file: line_cj.txt |
729e4ab9 A |
7 | # |
8 | # Line Breaking Rules | |
2ca993e8 | 9 | # Implement default line breaking as defined by |
3d1f044b A |
10 | # Unicode Standard Annex #14 Revision 42 for Unicode 12.0 |
11 | # http://www.unicode.org/reports/tr14/, with the following modification: | |
729e4ab9 | 12 | # |
3d1f044b A |
13 | # Boundaries between hyphens and following letters are suppressed when |
14 | # there is a boundary preceding the hyphen. See rule 20.9 | |
f3c0d7a5 A |
15 | # |
16 | # This corresponds to CSS line-break=strict (BCP47 -u-lb-strict). | |
17 | # It sets characters of class CJ to behave like NS. | |
3d1f044b | 18 | # It allows breaking before 201C and after 201D, for zh_Hans, zh_Hant, and ja. |
729e4ab9 A |
19 | |
20 | # | |
21 | # Character Classes defined by TR 14. | |
22 | # | |
23 | ||
24 | !!chain; | |
0f5d89e8 | 25 | !!quoted_literals_only; |
729e4ab9 | 26 | |
f3c0d7a5 A |
27 | $AI = [:LineBreak = Ambiguous:]; |
28 | $AL = [:LineBreak = Alphabetic:]; | |
2ca993e8 | 29 | $BA = [:LineBreak = Break_After:]; |
f3c0d7a5 | 30 | $HH = [\u2010]; # \u2010 is HYPHEN, default line break is BA. |
729e4ab9 A |
31 | $BB = [:LineBreak = Break_Before:]; |
32 | $BK = [:LineBreak = Mandatory_Break:]; | |
33 | $B2 = [:LineBreak = Break_Both:]; | |
34 | $CB = [:LineBreak = Contingent_Break:]; | |
4388f060 | 35 | $CJ = [:LineBreak = Conditional_Japanese_Starter:]; |
3d1f044b | 36 | $CL = [[:LineBreak = Close_Punctuation:] \u201d]; |
f3c0d7a5 | 37 | # $CM = [:LineBreak = Combining_Mark:]; |
729e4ab9 A |
38 | $CP = [:LineBreak = Close_Parenthesis:]; |
39 | $CR = [:LineBreak = Carriage_Return:]; | |
3d1f044b | 40 | $EB = [:LineBreak = EB:]; |
f3c0d7a5 | 41 | $EM = [:LineBreak = EM:]; |
729e4ab9 A |
42 | $EX = [:LineBreak = Exclamation:]; |
43 | $GL = [:LineBreak = Glue:]; | |
4388f060 | 44 | $HL = [:LineBreak = Hebrew_Letter:]; |
729e4ab9 A |
45 | $HY = [:LineBreak = Hyphen:]; |
46 | $H2 = [:LineBreak = H2:]; | |
47 | $H3 = [:LineBreak = H3:]; | |
f3c0d7a5 | 48 | $ID = [:LineBreak = Ideographic:]; |
729e4ab9 A |
49 | $IN = [:LineBreak = Inseperable:]; |
50 | $IS = [:LineBreak = Infix_Numeric:]; | |
51 | $JL = [:LineBreak = JL:]; | |
52 | $JV = [:LineBreak = JV:]; | |
53 | $JT = [:LineBreak = JT:]; | |
54 | $LF = [:LineBreak = Line_Feed:]; | |
55 | $NL = [:LineBreak = Next_Line:]; | |
f3c0d7a5 | 56 | # NS includes CJ for CSS strict line breaking. |
4388f060 | 57 | $NS = [[:LineBreak = Nonstarter:] $CJ]; |
729e4ab9 | 58 | $NU = [:LineBreak = Numeric:]; |
3d1f044b | 59 | $OP = [[:LineBreak = Open_Punctuation:] \u201c]; |
729e4ab9 A |
60 | $PO = [:LineBreak = Postfix_Numeric:]; |
61 | $PR = [:LineBreak = Prefix_Numeric:]; | |
3d1f044b | 62 | $QU = [[:LineBreak = Quotation:] - [\u201c\u201d]]; |
51004dcb | 63 | $RI = [:LineBreak = Regional_Indicator:]; |
729e4ab9 A |
64 | $SA = [:LineBreak = Complex_Context:]; |
65 | $SG = [:LineBreak = Surrogate:]; | |
66 | $SP = [:LineBreak = Space:]; | |
67 | $SY = [:LineBreak = Break_Symbols:]; | |
68 | $WJ = [:LineBreak = Word_Joiner:]; | |
69 | $XX = [:LineBreak = Unknown:]; | |
70 | $ZW = [:LineBreak = ZWSpace:]; | |
f3c0d7a5 A |
71 | $ZWJ = [:LineBreak = ZWJ:]; |
72 | ||
f3c0d7a5 A |
73 | # By LB9, a ZWJ also behaves as a CM. Including it in the definition of CM avoids having to explicitly |
74 | # list it in the numerous rules that use CM. | |
75 | # By LB1, SA characters with general categor of Mn or Mc also resolve to CM. | |
76 | ||
77 | $CM = [[:LineBreak = Combining_Mark:] $ZWJ [$SA & [[:Mn:][:Mc:]]]]; | |
3d1f044b | 78 | $CMX = [[$CM] - [$ZWJ]]; |
08b89b0a | 79 | |
729e4ab9 | 80 | # Dictionary character set, for triggering language-based break engines. Currently |
f3c0d7a5 | 81 | # limited to LineBreak=Complex_Context (SA). |
729e4ab9 | 82 | |
f3c0d7a5 | 83 | $dictionary = [$SA]; |
729e4ab9 A |
84 | |
85 | # | |
86 | # Rule LB1. By default, treat AI (characters with ambiguous east Asian width), | |
f3c0d7a5 | 87 | # SA (Dictionary chars, excluding Mn and Mc) |
729e4ab9 A |
88 | # SG (Unpaired Surrogates) |
89 | # XX (Unknown, unassigned) | |
90 | # as $AL (Alphabetic) | |
91 | # | |
f3c0d7a5 A |
92 | $ALPlus = [$AL $AI $SG $XX [$SA-[[:Mn:][:Mc:]]]]; |
93 | ||
729e4ab9 A |
94 | |
95 | ## ------------------------------------------------- | |
96 | ||
729e4ab9 A |
97 | # |
98 | # CAN_CM is the set of characters that may combine with CM combining chars. | |
99 | # Note that Linebreak UAX 14's concept of a combining char and the rules | |
100 | # for what they can combine with are _very_ different from the rest of Unicode. | |
101 | # | |
102 | # Note that $CM itself is left out of this set. If CM is needed as a base | |
103 | # it must be listed separately in the rule. | |
104 | # | |
105 | $CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs | |
106 | $CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs | |
107 | ||
108 | # | |
109 | # AL_FOLLOW set of chars that can unconditionally follow an AL | |
110 | # Needed in rules where stand-alone $CM s are treated as AL. | |
729e4ab9 | 111 | # |
3d1f044b | 112 | $AL_FOLLOW = [$BK $CR $LF $NL $ZW $SP $CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $PR $PO $ALPlus]; |
729e4ab9 A |
113 | |
114 | ||
115 | # | |
116 | # Rule LB 4, 5 Mandatory (Hard) breaks. | |
117 | # | |
118 | $LB4Breaks = [$BK $CR $LF $NL]; | |
2ca993e8 | 119 | $LB4NonBreaks = [^$BK $CR $LF $NL $CM]; |
729e4ab9 A |
120 | $CR $LF {100}; |
121 | ||
122 | # | |
123 | # LB 6 Do not break before hard line breaks. | |
124 | # | |
125 | $LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks. | |
126 | $CAN_CM $CM* $LB4Breaks {100}; | |
2ca993e8 | 127 | ^$CM+ $LB4Breaks {100}; |
729e4ab9 A |
128 | |
129 | # LB 7 x SP | |
130 | # x ZW | |
131 | $LB4NonBreaks [$SP $ZW]; | |
132 | $CAN_CM $CM* [$SP $ZW]; | |
2ca993e8 | 133 | ^$CM+ [$SP $ZW]; |
729e4ab9 A |
134 | |
135 | # | |
136 | # LB 8 Break after zero width space | |
3d1f044b | 137 | # ZW SP* รท |
729e4ab9 A |
138 | # |
139 | $LB8Breaks = [$LB4Breaks $ZW]; | |
140 | $LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]]; | |
3d1f044b | 141 | $ZW $SP* / [^$SP $ZW $LB4Breaks]; |
729e4ab9 | 142 | |
0f5d89e8 | 143 | # LB 8a ZWJ x Do not break Emoji ZWJ sequences. |
2ca993e8 | 144 | # |
0f5d89e8 | 145 | $ZWJ [^$CM]; |
729e4ab9 | 146 | |
2ca993e8 A |
147 | # LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL |
148 | # $CM not covered by the above needs to behave like $AL | |
729e4ab9 A |
149 | # See definition of $CAN_CM. |
150 | ||
151 | $CAN_CM $CM+; # Stick together any combining sequences that don't match other rules. | |
2ca993e8 | 152 | ^$CM+; |
729e4ab9 A |
153 | |
154 | # | |
155 | # LB 11 Do not break before or after WORD JOINER & related characters. | |
156 | # | |
f3c0d7a5 A |
157 | $CAN_CM $CM* $WJ; |
158 | $LB8NonBreaks $WJ; | |
159 | ^$CM+ $WJ; | |
729e4ab9 | 160 | |
f3c0d7a5 | 161 | $WJ $CM* .; |
729e4ab9 A |
162 | |
163 | # | |
164 | # LB 12 Do not break after NBSP and related characters. | |
165 | # GL x | |
166 | # | |
f3c0d7a5 | 167 | $GL $CM* .; |
2ca993e8 | 168 | |
729e4ab9 A |
169 | # |
170 | # LB 12a Do not break before NBSP and related characters ... | |
171 | # [^SP BA HY] x GL | |
172 | # | |
3d1f044b | 173 | [[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GL; |
f3c0d7a5 | 174 | ^$CM+ $GL; |
729e4ab9 A |
175 | |
176 | ||
177 | ||
3d1f044b A |
178 | |
179 | # LB 13 Don't break before ']' or '!' or or '/', even after spaces. | |
729e4ab9 A |
180 | # |
181 | $LB8NonBreaks $CL; | |
182 | $CAN_CM $CM* $CL; | |
2ca993e8 | 183 | ^$CM+ $CL; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
184 | |
185 | $LB8NonBreaks $CP; | |
186 | $CAN_CM $CM* $CP; | |
2ca993e8 | 187 | ^$CM+ $CP; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
188 | |
189 | $LB8NonBreaks $EX; | |
190 | $CAN_CM $CM* $EX; | |
2ca993e8 | 191 | ^$CM+ $EX; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 | 192 | |
729e4ab9 A |
193 | $LB8NonBreaks $SY; |
194 | $CAN_CM $CM* $SY; | |
2ca993e8 | 195 | ^$CM+ $SY; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
196 | |
197 | ||
198 | # | |
199 | # LB 14 Do not break after OP, even after spaces | |
3d1f044b A |
200 | # Note subtle interaction with "SP IS /" rules in LB14a. |
201 | # This rule consumes the SP, chaining happens on the IS, effectivley overriding the SP IS rules, | |
202 | # which is the desired behavior. | |
729e4ab9 | 203 | # |
f3c0d7a5 | 204 | $OP $CM* $SP* .; |
729e4ab9 | 205 | |
f3c0d7a5 A |
206 | $OP $CM* $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL |
207 | # by rule 8, CM following a SP is stand-alone. | |
729e4ab9 | 208 | |
3d1f044b A |
209 | |
210 | # LB 14a Force a break before start of a number with a leading decimal pt, e.g. " .23" | |
211 | # Note: would be simpler to express as "$SP / $IS $CM* $NU;", but ICU rules have limitations. | |
212 | # See issue ICU-20303 | |
213 | ||
214 | ||
215 | $CanFollowIS = [$BK $CR $LF $NL $SP $ZW $WJ $GL $CL $CP $EX $IS $SY $QU $BA $HY $NS $ALPlus $HL]; | |
216 | $SP $IS / [^ $CanFollowIS $NU $CM]; | |
217 | $SP $IS $CM* $CMX / [^ $CanFollowIS $NU $CM]; | |
218 | ||
219 | # | |
220 | # LB 14b Do not break before numeric separators (IS), even after spaces. | |
221 | ||
222 | [$LB8NonBreaks - $SP] $IS; | |
223 | $SP $IS $CM* [$CanFollowIS {eof}]; | |
224 | $SP $IS $CM* $ZWJ [^$CM $NU]; | |
225 | ||
226 | $CAN_CM $CM* $IS; | |
227 | ^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL | |
228 | ||
229 | ||
729e4ab9 | 230 | # LB 15 |
f3c0d7a5 | 231 | $QU $CM* $SP* $OP; |
729e4ab9 A |
232 | |
233 | # LB 16 | |
f3c0d7a5 | 234 | ($CL | $CP) $CM* $SP* $NS; |
729e4ab9 A |
235 | |
236 | # LB 17 | |
f3c0d7a5 | 237 | $B2 $CM* $SP* $B2; |
729e4ab9 A |
238 | |
239 | # | |
240 | # LB 18 Break after spaces. | |
241 | # | |
242 | $LB18NonBreaks = [$LB8NonBreaks - [$SP]]; | |
243 | $LB18Breaks = [$LB8Breaks $SP]; | |
244 | ||
245 | ||
246 | # LB 19 | |
247 | # x QU | |
f3c0d7a5 A |
248 | $LB18NonBreaks $CM* $QU; |
249 | ^$CM+ $QU; | |
729e4ab9 A |
250 | |
251 | # QU x | |
f3c0d7a5 | 252 | $QU $CM* .; |
729e4ab9 | 253 | |
729e4ab9 A |
254 | # LB 20 |
255 | # <break> $CB | |
256 | # $CB <break> | |
3d1f044b | 257 | # |
729e4ab9 A |
258 | $LB20NonBreaks = [$LB18NonBreaks - $CB]; |
259 | ||
3d1f044b A |
260 | # LB 20.09 Don't break between Hyphens and Letters when there is a break preceding the hyphen. |
261 | # Originally added as a Finnish tailoring, now promoted to default ICU behavior. | |
262 | # Note: this is not default UAX-14 behaviour. See issue ICU-8151. | |
263 | # | |
264 | ^($HY | $HH) $CM* $ALPlus; | |
265 | ||
729e4ab9 A |
266 | # LB 21 x (BA | HY | NS) |
267 | # BB x | |
268 | # | |
3d1f044b A |
269 | $LB20NonBreaks $CM* ($BA | $HY | $NS); |
270 | ||
271 | ||
272 | ^$CM+ ($BA | $HY | $NS); | |
729e4ab9 | 273 | |
f3c0d7a5 A |
274 | $BB $CM* [^$CB]; # $BB x |
275 | $BB $CM* $LB20NonBreaks; | |
729e4ab9 | 276 | |
4388f060 A |
277 | # LB 21a Don't break after Hebrew + Hyphen |
278 | # HL (HY | BA) x | |
f3c0d7a5 | 279 | # |
3d1f044b | 280 | $HL $CM* ($HY | $BA) $CM* [^$CB]?; |
4388f060 | 281 | |
57a6839d A |
282 | # LB 21b (forward) Don't break between SY and HL |
283 | # (break between HL and SY already disallowed by LB 13 above) | |
f3c0d7a5 | 284 | $SY $CM* $HL; |
57a6839d | 285 | |
729e4ab9 | 286 | # LB 22 |
f3c0d7a5 A |
287 | ($ALPlus | $HL) $CM* $IN; |
288 | ^$CM+ $IN; # by rule 10, any otherwise unattached CM behaves as AL | |
289 | $EX $CM* $IN; | |
290 | ($ID | $EB | $EM) $CM* $IN; | |
291 | $IN $CM* $IN; | |
292 | $NU $CM* $IN; | |
729e4ab9 A |
293 | |
294 | ||
295 | # $LB 23 | |
f3c0d7a5 A |
296 | # |
297 | ($ALPlus | $HL) $CM* $NU; | |
298 | ^$CM+ $NU; # Rule 10, any otherwise unattached CM behaves as AL | |
299 | $NU $CM* ($ALPlus | $HL); | |
300 | ||
301 | # LB 23a | |
302 | # | |
303 | $PR $CM* ($ID | $EB | $EM); | |
304 | ($ID | $EB | $EM) $CM* $PO; | |
305 | ||
729e4ab9 A |
306 | |
307 | # | |
308 | # LB 24 | |
309 | # | |
f3c0d7a5 A |
310 | ($PR | $PO) $CM* ($ALPlus | $HL); |
311 | ($ALPlus | $HL) $CM* ($PR | $PO); | |
312 | ^$CM+ ($PR | $PO); # Rule 10, any otherwise unattached CM behaves as AL | |
729e4ab9 A |
313 | |
314 | # | |
315 | # LB 25 Numbers. | |
316 | # | |
3d1f044b | 317 | (($PR | $PO) $CM*)? (($OP | $HY) $CM*)? ($IS $CM*)? $NU ($CM* ($NU | $SY | $IS))* |
f3c0d7a5 | 318 | ($CM* ($CL | $CP))? ($CM* ($PR | $PO))?; |
729e4ab9 A |
319 | |
320 | # LB 26 Do not break a Korean syllable | |
321 | # | |
f3c0d7a5 A |
322 | $JL $CM* ($JL | $JV | $H2 | $H3); |
323 | ($JV | $H2) $CM* ($JV | $JT); | |
324 | ($JT | $H3) $CM* $JT; | |
729e4ab9 A |
325 | |
326 | # LB 27 Treat korean Syllable Block the same as ID (don't break it) | |
f3c0d7a5 A |
327 | ($JL | $JV | $JT | $H2 | $H3) $CM* $IN; |
328 | ($JL | $JV | $JT | $H2 | $H3) $CM* $PO; | |
329 | $PR $CM* ($JL | $JV | $JT | $H2 | $H3); | |
729e4ab9 A |
330 | |
331 | ||
332 | # LB 28 Do not break between alphabetics | |
333 | # | |
f3c0d7a5 A |
334 | ($ALPlus | $HL) $CM* ($ALPlus | $HL); |
335 | ^$CM+ ($ALPlus | $HL); # The $CM+ is from rule 10, an unattached CM is treated as AL | |
729e4ab9 A |
336 | |
337 | # LB 29 | |
f3c0d7a5 | 338 | $IS $CM* ($ALPlus | $HL); |
729e4ab9 A |
339 | |
340 | # LB 30 | |
f3c0d7a5 A |
341 | ($ALPlus | $HL | $NU) $CM* $OP; |
342 | ^$CM+ $OP; # The $CM+ is from rule 10, an unattached CM is treated as AL. | |
343 | $CP $CM* ($ALPlus | $HL | $NU); | |
729e4ab9 | 344 | |
2ca993e8 | 345 | # LB 30a Do not break between regional indicators. Break after pairs of them. |
0f5d89e8 A |
346 | # Tricky interaction with LB8a: ZWJ x . together with ZWJ acting like a CM. |
347 | $RI $CM* $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM]]; | |
348 | $RI $CM* $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM]]; | |
349 | $RI $CM* $RI $CM* [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $ZWJ {eof}]; | |
350 | # note: the preceding rule includes {eof} rather than having the last [set] term qualified with '?' | |
351 | # because of the chain-out behavior difference. The rule must chain out only from the [set characters], | |
352 | # not from the preceding $RI or $CM, which it would be able to do if the set were optional. | |
729e4ab9 | 353 | |
2ca993e8 A |
354 | # LB 30b Do not break between an Emoji Base and an Emoji Modifier |
355 | $EB $CM* $EM; | |
08b89b0a | 356 | |
0f5d89e8 A |
357 | # LB 31 Break everywhere else. |
358 | # Match a single code point if no other rule applies. | |
359 | .; |