]>
Commit | Line | Data |
---|---|---|
2ca993e8 | 1 | # Copyright (c) 2002-2016 International Business Machines Corporation and |
729e4ab9 A |
2 | # others. All Rights Reserved. |
3 | # | |
4 | # file: line_fi.txt | |
5 | # | |
6 | # Line Breaking Rules | |
2ca993e8 A |
7 | # Implement default line breaking as defined by |
8 | # Unicode Standard Annex #14 Revision 35 for Unicode 8.0 | |
729e4ab9 | 9 | # http://www.unicode.org/reports/tr14/ |
08b89b0a | 10 | # tailored as noted in 2nd paragraph below.. |
729e4ab9 | 11 | # |
2ca993e8 A |
12 | # Includes the Emoji breaking proposals from Unicode L2/16-011R3. |
13 | # http://www.unicode.org/L2/L2016/16011r3-break-prop-emoji.pdf | |
14 | # | |
729e4ab9 A |
15 | # TODO: Rule LB 8 remains as it was in Unicode 5.2 |
16 | # This is only because of a limitation of ICU break engine implementation, | |
17 | # not because the older behavior is desirable. | |
08b89b0a A |
18 | # |
19 | # This tailors the line break behavior for Finnish, while otherwise behaving | |
20 | # per UAX 14 which corresponds to CSS line-break=strict (BCP47 -u-lb-strict). | |
21 | # It sets characters of class CJ to behave like NS. | |
729e4ab9 A |
22 | |
23 | # | |
24 | # Character Classes defined by TR 14. | |
25 | # | |
26 | ||
27 | !!chain; | |
729e4ab9 A |
28 | |
29 | !!lookAheadHardBreak; | |
30 | # | |
31 | # !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere | |
32 | # and only used for the line break rules. | |
33 | # | |
34 | # It is used in the implementation of rule LB 10 | |
35 | # which says to treat any combining mark that is not attached to a base | |
36 | # character as if it were of class AL (alphabetic). | |
37 | # | |
38 | # The problem occurs in the reverse rules. | |
39 | # | |
40 | # Consider a sequence like, with correct breaks as shown | |
41 | # LF ID CM AL AL | |
42 | # ^ ^ ^ | |
43 | # Then consider the sequence without the initial ID (ideographic) | |
44 | # LF CM AL AL | |
45 | # ^ ^ | |
46 | # Our CM, which in the first example was attached to the ideograph, | |
47 | # is now unattached, becomes an alpha, and joins in with the other | |
48 | # alphas. | |
49 | # | |
50 | # When iterating forwards, these sequences do not present any problems | |
51 | # When iterating backwards, we need to look ahead when encountering | |
52 | # a CM to see whether it attaches to something further on or not. | |
53 | # (Look-ahead in a reverse rule is looking towards the start) | |
54 | # | |
55 | # If the CM is unattached, we need to force a break. | |
56 | # | |
57 | # !!lookAheadHardBreak forces the run time state machine to | |
58 | # stop immediately when a look ahead rule ( '/' operator) matches, | |
59 | # and set the match position to that of the look-ahead operator, | |
60 | # no matter what other rules may be in play at the time. | |
61 | # | |
62 | # See rule LB 19 for an example. | |
63 | # | |
64 | ||
2ca993e8 A |
65 | # Temporary definitions of Emoji Base and Emoji Modifiers, until properties are available. |
66 | ||
a62d09fc | 67 | $EB = [\u261D\u26F9\u270A-\u270D\U0001F385\U0001F3C2-\U0001F3C4\U0001F3C7\U0001F3CA-\U0001F3CC\U0001F442-\U0001F443\U0001F446-\U0001F450\U0001F466-\U0001F478\U0001F47C\U0001F481-\U0001F483\U0001F485-\U0001F487\U0001F4AA\U0001F574-\U0001F575\U0001F57A\U0001F590\U0001F595-\U0001F596\U0001F645-\U0001F647\U0001F64B-\U0001F64F\U0001F6A3\U0001F6B4-\U0001F6B6\U0001F6C0\U0001F6CC\U0001F918-\U0001F91E\U0001F926\U0001F930\U0001F933-\U0001F939\U0001F93C-\U0001F93E]; |
2ca993e8 A |
68 | $EM = [\U0001F3FB-\U0001F3FF]; |
69 | ||
70 | $AI = [[:LineBreak = Ambiguous:] - [\u2640\u2642]]; | |
a62d09fc | 71 | $AL = [[:LineBreak = Alphabetic:] - [$EM\u2695\u2696\u2764]]; |
2ca993e8 | 72 | $BA = [:LineBreak = Break_After:]; |
729e4ab9 A |
73 | $HH = [\u2010]; |
74 | $BB = [:LineBreak = Break_Before:]; | |
75 | $BK = [:LineBreak = Mandatory_Break:]; | |
76 | $B2 = [:LineBreak = Break_Both:]; | |
77 | $CB = [:LineBreak = Contingent_Break:]; | |
4388f060 | 78 | $CJ = [:LineBreak = Conditional_Japanese_Starter:]; |
729e4ab9 | 79 | $CL = [:LineBreak = Close_Punctuation:]; |
2ca993e8 | 80 | $CM = [[:LineBreak = Combining_Mark:] \u200d]; |
729e4ab9 A |
81 | $CP = [:LineBreak = Close_Parenthesis:]; |
82 | $CR = [:LineBreak = Carriage_Return:]; | |
83 | $EX = [:LineBreak = Exclamation:]; | |
84 | $GL = [:LineBreak = Glue:]; | |
4388f060 | 85 | $HL = [:LineBreak = Hebrew_Letter:]; |
729e4ab9 A |
86 | $HY = [:LineBreak = Hyphen:]; |
87 | $H2 = [:LineBreak = H2:]; | |
88 | $H3 = [:LineBreak = H3:]; | |
a62d09fc | 89 | $ID = [[:LineBreak = Ideographic:][\u2640\u2642\u2695\u2696\u2764] - $EB]; |
729e4ab9 A |
90 | $IN = [:LineBreak = Inseperable:]; |
91 | $IS = [:LineBreak = Infix_Numeric:]; | |
92 | $JL = [:LineBreak = JL:]; | |
93 | $JV = [:LineBreak = JV:]; | |
94 | $JT = [:LineBreak = JT:]; | |
95 | $LF = [:LineBreak = Line_Feed:]; | |
96 | $NL = [:LineBreak = Next_Line:]; | |
4388f060 | 97 | $NS = [[:LineBreak = Nonstarter:] $CJ]; |
729e4ab9 A |
98 | $NU = [:LineBreak = Numeric:]; |
99 | $OP = [:LineBreak = Open_Punctuation:]; | |
100 | $PO = [:LineBreak = Postfix_Numeric:]; | |
101 | $PR = [:LineBreak = Prefix_Numeric:]; | |
102 | $QU = [:LineBreak = Quotation:]; | |
51004dcb | 103 | $RI = [:LineBreak = Regional_Indicator:]; |
729e4ab9 A |
104 | $SA = [:LineBreak = Complex_Context:]; |
105 | $SG = [:LineBreak = Surrogate:]; | |
106 | $SP = [:LineBreak = Space:]; | |
107 | $SY = [:LineBreak = Break_Symbols:]; | |
108 | $WJ = [:LineBreak = Word_Joiner:]; | |
109 | $XX = [:LineBreak = Unknown:]; | |
110 | $ZW = [:LineBreak = ZWSpace:]; | |
2ca993e8 | 111 | $ZWJ = [\u200d]; |
08b89b0a | 112 | |
729e4ab9 A |
113 | # Dictionary character set, for triggering language-based break engines. Currently |
114 | # limited to LineBreak=Complex_Context. Note that this set only works in Unicode | |
115 | # 5.0 or later as the definition of Complex_Context was corrected to include all | |
116 | # characters requiring dictionary break. | |
117 | ||
118 | $dictionary = [:LineBreak = Complex_Context:]; | |
119 | ||
120 | # | |
121 | # Rule LB1. By default, treat AI (characters with ambiguous east Asian width), | |
122 | # SA (South East Asian: Thai, Lao, Khmer) | |
123 | # SG (Unpaired Surrogates) | |
124 | # XX (Unknown, unassigned) | |
125 | # as $AL (Alphabetic) | |
126 | # | |
127 | $ALPlus = [$AL $AI $SA $SG $XX]; | |
128 | ||
129 | # | |
130 | # Combining Marks. X $CM* behaves as if it were X. Rule LB6. | |
131 | # | |
132 | $ALcm = $ALPlus $CM*; | |
133 | $BAcm = $BA $CM*; | |
134 | $HHcm = $HH $CM*; | |
135 | $BBcm = $BB $CM*; | |
136 | $B2cm = $B2 $CM*; | |
137 | $CLcm = $CL $CM*; | |
138 | $CPcm = $CP $CM*; | |
139 | $EXcm = $EX $CM*; | |
140 | $GLcm = $GL $CM*; | |
4388f060 | 141 | $HLcm = $HL $CM*; |
729e4ab9 A |
142 | $HYcm = $HY $CM*; |
143 | $H2cm = $H2 $CM*; | |
144 | $H3cm = $H3 $CM*; | |
729e4ab9 A |
145 | $INcm = $IN $CM*; |
146 | $IScm = $IS $CM*; | |
147 | $JLcm = $JL $CM*; | |
148 | $JVcm = $JV $CM*; | |
149 | $JTcm = $JT $CM*; | |
150 | $NScm = $NS $CM*; | |
151 | $NUcm = $NU $CM*; | |
152 | $OPcm = $OP $CM*; | |
153 | $POcm = $PO $CM*; | |
154 | $PRcm = $PR $CM*; | |
155 | $QUcm = $QU $CM*; | |
51004dcb | 156 | $RIcm = $RI $CM*; |
729e4ab9 A |
157 | $SYcm = $SY $CM*; |
158 | $WJcm = $WJ $CM*; | |
159 | ||
160 | ## ------------------------------------------------- | |
161 | ||
162 | !!forward; | |
163 | ||
164 | # | |
165 | # Each class of character can stand by itself as an unbroken token, with trailing combining stuff | |
166 | # | |
167 | $ALPlus $CM+; | |
168 | $BA $CM+; | |
169 | $HH $CM+; | |
170 | $BB $CM+; | |
171 | $B2 $CM+; | |
172 | $CL $CM+; | |
173 | $CP $CM+; | |
2ca993e8 A |
174 | $EB $CM+; |
175 | $EM $CM+; | |
729e4ab9 A |
176 | $EX $CM+; |
177 | $GL $CM+; | |
4388f060 | 178 | $HL $CM+; |
729e4ab9 A |
179 | $HY $CM+; |
180 | $H2 $CM+; | |
181 | $H3 $CM+; | |
182 | $ID $CM+; | |
183 | $IN $CM+; | |
184 | $IS $CM+; | |
185 | $JL $CM+; | |
186 | $JV $CM+; | |
187 | $JT $CM+; | |
188 | $NS $CM+; | |
189 | $NU $CM+; | |
190 | $OP $CM+; | |
191 | $PO $CM+; | |
192 | $PR $CM+; | |
193 | $QU $CM+; | |
51004dcb | 194 | $RI $CM+; |
729e4ab9 A |
195 | $SY $CM+; |
196 | $WJ $CM+; | |
197 | ||
198 | # | |
199 | # CAN_CM is the set of characters that may combine with CM combining chars. | |
200 | # Note that Linebreak UAX 14's concept of a combining char and the rules | |
201 | # for what they can combine with are _very_ different from the rest of Unicode. | |
202 | # | |
203 | # Note that $CM itself is left out of this set. If CM is needed as a base | |
204 | # it must be listed separately in the rule. | |
205 | # | |
206 | $CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs | |
207 | $CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs | |
208 | ||
209 | # | |
210 | # AL_FOLLOW set of chars that can unconditionally follow an AL | |
211 | # Needed in rules where stand-alone $CM s are treated as AL. | |
212 | # Chaining is disabled with CM because it causes other failures, | |
213 | # so for this one case we need to manually list out longer sequences. | |
214 | # | |
215 | $AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP]; | |
4388f060 | 216 | $AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus]; |
729e4ab9 A |
217 | $AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM]; |
218 | ||
219 | ||
220 | # | |
221 | # Rule LB 4, 5 Mandatory (Hard) breaks. | |
222 | # | |
223 | $LB4Breaks = [$BK $CR $LF $NL]; | |
2ca993e8 | 224 | $LB4NonBreaks = [^$BK $CR $LF $NL $CM]; |
729e4ab9 A |
225 | $CR $LF {100}; |
226 | ||
227 | # | |
228 | # LB 6 Do not break before hard line breaks. | |
229 | # | |
230 | $LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks. | |
231 | $CAN_CM $CM* $LB4Breaks {100}; | |
2ca993e8 | 232 | ^$CM+ $LB4Breaks {100}; |
729e4ab9 A |
233 | |
234 | # LB 7 x SP | |
235 | # x ZW | |
236 | $LB4NonBreaks [$SP $ZW]; | |
237 | $CAN_CM $CM* [$SP $ZW]; | |
2ca993e8 | 238 | ^$CM+ [$SP $ZW]; |
729e4ab9 A |
239 | |
240 | # | |
241 | # LB 8 Break after zero width space | |
242 | # TODO: ZW SP* <break> | |
243 | # An engine change is required to write the reverse rule for this. | |
244 | # For now, leave the Unicode 5.2 rule, ZW <break> | |
245 | # | |
246 | $LB8Breaks = [$LB4Breaks $ZW]; | |
247 | $LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]]; | |
248 | ||
2ca993e8 A |
249 | # LB 8a ZWJ x ID Emoji proposal. |
250 | # | |
251 | $ZWJ ($ID | $EB | $EM); | |
729e4ab9 | 252 | |
2ca993e8 A |
253 | # LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL |
254 | # $CM not covered by the above needs to behave like $AL | |
729e4ab9 A |
255 | # See definition of $CAN_CM. |
256 | ||
257 | $CAN_CM $CM+; # Stick together any combining sequences that don't match other rules. | |
2ca993e8 | 258 | ^$CM+; |
729e4ab9 A |
259 | |
260 | # | |
261 | # LB 11 Do not break before or after WORD JOINER & related characters. | |
262 | # | |
263 | $CAN_CM $CM* $WJcm; | |
264 | $LB8NonBreaks $WJcm; | |
2ca993e8 | 265 | ^$CM+ $WJcm; |
729e4ab9 A |
266 | |
267 | $WJcm $CANT_CM; | |
268 | $WJcm $CAN_CM $CM*; | |
269 | ||
270 | # | |
271 | # LB 12 Do not break after NBSP and related characters. | |
272 | # GL x | |
273 | # | |
274 | $GLcm $CAN_CM $CM*; | |
275 | $GLcm $CANT_CM; | |
2ca993e8 | 276 | |
729e4ab9 A |
277 | # |
278 | # LB 12a Do not break before NBSP and related characters ... | |
279 | # [^SP BA HY] x GL | |
280 | # | |
281 | [[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm; | |
2ca993e8 | 282 | ^$CM+ $GLcm; |
729e4ab9 A |
283 | |
284 | ||
285 | ||
286 | # | |
287 | # LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces. | |
288 | # | |
289 | $LB8NonBreaks $CL; | |
290 | $CAN_CM $CM* $CL; | |
2ca993e8 | 291 | ^$CM+ $CL; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
292 | |
293 | $LB8NonBreaks $CP; | |
294 | $CAN_CM $CM* $CP; | |
2ca993e8 | 295 | ^$CM+ $CP; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
296 | |
297 | $LB8NonBreaks $EX; | |
298 | $CAN_CM $CM* $EX; | |
2ca993e8 | 299 | ^$CM+ $EX; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
300 | |
301 | $LB8NonBreaks $IS; | |
302 | $CAN_CM $CM* $IS; | |
2ca993e8 | 303 | ^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
304 | |
305 | $LB8NonBreaks $SY; | |
306 | $CAN_CM $CM* $SY; | |
2ca993e8 | 307 | ^$CM+ $SY; # by rule 10, stand-alone CM behaves as AL |
729e4ab9 A |
308 | |
309 | ||
310 | # | |
311 | # LB 14 Do not break after OP, even after spaces | |
312 | # | |
313 | $OPcm $SP* $CAN_CM $CM*; | |
314 | $OPcm $SP* $CANT_CM; | |
315 | ||
316 | $OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL | |
317 | ||
318 | # LB 15 | |
319 | $QUcm $SP* $OPcm; | |
320 | ||
321 | # LB 16 | |
322 | ($CLcm | $CPcm) $SP* $NScm; | |
323 | ||
324 | # LB 17 | |
325 | $B2cm $SP* $B2cm; | |
326 | ||
327 | # | |
328 | # LB 18 Break after spaces. | |
329 | # | |
330 | $LB18NonBreaks = [$LB8NonBreaks - [$SP]]; | |
331 | $LB18Breaks = [$LB8Breaks $SP]; | |
332 | ||
333 | ||
334 | # LB 19 | |
335 | # x QU | |
336 | $LB18NonBreaks $CM* $QUcm; | |
2ca993e8 | 337 | ^$CM+ $QUcm; |
729e4ab9 A |
338 | |
339 | # QU x | |
340 | $QUcm .?; | |
729e4ab9 A |
341 | |
342 | ||
343 | # LB 20 | |
344 | # <break> $CB | |
345 | # $CB <break> | |
346 | ||
347 | $LB20NonBreaks = [$LB18NonBreaks - $CB]; | |
348 | ||
349 | # LB 20.09 added rule for Finnish tailoring | |
350 | # LB 21 x (BA | HY | NS) | |
351 | # BB x | |
352 | # | |
353 | $LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm) / $AL; | |
354 | $LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm); | |
355 | ($HY | $HH) $AL; | |
2ca993e8 | 356 | ^$CM+ ($BAcm | $HYcm | $HHcm | $NScm); |
729e4ab9 A |
357 | |
358 | $BBcm [^$CB]; # $BB x | |
359 | $BBcm $LB20NonBreaks $CM*; | |
360 | ||
4388f060 A |
361 | # LB 21a Don't break after Hebrew + Hyphen |
362 | # HL (HY | BA) x | |
363 | # | |
364 | $HLcm ($HYcm | $BAcm | $HHcm) [^$CB]?; | |
365 | ||
57a6839d A |
366 | # LB 21b (forward) Don't break between SY and HL |
367 | # (break between HL and SY already disallowed by LB 13 above) | |
368 | $SYcm $HLcm; | |
369 | ||
729e4ab9 | 370 | # LB 22 |
4388f060 | 371 | ($ALcm | $HLcm) $INcm; |
2ca993e8 A |
372 | ^$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL |
373 | $EXcm $INcm; | |
374 | ($ID | $EB | $EM) $CM* $INcm; | |
729e4ab9 A |
375 | $INcm $INcm; |
376 | $NUcm $INcm; | |
377 | ||
378 | ||
379 | # $LB 23 | |
2ca993e8 | 380 | ($ID | $EB | $EM) $CM* $POcm; |
729e4ab9 | 381 | $ALcm $NUcm; # includes $LB19 |
4388f060 | 382 | $HLcm $NUcm; |
2ca993e8 | 383 | ^$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL |
729e4ab9 | 384 | $NUcm $ALcm; |
4388f060 | 385 | $NUcm $HLcm; |
729e4ab9 A |
386 | |
387 | # | |
388 | # LB 24 | |
389 | # | |
2ca993e8 | 390 | $PRcm ($ID | $EB | $EM); |
4388f060 A |
391 | $PRcm ($ALcm | $HLcm); |
392 | $POcm ($ALcm | $HLcm); | |
2ca993e8 A |
393 | ($ALcm | $HLcm) ($PRcm | $POcm); |
394 | ^$CM+ ($PRcm | $POcm); # Rule 10, any otherwise unattached CM behaves as AL | |
729e4ab9 A |
395 | |
396 | # | |
397 | # LB 25 Numbers. | |
398 | # | |
399 | ($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?; | |
400 | ||
401 | # LB 26 Do not break a Korean syllable | |
402 | # | |
403 | $JLcm ($JLcm | $JVcm | $H2cm | $H3cm); | |
404 | ($JVcm | $H2cm) ($JVcm | $JTcm); | |
405 | ($JTcm | $H3cm) $JTcm; | |
406 | ||
407 | # LB 27 Treat korean Syllable Block the same as ID (don't break it) | |
408 | ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm; | |
409 | ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm; | |
410 | $PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm); | |
411 | ||
412 | ||
413 | # LB 28 Do not break between alphabetics | |
414 | # | |
4388f060 | 415 | ($ALcm | $HLcm) ($ALcm | $HLcm); |
2ca993e8 | 416 | ^$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL |
729e4ab9 A |
417 | |
418 | # LB 29 | |
4388f060 | 419 | $IScm ($ALcm | $HLcm); |
729e4ab9 A |
420 | |
421 | # LB 30 | |
4388f060 | 422 | ($ALcm | $HLcm | $NUcm) $OPcm; |
2ca993e8 | 423 | ^$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL. |
4388f060 | 424 | $CPcm ($ALcm | $HLcm | $NUcm); |
729e4ab9 | 425 | |
2ca993e8 A |
426 | # LB 30a Do not break between regional indicators. Break after pairs of them. |
427 | # Tricky interaction with LB8a: ZWJ x ID | |
428 | $RIcm $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM] {eof}]; | |
429 | $RIcm $RI $CM* $ZWJ / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM $ID $EB $EM] {eof}]; | |
430 | $RIcm $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM] {eof}]; | |
431 | ||
432 | $RIcm $RIcm [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS {eof}]; | |
433 | $RIcm $RIcm $ZWJ ($ID | $EB | $EM); | |
729e4ab9 | 434 | |
2ca993e8 A |
435 | # LB 30b Do not break between an Emoji Base and an Emoji Modifier |
436 | $EB $CM* $EM; | |
08b89b0a | 437 | |
729e4ab9 A |
438 | # |
439 | # Reverse Rules. | |
440 | # | |
441 | ## ------------------------------------------------- | |
442 | ||
443 | !!reverse; | |
444 | ||
2ca993e8 A |
445 | ^$CM+ $ALPlus; |
446 | ^$CM+ $BA; | |
447 | ^$CM+ $HH; | |
448 | ^$CM+ $BB; | |
449 | ^$CM+ $B2; | |
450 | ^$CM+ $CL; | |
451 | ^$CM+ $CP; | |
452 | ^$CM+ $EB; | |
453 | ^$CM+ $EM; | |
454 | ^$CM+ $EX; | |
455 | ^$CM+ $GL; | |
456 | ^$CM+ $HL; | |
457 | ^$CM+ $HY; | |
458 | ^$CM+ $H2; | |
459 | ^$CM+ $H3; | |
460 | ^$CM+ $ID; | |
461 | ^$CM+ $IN; | |
462 | ^$CM+ $IS; | |
463 | ^$CM+ $JL; | |
464 | ^$CM+ $JV; | |
465 | ^$CM+ $JT; | |
466 | ^$CM+ $NS; | |
467 | ^$CM+ $NU; | |
468 | ^$CM+ $OP; | |
469 | ^$CM+ $PO; | |
470 | ^$CM+ $PR; | |
471 | ^$CM+ $QU; | |
472 | ^$CM+ $RI; | |
473 | ^$CM+ $SY; | |
474 | ^$CM+ $WJ; | |
475 | ^$CM+; | |
729e4ab9 A |
476 | |
477 | ||
478 | # | |
479 | # Sequences of the form (shown forwards) | |
480 | # [CANT_CM] <break> [CM] [whatever] | |
481 | # The CM needs to behave as an AL | |
482 | # | |
483 | $AL_FOLLOW $CM+ / ( | |
484 | [$BK $CR $LF $NL $ZW {eof}] | | |
485 | $SP+ $CM+ $SP | | |
486 | $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break. | |
2ca993e8 | 487 | # LB14 says OP SP* x . |
729e4ab9 A |
488 | # becomes OP SP* x AL |
489 | # becomes OP SP* x CM+ AL_FOLLOW | |
490 | # | |
491 | # Further note: the $AL in [$AL {eof}] is only to work around | |
492 | # a rule compiler bug which complains about | |
493 | # empty sets otherwise. | |
2ca993e8 | 494 | |
729e4ab9 A |
495 | # |
496 | # Sequences of the form (shown forwards) | |
497 | # [CANT_CM] <break> [CM] <break> [PR] | |
498 | # The CM needs to behave as an AL | |
499 | # This rule is concerned about getting the second of the two <breaks> in place. | |
500 | # | |
501 | ||
2ca993e8 A |
502 | # Apple early addition, remove this, superseded by LB24 |
503 | # [$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}]; | |
729e4ab9 A |
504 | |
505 | ||
506 | ||
2ca993e8 | 507 | # LB 4, 5, 6 |
729e4ab9 A |
508 | |
509 | $LB4Breaks [$LB4NonBreaks-$CM]; | |
510 | $LB4Breaks $CM+ $CAN_CM; | |
511 | $LF $CR; | |
512 | ||
513 | ||
514 | # LB 7 x SP | |
515 | # x ZW | |
516 | [$SP $ZW] [$LB4NonBreaks-$CM]; | |
517 | [$SP $ZW] $CM+ $CAN_CM; | |
518 | ||
519 | # LB 8 ZW SP* <break> | |
520 | # TODO: to implement this, we need more than one look-ahead hard break in play at a time. | |
521 | # Requires an engine enhancement. | |
522 | # / $SP* $ZW | |
523 | ||
2ca993e8 A |
524 | # LB 8a ZWJ x ID Unicode Emoji proposal L2/16-011R3 |
525 | # The ZWJ will look like a CM to whatever precedes it. | |
526 | # | |
527 | ($ID | $EB | $EM) $ZWJ $CM* $CAN_CM?; | |
528 | ||
08b89b0a | 529 | |
729e4ab9 A |
530 | # LB 9,10 Combining marks. |
531 | # X $CM needs to behave like X, where X is not $SP or controls. | |
532 | # $CM not covered by the above needs to behave like $AL | |
533 | # Stick together any combining sequences that don't match other rules. | |
2ca993e8 | 534 | ^$CM+ $CAN_CM; |
729e4ab9 A |
535 | |
536 | ||
537 | # LB 11 | |
2ca993e8 A |
538 | # |
539 | $WJ $CM* $CAN_CM; | |
540 | $WJ [$LB8NonBreaks-$CM]; | |
729e4ab9 A |
541 | |
542 | $CANT_CM $CM* $WJ; | |
2ca993e8 | 543 | $CAN_CM $CM* $WJ; |
729e4ab9 A |
544 | |
545 | # LB 12a | |
546 | # [^SP BA HY] x GL | |
547 | # | |
2ca993e8 | 548 | $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]]; |
729e4ab9 A |
549 | |
550 | # LB 12 | |
551 | # GL x | |
552 | # | |
553 | $CANT_CM $CM* $GL; | |
2ca993e8 | 554 | $CAN_CM $CM* $GL; |
729e4ab9 A |
555 | |
556 | ||
557 | # LB 13 | |
558 | $CL $CM+ $CAN_CM; | |
559 | $CP $CM+ $CAN_CM; | |
560 | $EX $CM+ $CAN_CM; | |
561 | $IS $CM+ $CAN_CM; | |
562 | $SY $CM+ $CAN_CM; | |
563 | ||
564 | $CL [$LB8NonBreaks-$CM]; | |
565 | $CP [$LB8NonBreaks-$CM]; | |
566 | $EX [$LB8NonBreaks-$CM]; | |
567 | $IS [$LB8NonBreaks-$CM]; | |
568 | $SY [$LB8NonBreaks-$CM]; | |
569 | ||
570 | # Rule 13 & 14 taken together for an edge case. | |
571 | # Match this, shown forward | |
572 | # OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY) | |
573 | # This really wants to chain at the $CM+ (which is acting as an $AL) | |
574 | # except for $CM chaining being disabled. | |
2ca993e8 | 575 | [$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP; |
729e4ab9 A |
576 | |
577 | # LB 14 OP SP* x | |
578 | # | |
2ca993e8 | 579 | $CAN_CM $SP* $CM* $OP; |
729e4ab9 A |
580 | $CANT_CM $SP* $CM* $OP; |
581 | $AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP | |
729e4ab9 | 582 | |
2ca993e8 A |
583 | $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP; |
584 | $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP; | |
729e4ab9 A |
585 | |
586 | ||
587 | # LB 15 | |
2ca993e8 | 588 | $OP $SP* $CM* $QU; |
729e4ab9 A |
589 | |
590 | # LB 16 | |
2ca993e8 | 591 | $NS $SP* $CM* ($CL | $CP); |
729e4ab9 A |
592 | |
593 | # LB 17 | |
2ca993e8 | 594 | $B2 $SP* $CM* $B2; |
729e4ab9 A |
595 | |
596 | # LB 18 break after spaces | |
597 | # Nothing explicit needed here. | |
598 | ||
599 | ||
600 | # | |
601 | # LB 19 | |
602 | # | |
2ca993e8 A |
603 | $QU $CM* $CAN_CM; # . x QU |
604 | $QU $LB18NonBreaks; | |
729e4ab9 A |
605 | |
606 | ||
2ca993e8 | 607 | $CAN_CM $CM* $QU; # QU x . |
729e4ab9 | 608 | $CANT_CM $CM* $QU; |
2ca993e8 | 609 | |
729e4ab9 A |
610 | # |
611 | # LB 20 Break before and after CB. | |
612 | # nothing needed here. | |
613 | # | |
614 | ||
615 | # LB 20.09 added rule for Finnish tailoring | |
616 | $AL ($HY | $HH) / $SP; | |
617 | ||
618 | # LB 21 | |
2ca993e8 | 619 | ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS) |
729e4ab9 | 620 | |
2ca993e8 A |
621 | [$LB20NonBreaks-$CM] $CM* $BB; # BB x . |
622 | [^$CB] $CM* $BB; # | |
729e4ab9 | 623 | |
4388f060 A |
624 | # LB21a |
625 | [^$CB] $CM* ($HY | $BA | $HH) $CM* $HL; | |
729e4ab9 | 626 | |
57a6839d | 627 | # LB21b (reverse) |
2ca993e8 | 628 | $HL $CM* $SY; |
729e4ab9 A |
629 | |
630 | # LB 22 | |
2ca993e8 A |
631 | $IN $CM* ($ALPlus | $HL); |
632 | $IN $CM* $EX; | |
633 | $IN $CM* ($ID | $EB | $EM); | |
634 | $IN $CM* $IN; | |
635 | $IN $CM* $NU; | |
729e4ab9 A |
636 | |
637 | # LB 23 | |
2ca993e8 A |
638 | $PO $CM* ($ID | $EB | $EM); |
639 | $NU $CM* ($ALPlus | $HL); | |
640 | ($ALPlus | $HL) $CM* $NU; | |
729e4ab9 A |
641 | |
642 | # LB 24 | |
2ca993e8 A |
643 | ($ID | $EB | $EM) $CM* $PR; |
644 | ($ALPlus | $HL) $CM* $PR; | |
645 | ($ALPlus | $HL) $CM* $PO; | |
646 | $CM* ($PR | $PO) $CM* ($ALPlus | $HL); | |
647 | $CM* ($PR | $PO) $CM+ / [$BK $CR $LF $NL $ZW $SP {eof}]; | |
729e4ab9 A |
648 | |
649 | # LB 25 | |
650 | ($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?; | |
651 | ||
652 | # LB 26 | |
2ca993e8 A |
653 | ($H3 | $H2 | $JV | $JL) $CM* $JL; |
654 | ($JT | $JV) $CM* ($H2 | $JV); | |
655 | $JT $CM* ($H3 | $JT); | |
729e4ab9 A |
656 | |
657 | # LB 27 | |
2ca993e8 A |
658 | $IN $CM* ($H3 | $H2 | $JT | $JV | $JL); |
659 | $PO $CM* ($H3 | $H2 | $JT | $JV | $JL); | |
660 | ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR; | |
729e4ab9 A |
661 | |
662 | # LB 28 | |
2ca993e8 | 663 | ($ALPlus | $HL) $CM* ($ALPlus | $HL); |
729e4ab9 A |
664 | |
665 | ||
666 | # LB 29 | |
2ca993e8 | 667 | ($ALPlus | $HL) $CM* $IS; |
729e4ab9 A |
668 | |
669 | # LB 30 | |
2ca993e8 A |
670 | $OP $CM* ($ALPlus | $HL | $NU); |
671 | ($ALPlus | $HL | $NU) $CM* $CP; | |
729e4ab9 | 672 | |
51004dcb | 673 | # LB 30a |
2ca993e8 A |
674 | # Pairs of Regional Indicators. |
675 | # The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs, | |
676 | # the second with an even number. Stripping away the cruft they look like | |
677 | # [^RI] RI / (RI RI)+ ^RI; | |
678 | # [^RI] RI RI / (RI RI)+ ^RI; | |
679 | # | |
680 | [{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]]; | |
681 | [{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]]; | |
682 | ||
683 | # In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs. | |
684 | $RI $CM* $RI; | |
685 | ||
686 | # WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI". | |
687 | $RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL)); | |
688 | ||
689 | ||
690 | # LB 30b Do not break between an Emoji Base and an Emoji Modifier | |
691 | $EM $CM* $EB; | |
729e4ab9 | 692 | |
08b89b0a | 693 | |
729e4ab9 A |
694 | ## ------------------------------------------------- |
695 | ||
696 | !!safe_reverse; | |
697 | ||
698 | # LB 9 | |
2ca993e8 A |
699 | ^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP]; |
700 | ^$CM+ $SP / .; | |
729e4ab9 A |
701 | |
702 | # LB 14 | |
703 | $SP+ $CM* $OP; | |
704 | ||
705 | # LB 15 | |
706 | $SP+ $CM* $QU; | |
707 | ||
708 | # LB 16 | |
709 | $SP+ $CM* ($CL | $CP); | |
710 | ||
711 | # LB 17 | |
712 | $SP+ $CM* $B2; | |
713 | ||
4388f060 A |
714 | # LB 21 |
715 | $CM* ($HY | $BA | $HH) $CM* $HL; | |
716 | ||
729e4ab9 A |
717 | # LB 25 |
718 | ($CM* ($IS | $SY))+ $CM* $NU; | |
719 | ($CL | $CP) $CM* ($NU | $IS | $SY); | |
720 | ||
2ca993e8 A |
721 | # LB 30 |
722 | ($CM* $RI)+; | |
723 | ||
729e4ab9 A |
724 | # For dictionary-based break |
725 | $dictionary $dictionary; | |
726 | ||
727 | ## ------------------------------------------------- | |
728 | ||
729 | !!safe_forward; | |
730 | ||
731 | # Skip forward over all character classes that are involved in | |
732 | # rules containing patterns with possibly more than one char | |
733 | # of context. | |
734 | # | |
735 | # It might be slightly more efficient to have specific rules | |
736 | # instead of one generic one, but only if we could | |
737 | # turn off rule chaining. We don't want to move more | |
738 | # than necessary. | |
739 | # | |
2ca993e8 | 740 | ^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary]; |
729e4ab9 A |
741 | $dictionary $dictionary; |
742 |