-# Copyright (c) 2001-2003 International Business Machines
+# Copyright (c) 2001-2008 International Business Machines
# Corporation and others. All Rights Reserved.
#
# RBBI Test Data
# <sent> any following data is for sentence break testing
# <line> any following data is for line break testing
# <char> any following data is for char break testing
+# <locale local_name> Switch to the named locale at the next occurence of <word>, <sent>, etc.
# <data> ... </data> test data. May span multiple lines.
# <> Break position, status == 0
# • Break position, status == 0 (Bullet, \u2022)
#
+# Temp debugging tests
+<line>
+# to test for bug #4097920
+<data>•dog,cat,mouse •(one)•(two)\n<100></data>
########################################################################################
#
# Hindi combining chars. (An old test)
-<data>•भ••ा•\u0930•\u0924• •\u0938\u0941\u0902•\u0926•\u0930•
-•\u0939•\u094c•\u0964•</data>
-<data>•\u0916\u0947•\u0938\u0941\u0902•\u0926•\u0930•\u0939•\u094c•\u0964•</data>
+# TODO: Update these tests for Unicode 5.1 Extended Grapheme clusters
+#<data>•भ••ा•\u0930•\u0924• •\u0938\u0941\u0902•\u0926•\u0930•
+#•\u0939•\u094c•\u0964•</data>
+#<data>•\u0916\u0947•\u0938\u0941\u0902•\u0926•\u0930•\u0939•\u094c•\u0964•</data>
-# Bug 1587. Tamil. \u0baa\u0bc1 should be two separate characters, even though
-# Hyangmi would perfer that it be one.
-<data>•\u0baa•\u0bc1•\u0baa•\u0bc1•</data>
+# Bug 1587. Tamil. \u0baa\u0bc1 is an Extended Grpaheme Cluster
+<data>•\u0baa\u0bc1•\u0baa\u0bc1•</data>
# Regression test for bug 1889
<data>•\u0f40\u0f7d•\u0000•\u0f7e•</data>
# (Requires special casing in implementation, which is why it gets a test.)
<data>•\uffff•\uffff• •a•</data>
+# Treat Japanese Half Width voicing marks as combining
+<data>•A\uff9e•B\uff9f\uff9e\uff9f•C•</data>
+
+########################################################################################
+#
+#
+# E x t e n d e d G r a p h e m e C l u s t e r T e s t s
+#
+#
+##########################################################################################
+#<xgc>
+
+# Plain Vanilla grapheme clusters
+#<data>•a•b•c•</data>
+#<data>•a\u0301\u0302• •b\u0303\u0304•</data>
+
+# Assorted Hindi combining marks
+#<data>•\u0904\u0903• •\u0937\u093E• •\u0904\u093F• •\u0937\u0940• •\u0937\u0949• •\u0937\u094A• •\u0937\u094B• •\u0937\u094C•</data>
+
+# Thai Clusters
+# $Prepend $Extend* $PrependBase $Extend*;
+#
+#<data>•\u0e40\u0e01•\u0e44\u0301\u0e23\u0302\u0303•\u0e40•\u0e40\u0e02•\u0e02• •</data>
+
########################################################################################
#
<data>•abc<200>\u4e01<400>\u4e02<400>\u3005<200>\u4e03<400>\u4e03<400>abc<200> •</data>
-<data>•\u06c9\uc799<200>\ufffa•</data>
+<data>•\u06c9\uc799\ufffa<200></data>
#
# Try some words from other scripts.
<data>•\u0917\u092a\u00ad\u0936\u092a<200>!•\u092f\u0939<200> •\u0939\u093f\u0928\u094d\u200d\u0926\u0940<200> •\u0939\u0948<200> •\u0905\u093e\u092a<200> •\u0938\u093f\u0916\u094b\u0917\u0947<200>?•\n•:•\u092a\u094d\u0930\u093e\u092f\u0903<200>
•\u0935\u0930\u094d\u0937\u093e<200>\r\n•\u092a\u094d\u0930\u0915\u093e\u0936<200>,•\u0924\u0941\u092e\u093e\u0930\u094b<200> •\u092e\u093f\u0924\u094d\u0930<200> •\u0915\u093e<200> •\u092a\u0924\u094d\u0930<200> •\u092a\u095d\u094b<200> •\u0938\u094d\u0924\u094d\u0930\u093f<200>.• •\u0968\u0966.\u0969\u096f<100> •\u0967\u0966\u0966.\u0966\u0966<100>\u20a8•\u0967,\u0967\u0966\u0966.\u0966\u0966<100> •\u0905\u092e\u091c<200>\n•\u0938\u094d\u200d\u0935\u0924\u0902\u0924\u094d\u0930<200>\r•</data>
+#
+# Failures from monkey tests
+#
+<data>•\u8527<400>\u02ba<200>\u0027\u0d42•\u00b7•\u09ea<100></data>
+
+#
+# Jitterbug 5276 - treat Japanese half width voicing marks as Grapheme Extend
+#
+<data>•A\uff9e\uff9fBC<200> •1\uff9e\uff9f23<100></data>
+
########################################################################################
#
#
<sent>
-<data>•This\n•</data>
+<data>•This\n<100></data>
<data>•Hello! •how are you? •I'am fine. •Thankyou. •How are you \
-doing? •This\n• costs $20,00,000. •</data>
+doing? •This\n<100> costs $20,00,000. •</data>
# Sentence ending in a quote.
<data>•"Sentence ending with a quote." •Bye.•</data>
# Sentence, and test data, ending without a period or other terminator.
-<data>•Here is a random sentence, no ending period•</data>
+<data>•Here is a random sentence, no ending period<100></data>
<data>• (This is it). •Testing the sentence iterator. •\
"This isn't it." •Hi! \
•This is a simple sample sentence. •(This is it.) •This is a simple sample sentence. •\
"This isn't it." •\
-Hi! •This is a simple sample sentence. •It does not have to make any sense as you can see. •Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura. •Che la dritta via aveo smarrita. •He said, that I said, that you said!•! •Don't rock the boat.\u2029•Because I am the daddy, that is why.
-•Not on my time (el timo.)•! •</data>
+Hi! •This is a simple sample sentence. •It does not have to make any sense as you can see. •Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura. •Che la dritta via aveo smarrita. •He said, that I said, that you said!! •Don't rock the boat.\u2029•Because I am the daddy, that is why.
+•Not on my time (el timo.)! •</data>
-<data>•Hello. •So what!•!\u2029•"But now," he said, \
+<data>•Hello. •So what!!\u2029•"But now," he said, \
"I know!" •\
-Harris thumbed down several, including "Away We Go" (which became the huge success Oklahoma!)•. •One species, B. anthracis, is highly virulent.
+Harris thumbed down several, including "Away We Go" (which became the huge success Oklahoma!). •One species, B. anthracis, is highly virulent.
•Wolf said about Sounder:\
"Beautifully thought-out and directed." •\
Have you ever said, "This is where\tI shall live"? •He answered, \
-"You may not!" •Another popular saying is: "How do you do?"•. \n•\
+"You may not!" •Another popular saying is: "How do you do?". \n•\
Yet another popular saying is: \
'I'm fine thanks.' •\
-What is the proper use of the abbreviation pp.•? •Yes, I am definatelly 12" tall!•!\
-•Now\r•is\n•the\r\n•time\n•\r•for\r•\r•</data>
+What is the proper use of the abbreviation pp.? •Yes, I am definatelly 12" tall!!\
+•Now\r<100>is\n<100>the\r\n<100>time\n<100>\r<100>for\r<100>\r<100></data>
<data>•No breaks when . is surrounded by UPPER.Case letters. •</data>
<data>•No breaks when . is followed by Numeric .4 a.4 C.4 3.1 .•</data>
# Treat fullwidth variants of .!? the same as their
# normal counterparts
#
-<data>•I know I'm right\uff0e •Right\uff1f •Right\uff01 •</data>"
+<data>•I know I'm right\uff0e •Right\uff1f •Right\uff01 •</data>
#
# Don't break sentences at boundary between CJK and digits
#
-<data>•\u5487\u67ff\ue591\u5017\u61b3\u60a1\u9510\u8165\u9de8\u97e48888\u821c\u8165\u7fc8\u51ce\u306d\ue30b\u2494\u56d8\u4ec0\u60b1\u8560\u51ba\u611d\u57b6\u2510\u5d46\u97e5\u7751\u3002•Bye, now•</data>"
+<data>•\u5487\u67ff\ue591\u5017\u61b3\u60a1\u9510\u8165\u9de8\u97e48888\u821c\u8165\u7fc8\u51ce\u306d\ue30b\u2494\u56d8\u4ec0\u60b1\u8560\u51ba\u611d\u57b6\u2510\u5d46\u97e5\u7751\u3002•Bye, now<100></data>
#
# Breaks around '(' following a sentence TERM. (Rule 9)
<data>•How do you do? •(fine). •</data>
#
-<data>•Hello.123•</data> # Rule 6
-<data>•Hello?•123•</data>
+<data>•Hello.123<100></data> # Rule 6
+<data>•Hello?•123<100></data>
-<data>•HELLO.Bye•</data> # Rule 7
-<data>•HELLO?•Bye•</data>
+<data>•HELLO.Bye<100></data> # Rule 7
+<data>•HELLO?•Bye<100></data>
-<data>•Hello.goodbye•</data> #Rule 8
-<data>•Hello. •Goodbye•</data>
-<data>•Hello. goodbye•</data>
-<data>•Hello.)@#$%^&*()"" goodbye•</data>
+<data>•Hello.goodbye<100></data> #Rule 8
+<data>•Hello. •Goodbye<100></data>
+<data>•Hello. goodbye<100></data>
# make sure there is sentence break after ?,danda(hindi phrase separator),
# fullstop followed by space. (VERY old test)
#
-<data>•\u0928\u092e\u0938\u094d\u200d\u0924\u0947 \u0930\u092e\u0947\u0936\u0905\u093e\u092a\u0915\u0948\u0938\u0947 \u0939\u0948?•\u092e\u0948 \u0905\u091a\u094d\u200d \u091b\u093e \u0939\u0942\u0901\u0964 •\u0905\u093e\u092a\r\n•\
+<data>•\u0928\u092e\u0938\u094d\u200d\u0924\u0947 \u0930\u092e\u0947\u0936\u0905\u093e\u092a\u0915\u0948\u0938\u0947 \u0939\u0948?•\u092e\u0948 \u0905\u091a\u094d\u200d \u091b\u093e \u0939\u0942\u0901\u0964 •\u0905\u093e\u092a\r\n<100>\
\u0915\u0948\u0938\u0947 \u0939\u0948?•\u0935\u0939 \u0915\u094d\u200d\u092f\u093e\n\
-•\u0939\u0948?•\u092f\u0939 \u0905\u093e\u092e \u0939\u0948. •\u092f\u0939 means "this". •"\u092a\u095d\u093e\u0908" meaning "education" or "studies". •\u0905\u093e\u091c(\u0938\u094d\u200d\u0935\u0924\u0902\u0924\u094d\u0930 \u0926\u093f\u0935\u093e\u0938) \u0939\u0948\u0964 •Let's end here. •</data>
+<100>\u0939\u0948?•\u092f\u0939 \u0905\u093e\u092e \u0939\u0948. •\u092f\u0939 means "this". •"\u092a\u095d\u093e\u0908" meaning "education" or "studies". •\u0905\u093e\u091c(\u0938\u094d\u200d\u0935\u0924\u0902\u0924\u094d\u0930 \u0926\u093f\u0935\u093e\u0938) \u0939\u0948\u0964 •Let's end here. •</data>
# Regression test for bug #1984, Sentence break in Arabic text.
<data>\
-•\u0623\u0633\u0627\u0633\u064b\u0627\u060c\u0020\u062a\u062a\u0639\u0627"\u0645\u0644\u0020\u0627\u0644\u062d\u0648\u0627\u0633\u064a\u0628\u0020"\u0641\u0642\u0637\u0020\u0645\u0639\u0020\u0627\u0644\u0623\u0631\u0642\u0627\u0645\u060c\u0648\u062a\u0642\u0648\u0645\u0020\u0628\u062a\u062e\u0632\u064a\u0646\u0020\u0627\u0644\u0623\u062d\u0631\u0641\u0020\u0648\u0627\u0644\u0645\u062d\u0627\u0631\u0641\u0020\u0627\u0644\u0623\u062e\u0631\u0649\u0020\u0628\u0639\u062f\u0020\u0623\u0646\u062a\u064f\u0639\u0637\u064a\u0020\u0631\u0642\u0645\u0627\u0020\u0645\u0639\u064a\u0646\u0627\u0020\u0644\u0643\u0644\u0020\u0648\u0627\u062d\u062f\u0020\u0645\u0646\u0647\u0627\u002e\u0020•\u0648\u0642\u0628\u0644\u0020\u0627\u062e\u062a\u0631\u0627\u0639\u0022\u064a\u0648\u0646\u0650\u0643\u0648\u062f\u0022\u060c\u0020\u0643\u0627\u0646\u0020\u0647\u0646\u0627\u0643\u0020\u0645\u0626\u0627\u062a\u0020\u0627\u0644\u0623\u0646\u0638\u0645\u0629\u0020\u0644\u0644\u062a\u0634\u0641\u064a\u0631\u0648\u062a\u062e\u0635\u064a\u0635\u0020\u0647\u0630\u0647\u0020\u0627\u0644\u0623\u0631\u0642\u0627\u0645\u0020\u0644\u0644\u0645\u062d\u0627\u0631\u0641\u060c\u0020\u0648\u0644\u0645\u0020\u064a\u0648\u062c\u062f\u0020\u0646\u0638\u0627\u0645\u062a\u0634\u0641\u064a\u0020\u0639\u0644\u0649\u0020\u062c\u0645\u064a\u0639\u0020\u0627\u0644\u0645\u062d\u0627\u0631\u0641\u0020\u0627\u0644\u0636\u0631\u0648\u0631\u064a\u0629. •</data>"
+•\u0623\u0633\u0627\u0633\u064b\u0627\u060c\u0020\u062a\u062a\u0639\u0627"\u0645\u0644\u0020\u0627\u0644\u062d\u0648\u0627\u0633\u064a\u0628\u0020"\u0641\u0642\u0637\u0020\u0645\u0639\u0020\u0627\u0644\u0623\u0631\u0642\u0627\u0645\u060c\u0648\u062a\u0642\u0648\u0645\u0020\u0628\u062a\u062e\u0632\u064a\u0646\u0020\u0627\u0644\u0623\u062d\u0631\u0641\u0020\u0648\u0627\u0644\u0645\u062d\u0627\u0631\u0641\u0020\u0627\u0644\u0623\u062e\u0631\u0649\u0020\u0628\u0639\u062f\u0020\u0623\u0646\u062a\u064f\u0639\u0637\u064a\u0020\u0631\u0642\u0645\u0627\u0020\u0645\u0639\u064a\u0646\u0627\u0020\u0644\u0643\u0644\u0020\u0648\u0627\u062d\u062f\u0020\u0645\u0646\u0647\u0627\u002e\u0020•\u0648\u0642\u0628\u0644\u0020\u0627\u062e\u062a\u0631\u0627\u0639\u0022\u064a\u0648\u0646\u0650\u0643\u0648\u062f\u0022\u060c\u0020\u0643\u0627\u0646\u0020\u0647\u0646\u0627\u0643\u0020\u0645\u0626\u0627\u062a\u0020\u0627\u0644\u0623\u0646\u0638\u0645\u0629\u0020\u0644\u0644\u062a\u0634\u0641\u064a\u0631\u0648\u062a\u062e\u0635\u064a\u0635\u0020\u0647\u0630\u0647\u0020\u0627\u0644\u0623\u0631\u0642\u0627\u0645\u0020\u0644\u0644\u0645\u062d\u0627\u0631\u0641\u060c\u0020\u0648\u0644\u0645\u0020\u064a\u0648\u062c\u062f\u0020\u0646\u0638\u0627\u0645\u062a\u0634\u0641\u064a\u0020\u0639\u0644\u0649\u0020\u062c\u0645\u064a\u0639\u0020\u0627\u0644\u0645\u062d\u0627\u0631\u0641\u0020\u0627\u0644\u0636\u0631\u0648\u0631\u064a\u0629. •</data>
# Try a few more of the less common sentence endings.
<data>•Hello, world\u3002 •Hello, world\u1803 •Hello, world\u2048 •Hello, world\u203c •Let's end here. •</data>
<data>• •\u0041•</data>
<data>• •\u0009•</data>
<data>• •\u00B4•</data>
-<data>• \u000C•</data> # LB3C × BK
+<data>• \u000C<100></data> # LB3C × BK
<data>• •\u2014•</data>
<data>• •\uFFFC•</data>
<data>• \u0029•</data> # LB 8 × CL
<data>• •\u4E00•</data>
<data>• •\u2024•</data>
<data>• \u002C•</data> # LB 8 × IS
-<data>• \u000A•</data> # LB3C × ( BK | CR | LF | NL )
+<data>• \u000A<100></data> # LB3C × ( BK | CR | LF | NL )
<data>• •\u0E5A•</data>
<data>• •\u0032•</data>
<data>• •\u0028•</data>
# 3a Always break after hard line breaks.
# 3c Never break before hard line breaks.
-<data>• •\u00A1\u2028•\u00A1•</data>
-<data>• •\u0041\u2028•\u0041•</data>
-<data>• •\u0009\u2028•\u0009•</data>
-<data>• •\u00B4\u2028•\u00B4•</data>
-<data>• \u000C•\u2028•\u000C•</data>
-<data>• •\u2014\u2028•\u2014•</data>
-<data>• •\uFFFC\u2028•\uFFFC•</data>
-<data>• \u0029\u2028•\u0029•</data>
-#<data>• \u0301\u2028•\u0301•</data> # TODO: fix.
-<data>• \u0021\u2028•\u0021•</data>
-#<data>• \u00A0\u2028•\u00A0•</data> # TODO: fix
-<data>• •\u002D\u2028•\u002D•</data>
-<data>• •\u4E00\u2028•\u4E00•</data>
-<data>• •\u2024\u2028•\u2024•</data>
-<data>• \u002C\u2028•\u002C•</data>
-<data>• \u000A•\u2028•\u000A•</data>
-<data>• •\u0E5A\u2028•\u0E5A•</data>
-<data>• •\u0032\u2028•\u0032•</data>
-<data>• •\u0028\u2028•\u0028•</data>
-<data>• •\u0025\u2028•\u0025•</data>
-<data>• •\u0024\u2028•\u0024•</data>
-<data>• •\u0022\u2028•\u0022•</data>
-<data>• •\u0E01\u2028•\u0E01•</data>
-<data>• •\uDB7F\u2028•\uDB7F•</data>
-<data>• \u0020\u2028•\u0020•</data>
-<data>• \u002F\u2028•\u002F•</data>
-<data>• •\uF8FF\u2028•\uF8FF•</data>
-<data>• \u200B\u2028•\u200B•</data>
+<data>• •\u00A1\u2028<100>\u00A1•</data>
+<data>• •\u0041\u2028<100>\u0041•</data>
+<data>• •\u0009\u2028<100>\u0009•</data>
+<data>• •\u00B4\u2028<100>\u00B4•</data>
+<data>• \u000C<100>\u2028<100>\u000C<100></data>
+<data>• •\u2014\u2028<100>\u2014•</data>
+<data>• •\uFFFC\u2028<100>\uFFFC•</data>
+<data>• \u0029\u2028<100>\u0029•</data>
+#<data>• \u0301\u2028<100>\u0301•</data> # TODO: fix.
+<data>• \u0021\u2028<100>\u0021•</data>
+#<data>• \u00A0\u2028<100>\u00A0•</data> # TODO: fix
+<data>• •\u002D\u2028<100>\u002D•</data>
+<data>• •\u4E00\u2028<100>\u4E00•</data>
+<data>• •\u2024\u2028<100>\u2024•</data>
+<data>• \u002C\u2028<100>\u002C•</data>
+<data>• \u000A<100>\u2028<100>\u000A<100></data>
+<data>• •\u0E5A\u2028<100>\u0E5A•</data>
+<data>• •\u0032\u2028<100>\u0032•</data>
+<data>• •\u0028\u2028<100>\u0028•</data>
+<data>• •\u0025\u2028<100>\u0025•</data>
+<data>• •\u0024\u2028<100>\u0024•</data>
+<data>• •\u0022\u2028<100>\u0022•</data>
+<data>• •\u0E01\u2028<100>\u0E01•</data>
+<data>• •\uDB7F\u2028<100>\uDB7F•</data>
+<data>• \u0020\u2028<100>\u0020•</data>
+<data>• \u002F\u2028<100>\u002F•</data>
+<data>• •\uF8FF\u2028<100>\uF8FF•</data>
+<data>• \u200B\u2028<100>\u200B•</data>
#
<line>
<data>•Multi-•Level •example •of •a •semi-•idiotic •non-•sensical •(non-•important) •sentence.
-•Hi •Hello •How\n•are\r•you\u2028•fine.\t•good. •Now\r•is\n•the\r\n•time\n•\r•for\r•\r•all•</data>
+<100>Hi •Hello •How\n<100>are\r<100>you\u2028<100>fine.\t•good. •Now\r<100>is\n<100>the\r\n<100>time\n<100>\r<100>for\r<100>\r<100>all•</data>
<line>
-<data>•Hello! •how\r\n• •(are)\r• •you? •I'am •fine- •Thankyou. •foo\u00a0bar
-•How, •are, •you? •This, •costs •$20,00,000.•</data>
+<data>•Hello! •how\r\n<100> •(are)\r<100> •you? •I'am •fine- •Thankyou. •foo\u00a0bar
+<100>How, •are, •you? •This, •costs •$20,00,000.•</data>
# test for bug #4068133
#
<data>•foo\u00a0bar•</data>
# to test for bug #4097920
-<data>•dog,•cat,•mouse •(one)•(two)\n•</data>
+<data>•dog,cat,mouse •(one)•(two)\n<100></data>
# to test for bug #4035266
-<data>•The •balance •is •$-23,456.78, •not •-•$32,456.78!\n•</data>
+<data>•The •balance •is •$-23,456.78, •not •-•$32,456.78!\n<100></data>
# to test for bug #4098467
# Surrogate line break tests.
#
-<data>•\u4e01•\ud840\udc01•\u4e02•abc•\ue000•\udb80\udc01•</data>
+<data>•\u4e01•\ud840\udc01•\u4e02•abc •\ue000 •\udb80\udc01•</data>
# Regression for bug 836
+# Note: Unicode 5.1 changed this behavior
+# ICU will want to change it back before releasing,
+# so there is no break preceding the '('
<data>•AAA•(AAA •</data>
# Try some words from other scripts.
<data>•ΑΒΓ •БВГ •אבג֓ •ابت •١٢٣ •\u10A0\u10A1\u10A2 •ABC •</data>
+
########################################################################################
#
#
<data>•'•start •with •a •case-•ignorable •cha'r'a'cter•</data>
+
+##########################################################################################
+#
+# Thai Tests
+#
+##########################################################################################
+<locale th>
+<word>
+#
+# Test data originally from the test code source file
+# // @suwit -- Thai sample data from GVT Guideline
+#
+<data>•\u0E2B\u0E19\u0E36\u0E48\u0E07<200>\u0E04\u0E33<200>\u0E44\u0E17\u0E22<200>\
+\u0E2A\u0E32\u0E21\u0E32\u0E23\u0E16<200>\u0E1B\u0E23\u0E30\u0E01\u0E2D\u0E1A<200>\
+\u0E14\u0E49\u0E27\u0E22<200>\u0e2b\u0e25\u0e32\u0e22<200>\
+\u0e1e\u0e22\u0e32\u0e07\u0e04\u0e4c<200></data>
+
+#
+# Jitterbug 3671 Test Case
+#
+<data>•สวัสดี<200>ครับ<200>สบาย<200>ดี<200>ไหม<200> •ครับ<200></data>
+
+#
+# Trac ticket 5595 Test Case
+<data>•บท<200>ที่๑พายุ<200>ไซโคลน<200>โด<200>โรธี<200>อาศัย<200>อยู่<200>ท่ามกลาง<200>\
+ทุ่งใหญ่<200>ใน<200>แคนซัส<200>กับ<200>ลุง<200>เฮ<200>นรี<200>ชาวไร่<200>และ<200>ป้า<200>เอ็ม<200>\
+ภรรยา<200>ชาวไร่<200>บ้าน<200>ของ<200>พวก<200>เขา<200>หลัง<200>เล็ก<200>เพราะ<200>ไม้<200>\
+สร้าง<200>บ้าน<200>ต้อง<200>ขน<200>มา<200>ด้วย<200>เกวียน<200>เป็น<200>ระยะ<200>ทาง<200>หลาย<200>\
+ไมล์<200></data>
+
+