ICU-64232.0.1.tar.gz

[apple/icu.git] / icuSources / common / rbbiscan.cpp

diff --git a/icuSources/common/rbbiscan.cpp b/icuSources/common/rbbiscan.cpp
index dacbd63ab4393bd2fffa2a1ca830c56bdd887cff..f536ab5d39863c270e711d48e951858b48ae11c7 100644 (file)
--- a/icuSources/common/rbbiscan.cpp
+++ b/icuSources/common/rbbiscan.cpp
@@ -1,8 +1,9 @@
-
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html

 //
 //  file:  rbbiscan.cpp
 //
 //
 //  file:  rbbiscan.cpp
 //

-//  Copyright (C) 2002-2010, International Business Machines Corporation and others.
+//  Copyright (C) 2002-2016, International Business Machines Corporation and others.

 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for
 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for


@@ -23,7 +24,6 @@
 #include "unicode/uchriter.h"
 #include "unicode/parsepos.h"
 #include "unicode/parseerr.h"
 #include "unicode/uchriter.h"
 #include "unicode/parsepos.h"
 #include "unicode/parseerr.h"

-#include "util.h"

 #include "cmemory.h"
 #include "cstring.h"
 
 #include "cmemory.h"
 #include "cstring.h"
 


@@ -36,7 +36,6 @@
 
 #include "uassert.h"
 
 
 #include "uassert.h"
 

-

 //------------------------------------------------------------------------------
 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.
 //------------------------------------------------------------------------------
 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.


@@ -48,6 +47,7 @@
 //
 //------------------------------------------------------------------------------
 static const UChar gRuleSet_rule_char_pattern[]       = {
 //
 //------------------------------------------------------------------------------
 static const UChar gRuleSet_rule_char_pattern[]       = {

+ // Characters that may appear as literals in patterns without escaping or quoting.

  //   [    ^      [    \     p     {      Z     }     \     u    0      0    2      0
     0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
  //   -    \      u    0     0     7      f     ]     -     [    \      p
  //   [    ^      [    \     p     {      Z     }     \     u    0      0    2      0
     0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
  //   -    \      u    0     0     7      f     ]     -     [    \      p


@@ -72,7 +72,7 @@ static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00};  // "any"
 
 U_CDECL_BEGIN
 static void U_CALLCONV RBBISetTable_deleter(void *p) {
 
 U_CDECL_BEGIN
 static void U_CALLCONV RBBISetTable_deleter(void *p) {

-    U_NAMESPACE_QUALIFIER RBBISetTableEl *px = (U_NAMESPACE_QUALIFIER RBBISetTableEl *)p;
+    icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p;

     delete px->key;
     // Note:  px->val is owned by the linked list "fSetsListHead" in scanner.
     //        Don't delete the value nodes here.
     delete px->key;
     // Note:  px->val is owned by the linked list "fSetsListHead" in scanner.
     //        Don't delete the value nodes here.


@@ -90,24 +90,27 @@ U_NAMESPACE_BEGIN
 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;
 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;

+    fScanIndex          = 0;
+    fNextIndex          = 0;
+    fQuoteMode          = FALSE;
+    fLineNum            = 1;
+    fCharNum            = 0;
+    fLastChar           = 0;
+    
+    fStateTable         = NULL;
+    fStack[0]           = 0;

     fStackPtr           = 0;
     fStackPtr           = 0;

-    fStack[fStackPtr]   = 0;
-    fNodeStackPtr       = 0;
-    fRuleNum            = 0;

     fNodeStack[0]       = NULL;
     fNodeStack[0]       = NULL;

-
-    fSymbolTable                            = NULL;
-    fSetTable                               = NULL;
-
-    fScanIndex = 0;
-    fNextIndex = 0;
+    fNodeStackPtr       = 0;

 
     fReverseRule        = FALSE;
     fLookAheadRule      = FALSE;
 
     fReverseRule        = FALSE;
     fLookAheadRule      = FALSE;

+    fNoChainInRule      = FALSE;

 
 

-    fLineNum    = 1;
-    fCharNum    = 0;
-    fQuoteMode  = FALSE;
+    fSymbolTable        = NULL;
+    fSetTable           = NULL;
+    fRuleNum            = 0;
+    fOptionStart        = 0;

 
     // Do not check status until after all critical fields are sufficiently initialized
     //   that the destructor can run cleanly.
 
     // Do not check status until after all critical fields are sufficiently initialized
     //   that the destructor can run cleanly.


@@ -121,19 +124,20 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     //            all instances of RBBIRuleScanners.  BUT this is quite a bit simpler,
     //            and the time to build these few sets should be small compared to a
     //            full break iterator build.
     //            all instances of RBBIRuleScanners.  BUT this is quite a bit simpler,
     //            and the time to build these few sets should be small compared to a
     //            full break iterator build.

-    fRuleSets[kRuleSet_rule_char-128]       = UnicodeSet(gRuleSet_rule_char_pattern,       *rb->fStatus);
-    UnicodeSet *whitespaceSet = uprv_openRuleWhiteSpaceSet(rb->fStatus);
-    if (U_FAILURE(*rb->fStatus)) {
-        return;
-    }
-    fRuleSets[kRuleSet_white_space-128]     = *whitespaceSet;
-    delete whitespaceSet;
-    fRuleSets[kRuleSet_name_char-128]       = UnicodeSet(gRuleSet_name_char_pattern,       *rb->fStatus);
-    fRuleSets[kRuleSet_name_start_char-128] = UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus);
-    fRuleSets[kRuleSet_digit_char-128]      = UnicodeSet(gRuleSet_digit_char_pattern,      *rb->fStatus);
+    fRuleSets[kRuleSet_rule_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern),       *rb->fStatus);
+    // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:]
+    fRuleSets[kRuleSet_white_space-128].
+        add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029);
+    fRuleSets[kRuleSet_name_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern),       *rb->fStatus);
+    fRuleSets[kRuleSet_name_start_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus);
+    fRuleSets[kRuleSet_digit_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern),      *rb->fStatus);

     if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
         // This case happens if ICU's data is missing.  UnicodeSet tries to look up property
     if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
         // This case happens if ICU's data is missing.  UnicodeSet tries to look up property

-        //   names from the init string, can't find them, and claims an illegal arguement.
+        //   names from the init string, can't find them, and claims an illegal argument.

         //   Change the error so that the actual problem will be clearer to users.
         *rb->fStatus = U_BRK_INIT_ERROR;
     }
         //   Change the error so that the actual problem will be clearer to users.
         *rb->fStatus = U_BRK_INIT_ERROR;
     }


@@ -207,11 +211,20 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         break;
 
 
         break;
 
 

+    case doNoChain:
+        // Scanned a '^' while on the rule start state.
+        fNoChainInRule = TRUE;
+        break;
+
+

     case doExprOrOperator:
         {
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);
     case doExprOrOperator:
         {
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             orNode->fLeftChild     = operandNode;
             operandNode->fParent   = orNode;
         }
             orNode->fLeftChild     = operandNode;
             operandNode->fParent   = orNode;
         }


@@ -226,6 +239,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *catNode     = pushNewNode(RBBINode::opCat);
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *catNode     = pushNewNode(RBBINode::opCat);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             catNode->fLeftChild    = operandNode;
             operandNode->fParent   = catNode;
         }
             catNode->fLeftChild    = operandNode;
             operandNode->fParent   = catNode;
         }


@@ -314,21 +330,40 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
 #endif
         U_ASSERT(fNodeStackPtr == 1);
         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
 #endif
         U_ASSERT(fNodeStackPtr == 1);

+        RBBINode *thisRule = fNodeStack[fNodeStackPtr];

 
         // If this rule includes a look-ahead '/', add a endMark node to the
         //   expression tree.
         if (fLookAheadRule) {
 
         // If this rule includes a look-ahead '/', add a endMark node to the
         //   expression tree.
         if (fLookAheadRule) {

-            RBBINode  *thisRule       = fNodeStack[fNodeStackPtr];

             RBBINode  *endNode        = pushNewNode(RBBINode::endMark);
             RBBINode  *catNode        = pushNewNode(RBBINode::opCat);
             RBBINode  *endNode        = pushNewNode(RBBINode::endMark);
             RBBINode  *catNode        = pushNewNode(RBBINode::opCat);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             fNodeStackPtr -= 2;
             catNode->fLeftChild       = thisRule;
             catNode->fRightChild      = endNode;
             fNodeStack[fNodeStackPtr] = catNode;
             endNode->fVal             = fRuleNum;
             endNode->fLookAheadEnd    = TRUE;
             fNodeStackPtr -= 2;
             catNode->fLeftChild       = thisRule;
             catNode->fRightChild      = endNode;
             fNodeStack[fNodeStackPtr] = catNode;
             endNode->fVal             = fRuleNum;
             endNode->fLookAheadEnd    = TRUE;

+            thisRule                  = catNode;
+
+            // TODO: Disable chaining out of look-ahead (hard break) rules.
+            //   The break on rule match is forced, so there is no point in building up
+            //   the state table to chain into another rule for a longer match.

         }
 
         }
 

+        // Mark this node as being the root of a rule.
+        thisRule->fRuleRoot = TRUE;
+
+        // Flag if chaining into this rule is wanted.
+        //    
+        if (fRB->fChainRules &&         // If rule chaining is enabled globally via !!chain
+                !fNoChainInRule) {      //     and no '^' chain-in inhibit was on this rule
+            thisRule->fChainIn = TRUE;
+        }
+
+

         // All rule expressions are ORed together.
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.
         // All rule expressions are ORed together.
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.


@@ -337,7 +372,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         //  (forward, reverse, safe_forward, safe_reverse)
         //  OR this rule into the appropriate group of them.
         //
         //  (forward, reverse, safe_forward, safe_reverse)
         //  OR this rule into the appropriate group of them.
         //

-        RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : fRB->fDefaultTree);
+        RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree);

 
         if (*destRules != NULL) {
             // This is not the first rule encounted.
 
         if (*destRules != NULL) {
             // This is not the first rule encounted.


@@ -345,9 +380,12 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
             // with the current rule expression (on the Node Stack)
             //  with the resulting OR expression going to *destRules
             //
             // with the current rule expression (on the Node Stack)
             //  with the resulting OR expression going to *destRules
             //

-            RBBINode  *thisRule    = fNodeStack[fNodeStackPtr];
+                       thisRule    = fNodeStack[fNodeStackPtr];

             RBBINode  *prevRules   = *destRules;
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);
             RBBINode  *prevRules   = *destRules;
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             orNode->fLeftChild     = prevRules;
             prevRules->fParent     = orNode;
             orNode->fRightChild    = thisRule;
             orNode->fLeftChild     = prevRules;
             prevRules->fParent     = orNode;
             orNode->fRightChild    = thisRule;


@@ -362,6 +400,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         }
         fReverseRule   = FALSE;   // in preparation for the next rule.
         fLookAheadRule = FALSE;
         }
         fReverseRule   = FALSE;   // in preparation for the next rule.
         fLookAheadRule = FALSE;

+        fNoChainInRule = FALSE;

         fNodeStackPtr  = 0;
         }
         break;
         fNodeStackPtr  = 0;
         }
         break;


@@ -388,6 +427,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *plusNode    = pushNewNode(RBBINode::opPlus);
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *plusNode    = pushNewNode(RBBINode::opPlus);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             plusNode->fLeftChild   = operandNode;
             operandNode->fParent   = plusNode;
         }
             plusNode->fLeftChild   = operandNode;
             operandNode->fParent   = plusNode;
         }


@@ -397,6 +439,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *qNode       = pushNewNode(RBBINode::opQuestion);
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *qNode       = pushNewNode(RBBINode::opQuestion);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             qNode->fLeftChild      = operandNode;
             operandNode->fParent   = qNode;
         }
             qNode->fLeftChild      = operandNode;
             operandNode->fParent   = qNode;
         }


@@ -406,6 +451,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *starNode    = pushNewNode(RBBINode::opStar);
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *starNode    = pushNewNode(RBBINode::opStar);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             starNode->fLeftChild   = operandNode;
             operandNode->fParent   = starNode;
         }
             starNode->fLeftChild   = operandNode;
             operandNode->fParent   = starNode;
         }


@@ -419,7 +467,10 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);

-            findSetFor(fC.fChar, n);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
+            findSetFor(UnicodeString(fC.fChar), n);

             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);


@@ -430,7 +481,10 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);

-            findSetFor(kAny, n);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
+            findSetFor(UnicodeString(TRUE, kAny, 3), n);

             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);


@@ -440,6 +494,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.
         n = pushNewNode(RBBINode::lookAhead);
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.
         n = pushNewNode(RBBINode::lookAhead);

+        if (U_FAILURE(*fRB->fStatus)) {
+            break;
+        }

         n->fVal      = fRuleNum;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;
         n->fVal      = fRuleNum;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;


@@ -451,6 +508,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
     case doStartTagValue:
         // Scanned a '{', the opening delimiter for a tag value within a rule.
         n = pushNewNode(RBBINode::tag);
     case doStartTagValue:
         // Scanned a '{', the opening delimiter for a tag value within a rule.
         n = pushNewNode(RBBINode::tag);

+        if (U_FAILURE(*fRB->fStatus)) {
+            break;
+        }

         n->fVal      = 0;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;
         n->fVal      = 0;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;


@@ -499,6 +559,10 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
                 fRB->fDefaultTree   = &fRB->fSafeRevTree;
             } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
                 fRB->fLookAheadHardBreak = TRUE;
                 fRB->fDefaultTree   = &fRB->fSafeRevTree;
             } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
                 fRB->fLookAheadHardBreak = TRUE;

+            } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) {
+                fRuleSets[kRuleSet_rule_char-128].clear();
+            } else if (opt == UNICODE_STRING("unquoted_literals",  17)) {
+                fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus);

             } else {
                 error(U_BRK_UNRECOGNIZED_OPTION);
             }
             } else {
                 error(U_BRK_UNRECOGNIZED_OPTION);
             }


@@ -561,7 +625,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         returnVal = FALSE;
         break;
     }
         returnVal = FALSE;
         break;
     }

-    return returnVal;
+    return returnVal && U_SUCCESS(*fRB->fStatus);

 }
 
 
 }
 
 


@@ -580,7 +644,7 @@ void RBBIRuleScanner::error(UErrorCode e) {
             fRB->fParseError->line  = fLineNum;
             fRB->fParseError->offset = fCharNum;
             fRB->fParseError->preContext[0] = 0;
             fRB->fParseError->line  = fLineNum;
             fRB->fParseError->offset = fCharNum;
             fRB->fParseError->preContext[0] = 0;

-            fRB->fParseError->preContext[0] = 0;
+            fRB->fParseError->postContext[0] = 0;

         }
     }
 }
         }
     }
 }


@@ -758,27 +822,24 @@ static const UChar      chRParen    = 0x29;
 
 //------------------------------------------------------------------------------
 //
 
 //------------------------------------------------------------------------------
 //

-//  stripRules    Return a rules string without unnecessary
-//                characters.
+//  stripRules    Return a rules string without extra spaces.
+//                (Comments are removed separately, during rule parsing.)

 //
 //------------------------------------------------------------------------------
 UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
     UnicodeString strippedRules;
 //
 //------------------------------------------------------------------------------
 UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
     UnicodeString strippedRules;

-    int rulesLength = rules.length();
-    for (int idx = 0; idx < rulesLength; ) {
-        UChar ch = rules[idx++];
-        if (ch == chPound) {
-            while (idx < rulesLength
-                && ch != chCR && ch != chLF && ch != chNEL)
-            {
-                ch = rules[idx++];
-            }
-        }
-        if (!u_isISOControl(ch)) {
-            strippedRules.append(ch);
+    int32_t rulesLength = rules.length();
+    bool skippingSpaces = false;
+
+    for (int32_t idx=0; idx<rulesLength; idx = rules.moveIndex32(idx, 1)) {
+        UChar32 cp = rules.char32At(idx);
+        bool whiteSpace = u_hasBinaryProperty(cp, UCHAR_PATTERN_WHITE_SPACE);
+        if (skippingSpaces && whiteSpace) {
+            continue;

         }
         }

+        strippedRules.append(cp);
+        skippingSpaces = whiteSpace;

     }
     }

-    // strippedRules = strippedRules.unescape();

     return strippedRules;
 }
 
     return strippedRules;
 }
 


@@ -878,6 +939,7 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
             //  It will be treated as white-space, and serves to break up anything
             //    that might otherwise incorrectly clump together with a comment in
             //    the middle (a variable name, for example.)
             //  It will be treated as white-space, and serves to break up anything
             //    that might otherwise incorrectly clump together with a comment in
             //    the middle (a variable name, for example.)

+            int32_t commentStart = fScanIndex;

             for (;;) {
                 c.fChar = nextCharLL();
                 if (c.fChar == (UChar32)-1 ||  // EOF
             for (;;) {
                 c.fChar = nextCharLL();
                 if (c.fChar == (UChar32)-1 ||  // EOF


@@ -886,6 +948,9 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
                     c.fChar == chNEL    ||
                     c.fChar == chLS)       {break;}
             }
                     c.fChar == chNEL    ||
                     c.fChar == chLS)       {break;}
             }

+            for (int32_t i=commentStart; i<fNextIndex-1; ++i) {
+                fRB->fStrippedRules.setCharAt(i, u' ');
+            }

         }
         if (c.fChar == (UChar32)-1) {
             return;
         }
         if (c.fChar == (UChar32)-1) {
             return;


@@ -963,7 +1028,7 @@ void RBBIRuleScanner::parse() {
 
         for (;;) {
             #ifdef RBBI_DEBUG
 
         for (;;) {
             #ifdef RBBI_DEBUG

-                if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+                if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);}

             #endif
             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and
             #endif
             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and


@@ -992,6 +1057,7 @@ void RBBIRuleScanner::parse() {
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF

+                U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets));

                 if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.
                 if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.


@@ -1045,19 +1111,17 @@ void RBBIRuleScanner::parse() {
 
     }
 
 
     }
 

+    if (U_FAILURE(*fRB->fStatus)) {
+        return;
+    }
+    
+    // If there are no forward rules set an error.

     //
     //

-    // If there were NO user specified reverse rules, set up the equivalent of ".*;"
-    //
-    if (fRB->fReverseTree == NULL) {
-        fRB->fReverseTree  = pushNewNode(RBBINode::opStar);
-        RBBINode  *operand = pushNewNode(RBBINode::setRef);
-        findSetFor(kAny, operand);
-        fRB->fReverseTree->fLeftChild = operand;
-        operand->fParent              = fRB->fReverseTree;
-        fNodeStackPtr -= 2;
+    if (fRB->fForwardTree == NULL) {
+        error(U_BRK_RULE_SYNTAX);
+        return;

     }
 
     }
 

-

     //
     // Parsing of the input RBBI rules is complete.
     // We now have a parse tree for the rule expressions
     //
     // Parsing of the input RBBI rules is complete.
     // We now have a parse tree for the rule expressions


@@ -1065,16 +1129,15 @@ void RBBIRuleScanner::parse() {
     //
 #ifdef RBBI_DEBUG
     if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();}
     //
 #ifdef RBBI_DEBUG
     if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();}

-    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree"))
-    {
+    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) {

         RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");
         RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");

-        fRB->fForwardTree->printTree(TRUE);
+        RBBINode::printTree(fRB->fForwardTree, TRUE);

         RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");
         RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");

-        fRB->fReverseTree->printTree(TRUE);
+        RBBINode::printTree(fRB->fReverseTree, TRUE);

         RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n");
         RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n");

-        fRB->fSafeFwdTree->printTree(TRUE);
+        RBBINode::printTree(fRB->fSafeFwdTree, TRUE);

         RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n");
         RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n");

-        fRB->fSafeRevTree->printTree(TRUE);
+        RBBINode::printTree(fRB->fSafeRevTree, TRUE);

     }
 #endif
 }
     }
 #endif
 }


@@ -1089,7 +1152,7 @@ void RBBIRuleScanner::parse() {
 void RBBIRuleScanner::printNodeStack(const char *title) {
     int i;
     RBBIDebugPrintf("%s.  Dumping node stack...\n", title);
 void RBBIRuleScanner::printNodeStack(const char *title) {
     int i;
     RBBIDebugPrintf("%s.  Dumping node stack...\n", title);

-    for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree(TRUE);}
+    for (i=fNodeStackPtr; i>0; i--) {RBBINode::printTree(fNodeStack[i], TRUE);}

 }
 #endif
 
 }
 #endif
 


@@ -1103,13 +1166,15 @@ void RBBIRuleScanner::printNodeStack(const char *title) {
 //
 //------------------------------------------------------------------------------
 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
 //
 //------------------------------------------------------------------------------
 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {

-    fNodeStackPtr++;
-    if (fNodeStackPtr >= kStackSize) {
-        error(U_BRK_INTERNAL_ERROR);
+    if (U_FAILURE(*fRB->fStatus)) {
+        return NULL;
+    }
+    if (fNodeStackPtr >= kStackSize - 1) {
+        error(U_BRK_RULE_SYNTAX);

         RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow.");
         RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow.");

-        *fRB->fStatus = U_BRK_INTERNAL_ERROR;

         return NULL;
     }
         return NULL;
     }

+    fNodeStackPtr++;

     fNodeStack[fNodeStackPtr] = new RBBINode(t);
     if (fNodeStack[fNodeStackPtr] == NULL) {
         *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR;
     fNodeStack[fNodeStackPtr] = new RBBINode(t);
     if (fNodeStack[fNodeStackPtr] == NULL) {
         *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR;


@@ -1146,11 +1211,11 @@ void RBBIRuleScanner::scanSet() {
     pos.setIndex(fScanIndex);
     startPos = fScanIndex;
     UErrorCode localStatus = U_ZERO_ERROR;
     pos.setIndex(fScanIndex);
     startPos = fScanIndex;
     UErrorCode localStatus = U_ZERO_ERROR;

-    uset = new UnicodeSet(fRB->fRules, pos, USET_IGNORE_SPACE,
-                         fSymbolTable,
-                         localStatus);
+    uset = new UnicodeSet();

     if (uset == NULL) {
         localStatus = U_MEMORY_ALLOCATION_ERROR;
     if (uset == NULL) {
         localStatus = U_MEMORY_ALLOCATION_ERROR;

+    } else {
+        uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus);

     }
     if (U_FAILURE(localStatus)) {
         //  TODO:  Get more accurate position of the error from UnicodeSet's return info.
     }
     if (U_FAILURE(localStatus)) {
         //  TODO:  Get more accurate position of the error from UnicodeSet's return info.


@@ -1165,6 +1230,7 @@ void RBBIRuleScanner::scanSet() {
 
     // Verify that the set contains at least one code point.
     //
 
     // Verify that the set contains at least one code point.
     //

+    U_ASSERT(uset!=NULL);

     if (uset->isEmpty()) {
         // This set is empty.
         //  Make it an error, because it almost certainly is not what the user wanted.
     if (uset->isEmpty()) {
         // This set is empty.
         //  Make it an error, because it almost certainly is not what the user wanted.


@@ -1191,6 +1257,9 @@ void RBBIRuleScanner::scanSet() {
         RBBINode         *n;
 
         n = pushNewNode(RBBINode::setRef);
         RBBINode         *n;
 
         n = pushNewNode(RBBINode::setRef);

+        if (U_FAILURE(*fRB->fStatus)) {
+            return;
+        }

         n->fFirstPos = startPos;
         n->fLastPos  = fNextIndex;
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
         n->fFirstPos = startPos;
         n->fLastPos  = fNextIndex;
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);