ICU-64232.0.1.tar.gz

[apple/icu.git] / icuSources / common / rbbiscan.cpp

diff --git a/icuSources/common/rbbiscan.cpp b/icuSources/common/rbbiscan.cpp
index c5f15635fa20bb2489777d12191f46c641827c7e..f536ab5d39863c270e711d48e951858b48ae11c7 100644 (file)
--- a/icuSources/common/rbbiscan.cpp
+++ b/icuSources/common/rbbiscan.cpp
@@ -1,8 +1,9 @@
-
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html

 //
 //  file:  rbbiscan.cpp
 //
 //
 //  file:  rbbiscan.cpp
 //

-//  Copyright (C) 2002-2003, International Business Machines Corporation and others.
+//  Copyright (C) 2002-2016, International Business Machines Corporation and others.

 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for
 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for


@@ -23,7 +24,6 @@
 #include "unicode/uchriter.h"
 #include "unicode/parsepos.h"
 #include "unicode/parseerr.h"
 #include "unicode/uchriter.h"
 #include "unicode/parsepos.h"
 #include "unicode/parseerr.h"

-#include "uprops.h"

 #include "cmemory.h"
 #include "cstring.h"
 
 #include "cmemory.h"
 #include "cstring.h"
 


@@ -32,11 +32,11 @@
 #include "rbbirb.h"
 #include "rbbinode.h"
 #include "rbbiscan.h"
 #include "rbbirb.h"
 #include "rbbinode.h"
 #include "rbbiscan.h"

+#include "rbbitblb.h"

 
 #include "uassert.h"
 
 
 #include "uassert.h"
 

-
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.
 //               (Initialized with hex values for portability to EBCDIC based machines.
 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.
 //               (Initialized with hex values for portability to EBCDIC based machines.


@@ -45,8 +45,9 @@
 //              The sets are referred to by name in the rbbirpt.txt, which is the
 //              source form of the state transition table for the RBBI rule parser.
 //
 //              The sets are referred to by name in the rbbirpt.txt, which is the
 //              source form of the state transition table for the RBBI rule parser.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 static const UChar gRuleSet_rule_char_pattern[]       = {
 static const UChar gRuleSet_rule_char_pattern[]       = {

+ // Characters that may appear as literals in patterns without escaping or quoting.

  //   [    ^      [    \     p     {      Z     }     \     u    0      0    2      0
     0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
  //   -    \      u    0     0     7      f     ]     -     [    \      p
  //   [    ^      [    \     p     {      Z     }     \     u    0      0    2      0
     0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
  //   -    \      u    0     0     7      f     ]     -     [    \      p


@@ -70,8 +71,8 @@ static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00};  // "any"
 
 
 U_CDECL_BEGIN
 
 
 U_CDECL_BEGIN

-static void  U_EXPORT2 U_CALLCONV RBBISetTable_deleter(void *p) {
-    RBBISetTableEl *px = (RBBISetTableEl *)p;
+static void U_CALLCONV RBBISetTable_deleter(void *p) {
+    icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p;

     delete px->key;
     // Note:  px->val is owned by the linked list "fSetsListHead" in scanner.
     //        Don't delete the value nodes here.
     delete px->key;
     // Note:  px->val is owned by the linked list "fSetsListHead" in scanner.
     //        Don't delete the value nodes here.


@@ -81,38 +82,38 @@ U_CDECL_END
 
 U_NAMESPACE_BEGIN
 
 
 U_NAMESPACE_BEGIN
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Constructor.
 //
 //
 //  Constructor.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;
 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;

+    fScanIndex          = 0;
+    fNextIndex          = 0;
+    fQuoteMode          = FALSE;
+    fLineNum            = 1;
+    fCharNum            = 0;
+    fLastChar           = 0;
+    
+    fStateTable         = NULL;
+    fStack[0]           = 0;

     fStackPtr           = 0;
     fStackPtr           = 0;

-    fStack[fStackPtr]   = 0;
-    fNodeStackPtr       = 0;
-    fRuleNum            = 0;

     fNodeStack[0]       = NULL;
     fNodeStack[0]       = NULL;

-
-    fRuleSets[kRuleSet_rule_char-128]       = NULL;
-    fRuleSets[kRuleSet_white_space-128]     = NULL;
-    fRuleSets[kRuleSet_name_char-128]       = NULL;
-    fRuleSets[kRuleSet_name_start_char-128] = NULL;
-    fRuleSets[kRuleSet_digit_char-128]      = NULL;
-    fSymbolTable                            = NULL;
-    fSetTable                               = NULL;
-
-    fScanIndex = 0;
-    fNextIndex = 0;
+    fNodeStackPtr       = 0;

 
     fReverseRule        = FALSE;
     fLookAheadRule      = FALSE;
 
     fReverseRule        = FALSE;
     fLookAheadRule      = FALSE;

+    fNoChainInRule      = FALSE;

 
 

-    fLineNum    = 1;
-    fCharNum    = 0;
-    fQuoteMode  = FALSE;
+    fSymbolTable        = NULL;
+    fSetTable           = NULL;
+    fRuleNum            = 0;
+    fOptionStart        = 0;

 
 

+    // Do not check status until after all critical fields are sufficiently initialized
+    //   that the destructor can run cleanly.

     if (U_FAILURE(*rb->fStatus)) {
         return;
     }
     if (U_FAILURE(*rb->fStatus)) {
         return;
     }


@@ -123,14 +124,20 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     //            all instances of RBBIRuleScanners.  BUT this is quite a bit simpler,
     //            and the time to build these few sets should be small compared to a
     //            full break iterator build.
     //            all instances of RBBIRuleScanners.  BUT this is quite a bit simpler,
     //            and the time to build these few sets should be small compared to a
     //            full break iterator build.

-    fRuleSets[kRuleSet_rule_char-128]       = new UnicodeSet(gRuleSet_rule_char_pattern,       *rb->fStatus);
-    fRuleSets[kRuleSet_white_space-128]     = (UnicodeSet*) uprv_openRuleWhiteSpaceSet(rb->fStatus);
-    fRuleSets[kRuleSet_name_char-128]       = new UnicodeSet(gRuleSet_name_char_pattern,       *rb->fStatus);
-    fRuleSets[kRuleSet_name_start_char-128] = new UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus);
-    fRuleSets[kRuleSet_digit_char-128]      = new UnicodeSet(gRuleSet_digit_char_pattern,      *rb->fStatus);
+    fRuleSets[kRuleSet_rule_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern),       *rb->fStatus);
+    // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:]
+    fRuleSets[kRuleSet_white_space-128].
+        add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029);
+    fRuleSets[kRuleSet_name_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern),       *rb->fStatus);
+    fRuleSets[kRuleSet_name_start_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus);
+    fRuleSets[kRuleSet_digit_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern),      *rb->fStatus);

     if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
         // This case happens if ICU's data is missing.  UnicodeSet tries to look up property
     if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
         // This case happens if ICU's data is missing.  UnicodeSet tries to look up property

-        //   names from the init string, can't find them, and claims an illegal arguement.
+        //   names from the init string, can't find them, and claims an illegal argument.

         //   Change the error so that the actual problem will be clearer to users.
         *rb->fStatus = U_BRK_INIT_ERROR;
     }
         //   Change the error so that the actual problem will be clearer to users.
         *rb->fStatus = U_BRK_INIT_ERROR;
     }


@@ -139,24 +146,25 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     }
 
     fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus);
     }
 
     fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus);

-    fSetTable    = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, rb->fStatus);
+    if (fSymbolTable == NULL) {
+        *rb->fStatus = U_MEMORY_ALLOCATION_ERROR;
+        return;
+    }
+    fSetTable    = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus);
+    if (U_FAILURE(*rb->fStatus)) {
+        return;
+    }

     uhash_setValueDeleter(fSetTable, RBBISetTable_deleter);
 }
 
 
 
     uhash_setValueDeleter(fSetTable, RBBISetTable_deleter);
 }
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Destructor
 //
 //
 //  Destructor
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 RBBIRuleScanner::~RBBIRuleScanner() {
 RBBIRuleScanner::~RBBIRuleScanner() {

-    delete fRuleSets[kRuleSet_rule_char-128];
-    delete fRuleSets[kRuleSet_white_space-128];
-    delete fRuleSets[kRuleSet_name_char-128];
-    delete fRuleSets[kRuleSet_name_start_char-128];
-    delete fRuleSets[kRuleSet_digit_char-128];
-

     delete fSymbolTable;
     if (fSetTable != NULL) {
          uhash_close(fSetTable);
     delete fSymbolTable;
     if (fSetTable != NULL) {
          uhash_close(fSetTable);


@@ -175,7 +183,7 @@ RBBIRuleScanner::~RBBIRuleScanner() {
 
 }
 
 
 }
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  doParseAction        Do some action during rule parsing.
 //                       Called by the parse state machine.
 //
 //  doParseAction        Do some action during rule parsing.
 //                       Called by the parse state machine.


@@ -188,14 +196,14 @@ RBBIRuleScanner::~RBBIRuleScanner() {
 //                              in some compilers, while at the same time avoiding multiple
 //                              definitions problems.  I'm sure that there's a better way.
 //
 //                              in some compilers, while at the same time avoiding multiple
 //                              definitions problems.  I'm sure that there's a better way.
 //

-//----------------------------------------------------------------------------------------
-UBool RBBIRuleScanner::doParseActions(EParseAction action)
+//------------------------------------------------------------------------------
+UBool RBBIRuleScanner::doParseActions(int32_t action)

 {
     RBBINode *n       = NULL;
 
     UBool   returnVal = TRUE;
 
 {
     RBBINode *n       = NULL;
 
     UBool   returnVal = TRUE;
 

-    switch ((RBBI_RuleParseAction)action) {
+    switch (action) {

 
     case doExprStart:
         pushNewNode(RBBINode::opStart);
 
     case doExprStart:
         pushNewNode(RBBINode::opStart);


@@ -203,11 +211,20 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         break;
 
 
         break;
 
 

+    case doNoChain:
+        // Scanned a '^' while on the rule start state.
+        fNoChainInRule = TRUE;
+        break;
+
+

     case doExprOrOperator:
         {
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);
     case doExprOrOperator:
         {
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             orNode->fLeftChild     = operandNode;
             operandNode->fParent   = orNode;
         }
             orNode->fLeftChild     = operandNode;
             operandNode->fParent   = orNode;
         }


@@ -222,6 +239,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *catNode     = pushNewNode(RBBINode::opCat);
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *catNode     = pushNewNode(RBBINode::opCat);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             catNode->fLeftChild    = operandNode;
             operandNode->fParent   = catNode;
         }
             catNode->fLeftChild    = operandNode;
             operandNode->fParent   = catNode;
         }


@@ -286,6 +306,13 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
 
             // Make a symbol table entry for the $variableRef node.
             fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus);
 
             // Make a symbol table entry for the $variableRef node.
             fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus);

+            if (U_FAILURE(*fRB->fStatus)) {
+                // This is a round-about way to get the parse position set
+                //  so that duplicate symbols error messages include a line number.
+                UErrorCode t = *fRB->fStatus;
+                *fRB->fStatus = U_ZERO_ERROR;
+                error(t);
+            }

 
             // Clean up the stack.
             delete startExprNode;
 
             // Clean up the stack.
             delete startExprNode;


@@ -299,31 +326,53 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         if (U_FAILURE(*fRB->fStatus)) {       //   parse tree rooted in TOS node.
             break;
         }
         if (U_FAILURE(*fRB->fStatus)) {       //   parse tree rooted in TOS node.
             break;
         }

+#ifdef RBBI_DEBUG

         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}

+#endif

         U_ASSERT(fNodeStackPtr == 1);
         U_ASSERT(fNodeStackPtr == 1);

+        RBBINode *thisRule = fNodeStack[fNodeStackPtr];

 
         // If this rule includes a look-ahead '/', add a endMark node to the
         //   expression tree.
         if (fLookAheadRule) {
 
         // If this rule includes a look-ahead '/', add a endMark node to the
         //   expression tree.
         if (fLookAheadRule) {

-            RBBINode  *thisRule       = fNodeStack[fNodeStackPtr];

             RBBINode  *endNode        = pushNewNode(RBBINode::endMark);
             RBBINode  *catNode        = pushNewNode(RBBINode::opCat);
             RBBINode  *endNode        = pushNewNode(RBBINode::endMark);
             RBBINode  *catNode        = pushNewNode(RBBINode::opCat);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             fNodeStackPtr -= 2;
             catNode->fLeftChild       = thisRule;
             catNode->fRightChild      = endNode;
             fNodeStack[fNodeStackPtr] = catNode;
             endNode->fVal             = fRuleNum;
             endNode->fLookAheadEnd    = TRUE;
             fNodeStackPtr -= 2;
             catNode->fLeftChild       = thisRule;
             catNode->fRightChild      = endNode;
             fNodeStack[fNodeStackPtr] = catNode;
             endNode->fVal             = fRuleNum;
             endNode->fLookAheadEnd    = TRUE;

+            thisRule                  = catNode;
+
+            // TODO: Disable chaining out of look-ahead (hard break) rules.
+            //   The break on rule match is forced, so there is no point in building up
+            //   the state table to chain into another rule for a longer match.
+        }
+
+        // Mark this node as being the root of a rule.
+        thisRule->fRuleRoot = TRUE;
+
+        // Flag if chaining into this rule is wanted.
+        //    
+        if (fRB->fChainRules &&         // If rule chaining is enabled globally via !!chain
+                !fNoChainInRule) {      //     and no '^' chain-in inhibit was on this rule
+            thisRule->fChainIn = TRUE;

         }
 
         }
 

+

         // All rule expressions are ORed together.
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.
         //
         // All rule expressions are ORed together.
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.
         //

-        // Forward and reverse rules are collected separately.  Or this rule into
-        //  the appropriate group of them.
+        // Each of the four sets of rules are collected separately.
+        //  (forward, reverse, safe_forward, safe_reverse)
+        //  OR this rule into the appropriate group of them.

         //
         //

-        RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : &fRB->fForwardTree);
+        RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree);

 
         if (*destRules != NULL) {
             // This is not the first rule encounted.
 
         if (*destRules != NULL) {
             // This is not the first rule encounted.


@@ -331,9 +380,12 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
             // with the current rule expression (on the Node Stack)
             //  with the resulting OR expression going to *destRules
             //
             // with the current rule expression (on the Node Stack)
             //  with the resulting OR expression going to *destRules
             //

-            RBBINode  *thisRule    = fNodeStack[fNodeStackPtr];
+                       thisRule    = fNodeStack[fNodeStackPtr];

             RBBINode  *prevRules   = *destRules;
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);
             RBBINode  *prevRules   = *destRules;
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             orNode->fLeftChild     = prevRules;
             prevRules->fParent     = orNode;
             orNode->fRightChild    = thisRule;
             orNode->fLeftChild     = prevRules;
             prevRules->fParent     = orNode;
             orNode->fRightChild    = thisRule;


@@ -348,6 +400,7 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         }
         fReverseRule   = FALSE;   // in preparation for the next rule.
         fLookAheadRule = FALSE;
         }
         fReverseRule   = FALSE;   // in preparation for the next rule.
         fLookAheadRule = FALSE;

+        fNoChainInRule = FALSE;

         fNodeStackPtr  = 0;
         }
         break;
         fNodeStackPtr  = 0;
         }
         break;


@@ -374,6 +427,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *plusNode    = pushNewNode(RBBINode::opPlus);
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *plusNode    = pushNewNode(RBBINode::opPlus);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             plusNode->fLeftChild   = operandNode;
             operandNode->fParent   = plusNode;
         }
             plusNode->fLeftChild   = operandNode;
             operandNode->fParent   = plusNode;
         }


@@ -383,6 +439,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *qNode       = pushNewNode(RBBINode::opQuestion);
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *qNode       = pushNewNode(RBBINode::opQuestion);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             qNode->fLeftChild      = operandNode;
             operandNode->fParent   = qNode;
         }
             qNode->fLeftChild      = operandNode;
             operandNode->fParent   = qNode;
         }


@@ -392,6 +451,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *starNode    = pushNewNode(RBBINode::opStar);
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *starNode    = pushNewNode(RBBINode::opStar);

+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }

             starNode->fLeftChild   = operandNode;
             operandNode->fParent   = starNode;
         }
             starNode->fLeftChild   = operandNode;
             operandNode->fParent   = starNode;
         }


@@ -405,7 +467,10 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);

-            findSetFor(fC.fChar, n);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
+            findSetFor(UnicodeString(fC.fChar), n);

             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);


@@ -416,17 +481,22 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);

-            findSetFor(kAny, n);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
+            findSetFor(UnicodeString(TRUE, kAny, 3), n);

             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             break;
         }
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             break;
         }

-        break;

 
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.
         n = pushNewNode(RBBINode::lookAhead);
 
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.
         n = pushNewNode(RBBINode::lookAhead);

+        if (U_FAILURE(*fRB->fStatus)) {
+            break;
+        }

         n->fVal      = fRuleNum;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;
         n->fVal      = fRuleNum;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;


@@ -438,6 +508,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
     case doStartTagValue:
         // Scanned a '{', the opening delimiter for a tag value within a rule.
         n = pushNewNode(RBBINode::tag);
     case doStartTagValue:
         // Scanned a '{', the opening delimiter for a tag value within a rule.
         n = pushNewNode(RBBINode::tag);

+        if (U_FAILURE(*fRB->fStatus)) {
+            break;
+        }

         n->fVal      = 0;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;
         n->fVal      = 0;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;


@@ -459,7 +532,42 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
         break;
 
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
         break;
 

+    case doTagExpectedError:
+        error(U_BRK_MALFORMED_RULE_TAG);
+        returnVal = FALSE;
+        break;

 
 

+    case doOptionStart:
+        // Scanning a !!option.   At the start of string.
+        fOptionStart = fScanIndex;
+        break;
+
+    case doOptionEnd:
+        {
+            UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart);
+            if (opt == UNICODE_STRING("chain", 5)) {
+                fRB->fChainRules = TRUE;
+            } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) {
+                fRB->fLBCMNoChain = TRUE;
+            } else if (opt == UNICODE_STRING("forward", 7)) {
+                fRB->fDefaultTree   = &fRB->fForwardTree;
+            } else if (opt == UNICODE_STRING("reverse", 7)) {
+                fRB->fDefaultTree   = &fRB->fReverseTree;
+            } else if (opt == UNICODE_STRING("safe_forward", 12)) {
+                fRB->fDefaultTree   = &fRB->fSafeFwdTree;
+            } else if (opt == UNICODE_STRING("safe_reverse", 12)) {
+                fRB->fDefaultTree   = &fRB->fSafeRevTree;
+            } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
+                fRB->fLookAheadHardBreak = TRUE;
+            } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) {
+                fRuleSets[kRuleSet_rule_char-128].clear();
+            } else if (opt == UNICODE_STRING("unquoted_literals",  17)) {
+                fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus);
+            } else {
+                error(U_BRK_UNRECOGNIZED_OPTION);
+            }
+        }
+        break;

 
     case doReverseDir:
         fReverseRule = TRUE;
 
     case doReverseDir:
         fReverseRule = TRUE;


@@ -517,32 +625,34 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         returnVal = FALSE;
         break;
     }
         returnVal = FALSE;
         break;
     }

-    return returnVal;
+    return returnVal && U_SUCCESS(*fRB->fStatus);

 }
 
 
 
 
 }
 
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Error         Report a rule parse error.
 //                Only report it if no previous error has been recorded.
 //
 //
 //  Error         Report a rule parse error.
 //                Only report it if no previous error has been recorded.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::error(UErrorCode e) {
     if (U_SUCCESS(*fRB->fStatus)) {
         *fRB->fStatus = e;
 void RBBIRuleScanner::error(UErrorCode e) {
     if (U_SUCCESS(*fRB->fStatus)) {
         *fRB->fStatus = e;

-        fRB->fParseError->line  = fLineNum;
-        fRB->fParseError->offset = fCharNum;
-        fRB->fParseError->preContext[0] = 0;
-        fRB->fParseError->preContext[0] = 0;
+        if (fRB->fParseError) {
+            fRB->fParseError->line  = fLineNum;
+            fRB->fParseError->offset = fCharNum;
+            fRB->fParseError->preContext[0] = 0;
+            fRB->fParseError->postContext[0] = 0;
+        }

     }
 }
 
 
 
 
     }
 }
 
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  fixOpStack   The parse stack holds partially assembled chunks of the parse tree.
 //               An entry on the stack may be as small as a single setRef node,
 //
 //  fixOpStack   The parse stack holds partially assembled chunks of the parse tree.
 //               An entry on the stack may be as small as a single setRef node,


@@ -556,14 +666,14 @@ void RBBIRuleScanner::error(UErrorCode e) {
 //               the precedence of the current operator, binds the operand left,
 //               to the previously encountered operator.
 //
 //               the precedence of the current operator, binds the operand left,
 //               to the previously encountered operator.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
     RBBINode *n;
     // printNodeStack("entering fixOpStack()");
     for (;;) {
         n = fNodeStack[fNodeStackPtr-1];   // an operator node
         if (n->fPrecedence == 0) {
 void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
     RBBINode *n;
     // printNodeStack("entering fixOpStack()");
     for (;;) {
         n = fNodeStack[fNodeStackPtr-1];   // an operator node
         if (n->fPrecedence == 0) {

-            RBBIDebugPrintf("RBBIRuleScanner::fixOpStack, bad operator node\n");
+            RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node");

             error(U_BRK_INTERNAL_ERROR);
             return;
         }
             error(U_BRK_INTERNAL_ERROR);
             return;
         }


@@ -603,7 +713,7 @@ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
 
 
 
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //   findSetFor    given a UnicodeString,
 //                  - find the corresponding Unicode Set  (uset node)
 //
 //   findSetFor    given a UnicodeString,
 //                  - find the corresponding Unicode Set  (uset node)


@@ -618,7 +728,7 @@ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
 //                    just one element which is the char in question.
 //                 If the string is "any", return a set containing all chars.
 //
 //                    just one element which is the char in question.
 //                 If the string is "any", return a set containing all chars.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) {
 
     RBBISetTableEl   *el;
 void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) {
 
     RBBISetTableEl   *el;


@@ -652,6 +762,10 @@ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, Unicode
     // This new uset node becomes the child of the caller's setReference node.
     //
     RBBINode *usetNode    = new RBBINode(RBBINode::uset);
     // This new uset node becomes the child of the caller's setReference node.
     //
     RBBINode *usetNode    = new RBBINode(RBBINode::uset);

+    if (usetNode == NULL) {
+        error(U_MEMORY_ALLOCATION_ERROR);
+        return;
+    }

     usetNode->fInputSet   = setToAdopt;
     usetNode->fParent     = node;
     node->fLeftChild      = usetNode;
     usetNode->fInputSet   = setToAdopt;
     usetNode->fParent     = node;
     node->fLeftChild      = usetNode;


@@ -670,6 +784,14 @@ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, Unicode
     el      = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl));
     UnicodeString *tkey = new UnicodeString(s);
     if (tkey == NULL || el == NULL || setToAdopt == NULL) {
     el      = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl));
     UnicodeString *tkey = new UnicodeString(s);
     if (tkey == NULL || el == NULL || setToAdopt == NULL) {

+        // Delete to avoid memory leak
+        delete tkey;
+        tkey = NULL;
+        uprv_free(el);
+        el = NULL;
+        delete setToAdopt;
+        setToAdopt = NULL;
+

         error(U_MEMORY_ALLOCATION_ERROR);
         return;
     }
         error(U_MEMORY_ALLOCATION_ERROR);
         return;
     }


@@ -698,40 +820,37 @@ static const UChar      chLParen    = 0x28;
 static const UChar      chRParen    = 0x29;
 
 
 static const UChar      chRParen    = 0x29;
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //

-//  stripRules    Return a rules string without unnecessary
-//                characters.
+//  stripRules    Return a rules string without extra spaces.
+//                (Comments are removed separately, during rule parsing.)

 //
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
     UnicodeString strippedRules;
 UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
     UnicodeString strippedRules;

-    int rulesLength = rules.length();
-    for (int idx = 0; idx < rulesLength; ) {
-        UChar ch = rules[idx++];
-        if (ch == chPound) {
-            while (idx < rulesLength
-                && ch != chCR && ch != chLF && ch != chNEL)
-            {
-                ch = rules[idx++];
-            }
-        }
-        if (!u_isISOControl(ch)) {
-            strippedRules.append(ch);
+    int32_t rulesLength = rules.length();
+    bool skippingSpaces = false;
+
+    for (int32_t idx=0; idx<rulesLength; idx = rules.moveIndex32(idx, 1)) {
+        UChar32 cp = rules.char32At(idx);
+        bool whiteSpace = u_hasBinaryProperty(cp, UCHAR_PATTERN_WHITE_SPACE);
+        if (skippingSpaces && whiteSpace) {
+            continue;

         }
         }

+        strippedRules.append(cp);
+        skippingSpaces = whiteSpace;

     }
     }

-    // strippedRules = strippedRules.unescape();

     return strippedRules;
 }
 
 
     return strippedRules;
 }
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  nextCharLL    Low Level Next Char from rule input source.
 //                Get a char from the input character iterator,
 //                keep track of input position for error reporting.
 //
 //
 //  nextCharLL    Low Level Next Char from rule input source.
 //                Get a char from the input character iterator,
 //                keep track of input position for error reporting.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 UChar32  RBBIRuleScanner::nextCharLL() {
     UChar32  ch;
 
 UChar32  RBBIRuleScanner::nextCharLL() {
     UChar32  ch;
 


@@ -744,7 +863,7 @@ UChar32  RBBIRuleScanner::nextCharLL() {
     if (ch == chCR ||
         ch == chNEL ||
         ch == chLS   ||
     if (ch == chCR ||
         ch == chNEL ||
         ch == chLS   ||

-        ch == chLF && fLastChar != chCR) {
+        (ch == chLF && fLastChar != chCR)) {

         // Character is starting a new line.  Bump up the line number, and
         //  reset the column to 0.
         fLineNum++;
         // Character is starting a new line.  Bump up the line number, and
         //  reset the column to 0.
         fLineNum++;


@@ -766,13 +885,13 @@ UChar32  RBBIRuleScanner::nextCharLL() {
 }
 
 
 }
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //   nextChar     for rules scanning.  At this level, we handle stripping
 //                out comments and processing backslash character escapes.
 //                The rest of the rules grammar is handled at the next level up.
 //
 //
 //   nextChar     for rules scanning.  At this level, we handle stripping
 //                out comments and processing backslash character escapes.
 //                The rest of the rules grammar is handled at the next level up.
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
 
     // Unicode Character constants needed for the processing done by nextChar(),
 void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
 
     // Unicode Character constants needed for the processing done by nextChar(),


@@ -820,6 +939,7 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
             //  It will be treated as white-space, and serves to break up anything
             //    that might otherwise incorrectly clump together with a comment in
             //    the middle (a variable name, for example.)
             //  It will be treated as white-space, and serves to break up anything
             //    that might otherwise incorrectly clump together with a comment in
             //    the middle (a variable name, for example.)

+            int32_t commentStart = fScanIndex;

             for (;;) {
                 c.fChar = nextCharLL();
                 if (c.fChar == (UChar32)-1 ||  // EOF
             for (;;) {
                 c.fChar = nextCharLL();
                 if (c.fChar == (UChar32)-1 ||  // EOF


@@ -828,6 +948,9 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
                     c.fChar == chNEL    ||
                     c.fChar == chLS)       {break;}
             }
                     c.fChar == chNEL    ||
                     c.fChar == chLS)       {break;}
             }

+            for (int32_t i=commentStart; i<fNextIndex-1; ++i) {
+                fRB->fStrippedRules.setCharAt(i, u' ');
+            }

         }
         if (c.fChar == (UChar32)-1) {
             return;
         }
         if (c.fChar == (UChar32)-1) {
             return;


@@ -850,14 +973,14 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
     // putc(c.fChar, stdout);
 }
 
     // putc(c.fChar, stdout);
 }
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Parse RBBI rules.   The state machine for rules parsing is here.
 //
 //  Parse RBBI rules.   The state machine for rules parsing is here.

-//                      The state tables are hand-written in the file TODO.txt,
+//                      The state tables are hand-written in the file rbbirpt.txt,

 //                      and converted to the form used here by a perl
 //                      script rbbicst.pl
 //
 //                      and converted to the form used here by a perl
 //                      script rbbicst.pl
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::parse() {
     uint16_t                state;
     const RBBIRuleTableEl  *tableEl;
 void RBBIRuleScanner::parse() {
     uint16_t                state;
     const RBBIRuleTableEl  *tableEl;


@@ -896,13 +1019,17 @@ void RBBIRuleScanner::parse() {
         //    the search will stop there, if not before.
         //
         tableEl = &gRuleParseStateTable[state];
         //    the search will stop there, if not before.
         //
         tableEl = &gRuleParseStateTable[state];

-        if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) {
-            RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d)    state=%s ",
-                fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]);
-        }
+        #ifdef RBBI_DEBUG
+            if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) {
+                RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d)    state=%s ",
+                    fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]);
+            }
+        #endif

 
         for (;;) {
 
         for (;;) {

-            if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+            #ifdef RBBI_DEBUG
+                if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);}
+            #endif

             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and
                 //   the input character is not escaped, and
             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and
                 //   the input character is not escaped, and


@@ -930,8 +1057,8 @@ void RBBIRuleScanner::parse() {
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF

-                UnicodeSet *uniset = fRuleSets[tableEl->fCharClass-128];
-                if (uniset->contains(fC.fChar)) {
+                U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets));
+                if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {

                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.
                     break;
                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.
                     break;


@@ -941,13 +1068,13 @@ void RBBIRuleScanner::parse() {
             // No match on this row, advance to the next  row for this state,
             tableEl++;
         }
             // No match on this row, advance to the next  row for this state,
             tableEl++;
         }

-        if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("\n");}
+        if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");}

 
         //
         // We've found the row of the state table that matches the current input
         //   character from the rules string.
         // Perform any action specified  by this row in the state table.
 
         //
         // We've found the row of the state table that matches the current input
         //   character from the rules string.
         // Perform any action specified  by this row in the state table.

-        if (doParseActions((EParseAction)tableEl->fAction) == FALSE) {
+        if (doParseActions((int32_t)tableEl->fAction) == FALSE) {

             // Break out of the state machine loop if the
             //   the action signalled some kind of error, or
             //   the action was to exit, occurs on normal end-of-rules-input.
             // Break out of the state machine loop if the
             //   the action signalled some kind of error, or
             //   the action was to exit, occurs on normal end-of-rules-input.


@@ -958,7 +1085,7 @@ void RBBIRuleScanner::parse() {
             fStackPtr++;
             if (fStackPtr >= kStackSize) {
                 error(U_BRK_INTERNAL_ERROR);
             fStackPtr++;
             if (fStackPtr >= kStackSize) {
                 error(U_BRK_INTERNAL_ERROR);

-                RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack overflow.\n");
+                RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow.");

                 fStackPtr--;
             }
             fStack[fStackPtr] = tableEl->fPushState;
                 fStackPtr--;
             }
             fStack[fStackPtr] = tableEl->fPushState;


@@ -977,71 +1104,77 @@ void RBBIRuleScanner::parse() {
             fStackPtr--;
             if (fStackPtr < 0) {
                 error(U_BRK_INTERNAL_ERROR);
             fStackPtr--;
             if (fStackPtr < 0) {
                 error(U_BRK_INTERNAL_ERROR);

-                RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack underflow.\n");
+                RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow.");

                 fStackPtr++;
             }
         }
 
     }
 
                 fStackPtr++;
             }
         }
 
     }
 

+    if (U_FAILURE(*fRB->fStatus)) {
+        return;
+    }
+    
+    // If there are no forward rules set an error.

     //
     //

-    // If there were NO user specified reverse rules, set up the equivalent of ".*;"
-    //
-    if (fRB->fReverseTree == NULL) {
-        fRB->fReverseTree  = pushNewNode(RBBINode::opStar);
-        RBBINode  *operand = pushNewNode(RBBINode::setRef);
-        findSetFor(kAny, operand);
-        fRB->fReverseTree->fLeftChild = operand;
-        operand->fParent              = fRB->fReverseTree;
-        fNodeStackPtr -= 2;
+    if (fRB->fForwardTree == NULL) {
+        error(U_BRK_RULE_SYNTAX);
+        return;

     }
 
     }
 

-

     //
     // Parsing of the input RBBI rules is complete.
     // We now have a parse tree for the rule expressions
     // and a list of all UnicodeSets that are referenced.
     //
     //
     // Parsing of the input RBBI rules is complete.
     // We now have a parse tree for the rule expressions
     // and a list of all UnicodeSets that are referenced.
     //

-    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->print();}
-    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree"))
-    {
+#ifdef RBBI_DEBUG
+    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();}
+    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) {

         RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");
         RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");

-        fRB->fForwardTree->printTree();
+        RBBINode::printTree(fRB->fForwardTree, TRUE);

         RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");
         RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");

-        fRB->fReverseTree->printTree();
+        RBBINode::printTree(fRB->fReverseTree, TRUE);
+        RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n");
+        RBBINode::printTree(fRB->fSafeFwdTree, TRUE);
+        RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n");
+        RBBINode::printTree(fRB->fSafeRevTree, TRUE);

     }
     }

-
+#endif

 }
 
 
 }
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  printNodeStack     for debugging...
 //
 //
 //  printNodeStack     for debugging...
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
+#ifdef RBBI_DEBUG

 void RBBIRuleScanner::printNodeStack(const char *title) {
     int i;
     RBBIDebugPrintf("%s.  Dumping node stack...\n", title);
 void RBBIRuleScanner::printNodeStack(const char *title) {
     int i;
     RBBIDebugPrintf("%s.  Dumping node stack...\n", title);

-    for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree();}
+    for (i=fNodeStackPtr; i>0; i--) {RBBINode::printTree(fNodeStack[i], TRUE);}

 }
 }

+#endif

 
 
 
 
 
 
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  pushNewNode   create a new RBBINode of the specified type and push it
 //                onto the stack of nodes.
 //
 //
 //  pushNewNode   create a new RBBINode of the specified type and push it
 //                onto the stack of nodes.
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {

-    fNodeStackPtr++;
-    if (fNodeStackPtr >= kStackSize) {
-        error(U_BRK_INTERNAL_ERROR);
-        RBBIDebugPrintf("RBBIRuleScanner::pushNewNode - stack overflow.\n");
-        *fRB->fStatus = U_BRK_INTERNAL_ERROR;
+    if (U_FAILURE(*fRB->fStatus)) {
+        return NULL;
+    }
+    if (fNodeStackPtr >= kStackSize - 1) {
+        error(U_BRK_RULE_SYNTAX);
+        RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow.");

         return NULL;
     }
         return NULL;
     }

+    fNodeStackPtr++;

     fNodeStack[fNodeStackPtr] = new RBBINode(t);
     if (fNodeStack[fNodeStackPtr] == NULL) {
         *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR;
     fNodeStack[fNodeStackPtr] = new RBBINode(t);
     if (fNodeStack[fNodeStackPtr] == NULL) {
         *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR;


@@ -1051,7 +1184,7 @@ RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
 
 
 
 
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  scanSet    Construct a UnicodeSet from the text at the current scan
 //             position.  Advance the scan position to the first character
 //
 //  scanSet    Construct a UnicodeSet from the text at the current scan
 //             position.  Advance the scan position to the first character


@@ -1064,7 +1197,7 @@ RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
 //             that controls rule parsing.  UnicodeSets, however, are parsed by
 //             the UnicodeSet constructor, not by the RBBI rule parser.
 //
 //             that controls rule parsing.  UnicodeSets, however, are parsed by
 //             the UnicodeSet constructor, not by the RBBI rule parser.
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::scanSet() {
     UnicodeSet    *uset;
     ParsePosition  pos;
 void RBBIRuleScanner::scanSet() {
     UnicodeSet    *uset;
     ParsePosition  pos;


@@ -1078,13 +1211,18 @@ void RBBIRuleScanner::scanSet() {
     pos.setIndex(fScanIndex);
     startPos = fScanIndex;
     UErrorCode localStatus = U_ZERO_ERROR;
     pos.setIndex(fScanIndex);
     startPos = fScanIndex;
     UErrorCode localStatus = U_ZERO_ERROR;

-    uset = new UnicodeSet(fRB->fRules, pos,
-                         *fSymbolTable,
-                         localStatus);
+    uset = new UnicodeSet();
+    if (uset == NULL) {
+        localStatus = U_MEMORY_ALLOCATION_ERROR;
+    } else {
+        uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus);
+    }

     if (U_FAILURE(localStatus)) {
         //  TODO:  Get more accurate position of the error from UnicodeSet's return info.
         //         UnicodeSet appears to not be reporting correctly at this time.
     if (U_FAILURE(localStatus)) {
         //  TODO:  Get more accurate position of the error from UnicodeSet's return info.
         //         UnicodeSet appears to not be reporting correctly at this time.

-        RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex());
+        #ifdef RBBI_DEBUG
+            RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex());
+        #endif

         error(localStatus);
         delete uset;
         return;
         error(localStatus);
         delete uset;
         return;


@@ -1092,7 +1230,8 @@ void RBBIRuleScanner::scanSet() {
 
     // Verify that the set contains at least one code point.
     //
 
     // Verify that the set contains at least one code point.
     //

-    if (uset->charAt(0) == -1) {
+    U_ASSERT(uset!=NULL);
+    if (uset->isEmpty()) {

         // This set is empty.
         //  Make it an error, because it almost certainly is not what the user wanted.
         //  Also, avoids having to think about corner cases in the tree manipulation code
         // This set is empty.
         //  Make it an error, because it almost certainly is not what the user wanted.
         //  Also, avoids having to think about corner cases in the tree manipulation code


@@ -1118,6 +1257,9 @@ void RBBIRuleScanner::scanSet() {
         RBBINode         *n;
 
         n = pushNewNode(RBBINode::setRef);
         RBBINode         *n;
 
         n = pushNewNode(RBBINode::setRef);

+        if (U_FAILURE(*fRB->fStatus)) {
+            return;
+        }

         n->fFirstPos = startPos;
         n->fLastPos  = fNextIndex;
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
         n->fFirstPos = startPos;
         n->fLastPos  = fNextIndex;
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);