ICU-511.35.tar.gz

[apple/icu.git] / icuSources / common / rbbiscan.cpp

diff --git a/icuSources/common/rbbiscan.cpp b/icuSources/common/rbbiscan.cpp
index c5f15635fa20bb2489777d12191f46c641827c7e..0113c08ed377d25afca0912c12c261736da6163b 100644 (file)
--- a/icuSources/common/rbbiscan.cpp
+++ b/icuSources/common/rbbiscan.cpp
@@ -2,7 +2,7 @@
 //
 //  file:  rbbiscan.cpp
 //
 //
 //  file:  rbbiscan.cpp
 //

-//  Copyright (C) 2002-2003, International Business Machines Corporation and others.
+//  Copyright (C) 2002-2012, International Business Machines Corporation and others.

 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for
 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for


@@ -23,7 +23,6 @@
 #include "unicode/uchriter.h"
 #include "unicode/parsepos.h"
 #include "unicode/parseerr.h"
 #include "unicode/uchriter.h"
 #include "unicode/parsepos.h"
 #include "unicode/parseerr.h"

-#include "uprops.h"

 #include "cmemory.h"
 #include "cstring.h"
 
 #include "cmemory.h"
 #include "cstring.h"
 


@@ -32,11 +31,13 @@
 #include "rbbirb.h"
 #include "rbbinode.h"
 #include "rbbiscan.h"
 #include "rbbirb.h"
 #include "rbbinode.h"
 #include "rbbiscan.h"

+#include "rbbitblb.h"

 
 #include "uassert.h"
 
 
 #include "uassert.h"
 

+#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))

 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.
 //               (Initialized with hex values for portability to EBCDIC based machines.
 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.
 //               (Initialized with hex values for portability to EBCDIC based machines.


@@ -45,7 +46,7 @@
 //              The sets are referred to by name in the rbbirpt.txt, which is the
 //              source form of the state transition table for the RBBI rule parser.
 //
 //              The sets are referred to by name in the rbbirpt.txt, which is the
 //              source form of the state transition table for the RBBI rule parser.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 static const UChar gRuleSet_rule_char_pattern[]       = {
  //   [    ^      [    \     p     {      Z     }     \     u    0      0    2      0
     0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
 static const UChar gRuleSet_rule_char_pattern[]       = {
  //   [    ^      [    \     p     {      Z     }     \     u    0      0    2      0
     0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,


@@ -70,8 +71,8 @@ static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00};  // "any"
 
 
 U_CDECL_BEGIN
 
 
 U_CDECL_BEGIN

-static void  U_EXPORT2 U_CALLCONV RBBISetTable_deleter(void *p) {
-    RBBISetTableEl *px = (RBBISetTableEl *)p;
+static void U_CALLCONV RBBISetTable_deleter(void *p) {
+    icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p;

     delete px->key;
     // Note:  px->val is owned by the linked list "fSetsListHead" in scanner.
     //        Don't delete the value nodes here.
     delete px->key;
     // Note:  px->val is owned by the linked list "fSetsListHead" in scanner.
     //        Don't delete the value nodes here.


@@ -81,11 +82,11 @@ U_CDECL_END
 
 U_NAMESPACE_BEGIN
 
 
 U_NAMESPACE_BEGIN
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Constructor.
 //
 //
 //  Constructor.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;
 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;


@@ -95,11 +96,6 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     fRuleNum            = 0;
     fNodeStack[0]       = NULL;
 
     fRuleNum            = 0;
     fNodeStack[0]       = NULL;
 

-    fRuleSets[kRuleSet_rule_char-128]       = NULL;
-    fRuleSets[kRuleSet_white_space-128]     = NULL;
-    fRuleSets[kRuleSet_name_char-128]       = NULL;
-    fRuleSets[kRuleSet_name_start_char-128] = NULL;
-    fRuleSets[kRuleSet_digit_char-128]      = NULL;

     fSymbolTable                            = NULL;
     fSetTable                               = NULL;
 
     fSymbolTable                            = NULL;
     fSetTable                               = NULL;
 


@@ -113,6 +109,8 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     fCharNum    = 0;
     fQuoteMode  = FALSE;
 
     fCharNum    = 0;
     fQuoteMode  = FALSE;
 

+    // Do not check status until after all critical fields are sufficiently initialized
+    //   that the destructor can run cleanly.

     if (U_FAILURE(*rb->fStatus)) {
         return;
     }
     if (U_FAILURE(*rb->fStatus)) {
         return;
     }


@@ -123,14 +121,20 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     //            all instances of RBBIRuleScanners.  BUT this is quite a bit simpler,
     //            and the time to build these few sets should be small compared to a
     //            full break iterator build.
     //            all instances of RBBIRuleScanners.  BUT this is quite a bit simpler,
     //            and the time to build these few sets should be small compared to a
     //            full break iterator build.

-    fRuleSets[kRuleSet_rule_char-128]       = new UnicodeSet(gRuleSet_rule_char_pattern,       *rb->fStatus);
-    fRuleSets[kRuleSet_white_space-128]     = (UnicodeSet*) uprv_openRuleWhiteSpaceSet(rb->fStatus);
-    fRuleSets[kRuleSet_name_char-128]       = new UnicodeSet(gRuleSet_name_char_pattern,       *rb->fStatus);
-    fRuleSets[kRuleSet_name_start_char-128] = new UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus);
-    fRuleSets[kRuleSet_digit_char-128]      = new UnicodeSet(gRuleSet_digit_char_pattern,      *rb->fStatus);
+    fRuleSets[kRuleSet_rule_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern),       *rb->fStatus);
+    // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:]
+    fRuleSets[kRuleSet_white_space-128].
+        add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029);
+    fRuleSets[kRuleSet_name_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern),       *rb->fStatus);
+    fRuleSets[kRuleSet_name_start_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus);
+    fRuleSets[kRuleSet_digit_char-128]
+        = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern),      *rb->fStatus);

     if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
         // This case happens if ICU's data is missing.  UnicodeSet tries to look up property
     if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
         // This case happens if ICU's data is missing.  UnicodeSet tries to look up property

-        //   names from the init string, can't find them, and claims an illegal arguement.
+        //   names from the init string, can't find them, and claims an illegal argument.

         //   Change the error so that the actual problem will be clearer to users.
         *rb->fStatus = U_BRK_INIT_ERROR;
     }
         //   Change the error so that the actual problem will be clearer to users.
         *rb->fStatus = U_BRK_INIT_ERROR;
     }


@@ -139,24 +143,25 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
     }
 
     fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus);
     }
 
     fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus);

-    fSetTable    = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, rb->fStatus);
+    if (fSymbolTable == NULL) {
+        *rb->fStatus = U_MEMORY_ALLOCATION_ERROR;
+        return;
+    }
+    fSetTable    = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus);
+    if (U_FAILURE(*rb->fStatus)) {
+        return;
+    }

     uhash_setValueDeleter(fSetTable, RBBISetTable_deleter);
 }
 
 
 
     uhash_setValueDeleter(fSetTable, RBBISetTable_deleter);
 }
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Destructor
 //
 //
 //  Destructor
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 RBBIRuleScanner::~RBBIRuleScanner() {
 RBBIRuleScanner::~RBBIRuleScanner() {

-    delete fRuleSets[kRuleSet_rule_char-128];
-    delete fRuleSets[kRuleSet_white_space-128];
-    delete fRuleSets[kRuleSet_name_char-128];
-    delete fRuleSets[kRuleSet_name_start_char-128];
-    delete fRuleSets[kRuleSet_digit_char-128];
-

     delete fSymbolTable;
     if (fSetTable != NULL) {
          uhash_close(fSetTable);
     delete fSymbolTable;
     if (fSetTable != NULL) {
          uhash_close(fSetTable);


@@ -175,7 +180,7 @@ RBBIRuleScanner::~RBBIRuleScanner() {
 
 }
 
 
 }
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  doParseAction        Do some action during rule parsing.
 //                       Called by the parse state machine.
 //
 //  doParseAction        Do some action during rule parsing.
 //                       Called by the parse state machine.


@@ -188,14 +193,14 @@ RBBIRuleScanner::~RBBIRuleScanner() {
 //                              in some compilers, while at the same time avoiding multiple
 //                              definitions problems.  I'm sure that there's a better way.
 //
 //                              in some compilers, while at the same time avoiding multiple
 //                              definitions problems.  I'm sure that there's a better way.
 //

-//----------------------------------------------------------------------------------------
-UBool RBBIRuleScanner::doParseActions(EParseAction action)
+//------------------------------------------------------------------------------
+UBool RBBIRuleScanner::doParseActions(int32_t action)

 {
     RBBINode *n       = NULL;
 
     UBool   returnVal = TRUE;
 
 {
     RBBINode *n       = NULL;
 
     UBool   returnVal = TRUE;
 

-    switch ((RBBI_RuleParseAction)action) {
+    switch (action) {

 
     case doExprStart:
         pushNewNode(RBBINode::opStart);
 
     case doExprStart:
         pushNewNode(RBBINode::opStart);


@@ -286,6 +291,13 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
 
             // Make a symbol table entry for the $variableRef node.
             fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus);
 
             // Make a symbol table entry for the $variableRef node.
             fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus);

+            if (U_FAILURE(*fRB->fStatus)) {
+                // This is a round-about way to get the parse position set
+                //  so that duplicate symbols error messages include a line number.
+                UErrorCode t = *fRB->fStatus;
+                *fRB->fStatus = U_ZERO_ERROR;
+                error(t);
+            }

 
             // Clean up the stack.
             delete startExprNode;
 
             // Clean up the stack.
             delete startExprNode;


@@ -299,7 +311,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         if (U_FAILURE(*fRB->fStatus)) {       //   parse tree rooted in TOS node.
             break;
         }
         if (U_FAILURE(*fRB->fStatus)) {       //   parse tree rooted in TOS node.
             break;
         }

+#ifdef RBBI_DEBUG

         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}

+#endif

         U_ASSERT(fNodeStackPtr == 1);
 
         // If this rule includes a look-ahead '/', add a endMark node to the
         U_ASSERT(fNodeStackPtr == 1);
 
         // If this rule includes a look-ahead '/', add a endMark node to the


@@ -320,10 +334,11 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.
         //
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.
         //

-        // Forward and reverse rules are collected separately.  Or this rule into
-        //  the appropriate group of them.
+        // Each of the four sets of rules are collected separately.
+        //  (forward, reverse, safe_forward, safe_reverse)
+        //  OR this rule into the appropriate group of them.

         //
         //

-        RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : &fRB->fForwardTree);
+        RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : fRB->fDefaultTree);

 
         if (*destRules != NULL) {
             // This is not the first rule encounted.
 
         if (*destRules != NULL) {
             // This is not the first rule encounted.


@@ -405,7 +420,7 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);

-            findSetFor(fC.fChar, n);
+            findSetFor(UnicodeString(fC.fChar), n);

             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);


@@ -416,13 +431,12 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);

-            findSetFor(kAny, n);
+            findSetFor(UnicodeString(TRUE, kAny, 3), n);

             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             break;
         }
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
             fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
             break;
         }

-        break;

 
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.
 
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.


@@ -459,7 +473,38 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
         break;
 
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
         break;
 

+    case doTagExpectedError:
+        error(U_BRK_MALFORMED_RULE_TAG);
+        returnVal = FALSE;
+        break;

 
 

+    case doOptionStart:
+        // Scanning a !!option.   At the start of string.
+        fOptionStart = fScanIndex;
+        break;
+
+    case doOptionEnd:
+        {
+            UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart);
+            if (opt == UNICODE_STRING("chain", 5)) {
+                fRB->fChainRules = TRUE;
+            } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) {
+                fRB->fLBCMNoChain = TRUE;
+            } else if (opt == UNICODE_STRING("forward", 7)) {
+                fRB->fDefaultTree   = &fRB->fForwardTree;
+            } else if (opt == UNICODE_STRING("reverse", 7)) {
+                fRB->fDefaultTree   = &fRB->fReverseTree;
+            } else if (opt == UNICODE_STRING("safe_forward", 12)) {
+                fRB->fDefaultTree   = &fRB->fSafeFwdTree;
+            } else if (opt == UNICODE_STRING("safe_reverse", 12)) {
+                fRB->fDefaultTree   = &fRB->fSafeRevTree;
+            } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
+                fRB->fLookAheadHardBreak = TRUE;
+            } else {
+                error(U_BRK_UNRECOGNIZED_OPTION);
+            }
+        }
+        break;

 
     case doReverseDir:
         fReverseRule = TRUE;
 
     case doReverseDir:
         fReverseRule = TRUE;


@@ -523,26 +568,28 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action)
 
 
 
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Error         Report a rule parse error.
 //                Only report it if no previous error has been recorded.
 //
 //
 //  Error         Report a rule parse error.
 //                Only report it if no previous error has been recorded.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::error(UErrorCode e) {
     if (U_SUCCESS(*fRB->fStatus)) {
         *fRB->fStatus = e;
 void RBBIRuleScanner::error(UErrorCode e) {
     if (U_SUCCESS(*fRB->fStatus)) {
         *fRB->fStatus = e;

-        fRB->fParseError->line  = fLineNum;
-        fRB->fParseError->offset = fCharNum;
-        fRB->fParseError->preContext[0] = 0;
-        fRB->fParseError->preContext[0] = 0;
+        if (fRB->fParseError) {
+            fRB->fParseError->line  = fLineNum;
+            fRB->fParseError->offset = fCharNum;
+            fRB->fParseError->preContext[0] = 0;
+            fRB->fParseError->preContext[0] = 0;
+        }

     }
 }
 
 
 
 
     }
 }
 
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  fixOpStack   The parse stack holds partially assembled chunks of the parse tree.
 //               An entry on the stack may be as small as a single setRef node,
 //
 //  fixOpStack   The parse stack holds partially assembled chunks of the parse tree.
 //               An entry on the stack may be as small as a single setRef node,


@@ -556,14 +603,14 @@ void RBBIRuleScanner::error(UErrorCode e) {
 //               the precedence of the current operator, binds the operand left,
 //               to the previously encountered operator.
 //
 //               the precedence of the current operator, binds the operand left,
 //               to the previously encountered operator.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
     RBBINode *n;
     // printNodeStack("entering fixOpStack()");
     for (;;) {
         n = fNodeStack[fNodeStackPtr-1];   // an operator node
         if (n->fPrecedence == 0) {
 void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
     RBBINode *n;
     // printNodeStack("entering fixOpStack()");
     for (;;) {
         n = fNodeStack[fNodeStackPtr-1];   // an operator node
         if (n->fPrecedence == 0) {

-            RBBIDebugPrintf("RBBIRuleScanner::fixOpStack, bad operator node\n");
+            RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node");

             error(U_BRK_INTERNAL_ERROR);
             return;
         }
             error(U_BRK_INTERNAL_ERROR);
             return;
         }


@@ -603,7 +650,7 @@ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
 
 
 
 
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //   findSetFor    given a UnicodeString,
 //                  - find the corresponding Unicode Set  (uset node)
 //
 //   findSetFor    given a UnicodeString,
 //                  - find the corresponding Unicode Set  (uset node)


@@ -618,7 +665,7 @@ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
 //                    just one element which is the char in question.
 //                 If the string is "any", return a set containing all chars.
 //
 //                    just one element which is the char in question.
 //                 If the string is "any", return a set containing all chars.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) {
 
     RBBISetTableEl   *el;
 void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) {
 
     RBBISetTableEl   *el;


@@ -652,6 +699,10 @@ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, Unicode
     // This new uset node becomes the child of the caller's setReference node.
     //
     RBBINode *usetNode    = new RBBINode(RBBINode::uset);
     // This new uset node becomes the child of the caller's setReference node.
     //
     RBBINode *usetNode    = new RBBINode(RBBINode::uset);

+    if (usetNode == NULL) {
+        error(U_MEMORY_ALLOCATION_ERROR);
+        return;
+    }

     usetNode->fInputSet   = setToAdopt;
     usetNode->fParent     = node;
     node->fLeftChild      = usetNode;
     usetNode->fInputSet   = setToAdopt;
     usetNode->fParent     = node;
     node->fLeftChild      = usetNode;


@@ -670,6 +721,14 @@ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, Unicode
     el      = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl));
     UnicodeString *tkey = new UnicodeString(s);
     if (tkey == NULL || el == NULL || setToAdopt == NULL) {
     el      = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl));
     UnicodeString *tkey = new UnicodeString(s);
     if (tkey == NULL || el == NULL || setToAdopt == NULL) {

+        // Delete to avoid memory leak
+        delete tkey;
+        tkey = NULL;
+        uprv_free(el);
+        el = NULL;
+        delete setToAdopt;
+        setToAdopt = NULL;
+

         error(U_MEMORY_ALLOCATION_ERROR);
         return;
     }
         error(U_MEMORY_ALLOCATION_ERROR);
         return;
     }


@@ -698,12 +757,12 @@ static const UChar      chLParen    = 0x28;
 static const UChar      chRParen    = 0x29;
 
 
 static const UChar      chRParen    = 0x29;
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  stripRules    Return a rules string without unnecessary
 //                characters.
 //
 //
 //  stripRules    Return a rules string without unnecessary
 //                characters.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
     UnicodeString strippedRules;
     int rulesLength = rules.length();
 UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
     UnicodeString strippedRules;
     int rulesLength = rules.length();


@@ -725,13 +784,13 @@ UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
 }
 
 
 }
 
 

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  nextCharLL    Low Level Next Char from rule input source.
 //                Get a char from the input character iterator,
 //                keep track of input position for error reporting.
 //
 //
 //  nextCharLL    Low Level Next Char from rule input source.
 //                Get a char from the input character iterator,
 //                keep track of input position for error reporting.
 //

-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 UChar32  RBBIRuleScanner::nextCharLL() {
     UChar32  ch;
 
 UChar32  RBBIRuleScanner::nextCharLL() {
     UChar32  ch;
 


@@ -744,7 +803,7 @@ UChar32  RBBIRuleScanner::nextCharLL() {
     if (ch == chCR ||
         ch == chNEL ||
         ch == chLS   ||
     if (ch == chCR ||
         ch == chNEL ||
         ch == chLS   ||

-        ch == chLF && fLastChar != chCR) {
+        (ch == chLF && fLastChar != chCR)) {

         // Character is starting a new line.  Bump up the line number, and
         //  reset the column to 0.
         fLineNum++;
         // Character is starting a new line.  Bump up the line number, and
         //  reset the column to 0.
         fLineNum++;


@@ -766,13 +825,13 @@ UChar32  RBBIRuleScanner::nextCharLL() {
 }
 
 
 }
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //   nextChar     for rules scanning.  At this level, we handle stripping
 //                out comments and processing backslash character escapes.
 //                The rest of the rules grammar is handled at the next level up.
 //
 //
 //   nextChar     for rules scanning.  At this level, we handle stripping
 //                out comments and processing backslash character escapes.
 //                The rest of the rules grammar is handled at the next level up.
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
 
     // Unicode Character constants needed for the processing done by nextChar(),
 void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
 
     // Unicode Character constants needed for the processing done by nextChar(),


@@ -850,14 +909,14 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
     // putc(c.fChar, stdout);
 }
 
     // putc(c.fChar, stdout);
 }
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  Parse RBBI rules.   The state machine for rules parsing is here.
 //
 //  Parse RBBI rules.   The state machine for rules parsing is here.

-//                      The state tables are hand-written in the file TODO.txt,
+//                      The state tables are hand-written in the file rbbirpt.txt,

 //                      and converted to the form used here by a perl
 //                      script rbbicst.pl
 //
 //                      and converted to the form used here by a perl
 //                      script rbbicst.pl
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::parse() {
     uint16_t                state;
     const RBBIRuleTableEl  *tableEl;
 void RBBIRuleScanner::parse() {
     uint16_t                state;
     const RBBIRuleTableEl  *tableEl;


@@ -896,13 +955,17 @@ void RBBIRuleScanner::parse() {
         //    the search will stop there, if not before.
         //
         tableEl = &gRuleParseStateTable[state];
         //    the search will stop there, if not before.
         //
         tableEl = &gRuleParseStateTable[state];

-        if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) {
-            RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d)    state=%s ",
-                fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]);
-        }
+        #ifdef RBBI_DEBUG
+            if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) {
+                RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d)    state=%s ",
+                    fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]);
+            }
+        #endif

 
         for (;;) {
 
         for (;;) {

-            if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+            #ifdef RBBI_DEBUG
+                if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+            #endif

             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and
                 //   the input character is not escaped, and
             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and
                 //   the input character is not escaped, and


@@ -930,8 +993,8 @@ void RBBIRuleScanner::parse() {
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF

-                UnicodeSet *uniset = fRuleSets[tableEl->fCharClass-128];
-                if (uniset->contains(fC.fChar)) {
+                U_ASSERT((tableEl->fCharClass-128) < LENGTHOF(fRuleSets));
+                if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {

                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.
                     break;
                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.
                     break;


@@ -941,13 +1004,13 @@ void RBBIRuleScanner::parse() {
             // No match on this row, advance to the next  row for this state,
             tableEl++;
         }
             // No match on this row, advance to the next  row for this state,
             tableEl++;
         }

-        if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("\n");}
+        if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");}

 
         //
         // We've found the row of the state table that matches the current input
         //   character from the rules string.
         // Perform any action specified  by this row in the state table.
 
         //
         // We've found the row of the state table that matches the current input
         //   character from the rules string.
         // Perform any action specified  by this row in the state table.

-        if (doParseActions((EParseAction)tableEl->fAction) == FALSE) {
+        if (doParseActions((int32_t)tableEl->fAction) == FALSE) {

             // Break out of the state machine loop if the
             //   the action signalled some kind of error, or
             //   the action was to exit, occurs on normal end-of-rules-input.
             // Break out of the state machine loop if the
             //   the action signalled some kind of error, or
             //   the action was to exit, occurs on normal end-of-rules-input.


@@ -958,7 +1021,7 @@ void RBBIRuleScanner::parse() {
             fStackPtr++;
             if (fStackPtr >= kStackSize) {
                 error(U_BRK_INTERNAL_ERROR);
             fStackPtr++;
             if (fStackPtr >= kStackSize) {
                 error(U_BRK_INTERNAL_ERROR);

-                RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack overflow.\n");
+                RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow.");

                 fStackPtr--;
             }
             fStack[fStackPtr] = tableEl->fPushState;
                 fStackPtr--;
             }
             fStack[fStackPtr] = tableEl->fPushState;


@@ -977,7 +1040,7 @@ void RBBIRuleScanner::parse() {
             fStackPtr--;
             if (fStackPtr < 0) {
                 error(U_BRK_INTERNAL_ERROR);
             fStackPtr--;
             if (fStackPtr < 0) {
                 error(U_BRK_INTERNAL_ERROR);

-                RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack underflow.\n");
+                RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow.");

                 fStackPtr++;
             }
         }
                 fStackPtr++;
             }
         }


@@ -990,7 +1053,7 @@ void RBBIRuleScanner::parse() {
     if (fRB->fReverseTree == NULL) {
         fRB->fReverseTree  = pushNewNode(RBBINode::opStar);
         RBBINode  *operand = pushNewNode(RBBINode::setRef);
     if (fRB->fReverseTree == NULL) {
         fRB->fReverseTree  = pushNewNode(RBBINode::opStar);
         RBBINode  *operand = pushNewNode(RBBINode::setRef);

-        findSetFor(kAny, operand);
+        findSetFor(UnicodeString(TRUE, kAny, 3), operand);

         fRB->fReverseTree->fLeftChild = operand;
         operand->fParent              = fRB->fReverseTree;
         fNodeStackPtr -= 2;
         fRB->fReverseTree->fLeftChild = operand;
         operand->fParent              = fRB->fReverseTree;
         fNodeStackPtr -= 2;


@@ -1002,43 +1065,50 @@ void RBBIRuleScanner::parse() {
     // We now have a parse tree for the rule expressions
     // and a list of all UnicodeSets that are referenced.
     //
     // We now have a parse tree for the rule expressions
     // and a list of all UnicodeSets that are referenced.
     //

-    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->print();}
+#ifdef RBBI_DEBUG
+    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();}

     if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree"))
     {
         RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");
     if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree"))
     {
         RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");

-        fRB->fForwardTree->printTree();
+        fRB->fForwardTree->printTree(TRUE);

         RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");
         RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");

-        fRB->fReverseTree->printTree();
+        fRB->fReverseTree->printTree(TRUE);
+        RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n");
+        fRB->fSafeFwdTree->printTree(TRUE);
+        RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n");
+        fRB->fSafeRevTree->printTree(TRUE);

     }
     }

-
+#endif

 }
 
 
 }
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  printNodeStack     for debugging...
 //
 //
 //  printNodeStack     for debugging...
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
+#ifdef RBBI_DEBUG

 void RBBIRuleScanner::printNodeStack(const char *title) {
     int i;
     RBBIDebugPrintf("%s.  Dumping node stack...\n", title);
 void RBBIRuleScanner::printNodeStack(const char *title) {
     int i;
     RBBIDebugPrintf("%s.  Dumping node stack...\n", title);

-    for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree();}
+    for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree(TRUE);}

 }
 }

+#endif

 
 
 
 
 
 
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  pushNewNode   create a new RBBINode of the specified type and push it
 //                onto the stack of nodes.
 //
 //
 //  pushNewNode   create a new RBBINode of the specified type and push it
 //                onto the stack of nodes.
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
     fNodeStackPtr++;
     if (fNodeStackPtr >= kStackSize) {
         error(U_BRK_INTERNAL_ERROR);
 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
     fNodeStackPtr++;
     if (fNodeStackPtr >= kStackSize) {
         error(U_BRK_INTERNAL_ERROR);

-        RBBIDebugPrintf("RBBIRuleScanner::pushNewNode - stack overflow.\n");
+        RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow.");

         *fRB->fStatus = U_BRK_INTERNAL_ERROR;
         return NULL;
     }
         *fRB->fStatus = U_BRK_INTERNAL_ERROR;
         return NULL;
     }


@@ -1051,7 +1121,7 @@ RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
 
 
 
 
 
 

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 //
 //  scanSet    Construct a UnicodeSet from the text at the current scan
 //             position.  Advance the scan position to the first character
 //
 //  scanSet    Construct a UnicodeSet from the text at the current scan
 //             position.  Advance the scan position to the first character


@@ -1064,7 +1134,7 @@ RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
 //             that controls rule parsing.  UnicodeSets, however, are parsed by
 //             the UnicodeSet constructor, not by the RBBI rule parser.
 //
 //             that controls rule parsing.  UnicodeSets, however, are parsed by
 //             the UnicodeSet constructor, not by the RBBI rule parser.
 //

-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

 void RBBIRuleScanner::scanSet() {
     UnicodeSet    *uset;
     ParsePosition  pos;
 void RBBIRuleScanner::scanSet() {
     UnicodeSet    *uset;
     ParsePosition  pos;


@@ -1078,13 +1148,18 @@ void RBBIRuleScanner::scanSet() {
     pos.setIndex(fScanIndex);
     startPos = fScanIndex;
     UErrorCode localStatus = U_ZERO_ERROR;
     pos.setIndex(fScanIndex);
     startPos = fScanIndex;
     UErrorCode localStatus = U_ZERO_ERROR;

-    uset = new UnicodeSet(fRB->fRules, pos,
-                         *fSymbolTable,
-                         localStatus);
+    uset = new UnicodeSet();
+    if (uset == NULL) {
+        localStatus = U_MEMORY_ALLOCATION_ERROR;
+    } else {
+        uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus);
+    }

     if (U_FAILURE(localStatus)) {
         //  TODO:  Get more accurate position of the error from UnicodeSet's return info.
         //         UnicodeSet appears to not be reporting correctly at this time.
     if (U_FAILURE(localStatus)) {
         //  TODO:  Get more accurate position of the error from UnicodeSet's return info.
         //         UnicodeSet appears to not be reporting correctly at this time.

-        RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex());
+        #ifdef RBBI_DEBUG
+            RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex());
+        #endif

         error(localStatus);
         delete uset;
         return;
         error(localStatus);
         delete uset;
         return;


@@ -1092,7 +1167,8 @@ void RBBIRuleScanner::scanSet() {
 
     // Verify that the set contains at least one code point.
     //
 
     // Verify that the set contains at least one code point.
     //

-    if (uset->charAt(0) == -1) {
+    U_ASSERT(uset!=NULL);
+    if (uset->isEmpty()) {

         // This set is empty.
         //  Make it an error, because it almost certainly is not what the user wanted.
         //  Also, avoids having to think about corner cases in the tree manipulation code
         // This set is empty.
         //  Make it an error, because it almost certainly is not what the user wanted.
         //  Also, avoids having to think about corner cases in the tree manipulation code