ICU-57131.0.1.tar.gz
[apple/icu.git] / icuSources / common / rbbiscan.cpp
index 0113c08ed377d25afca0912c12c261736da6163b..3a74834bfaedf650cad29326730475f544cec5fa 100644 (file)
@@ -1,8 +1,7 @@
-
 //
 //  file:  rbbiscan.cpp
 //
-//  Copyright (C) 2002-2012, International Business Machines Corporation and others.
+//  Copyright (C) 2002-2016, International Business Machines Corporation and others.
 //  All Rights Reserved.
 //
 //  This file contains the Rule Based Break Iterator Rule Builder functions for
@@ -35,8 +34,6 @@
 
 #include "uassert.h"
 
-#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
-
 //------------------------------------------------------------------------------
 //
 // Unicode Set init strings for each of the character classes needed for parsing a rule file.
@@ -90,24 +87,27 @@ U_NAMESPACE_BEGIN
 RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
 {
     fRB                 = rb;
+    fScanIndex          = 0;
+    fNextIndex          = 0;
+    fQuoteMode          = FALSE;
+    fLineNum            = 1;
+    fCharNum            = 0;
+    fLastChar           = 0;
+    
+    fStateTable         = NULL;
+    fStack[0]           = 0;
     fStackPtr           = 0;
-    fStack[fStackPtr]   = 0;
-    fNodeStackPtr       = 0;
-    fRuleNum            = 0;
     fNodeStack[0]       = NULL;
-
-    fSymbolTable                            = NULL;
-    fSetTable                               = NULL;
-
-    fScanIndex = 0;
-    fNextIndex = 0;
+    fNodeStackPtr       = 0;
 
     fReverseRule        = FALSE;
     fLookAheadRule      = FALSE;
+    fNoChainInRule      = FALSE;
 
-    fLineNum    = 1;
-    fCharNum    = 0;
-    fQuoteMode  = FALSE;
+    fSymbolTable        = NULL;
+    fSetTable           = NULL;
+    fRuleNum            = 0;
+    fOptionStart        = 0;
 
     // Do not check status until after all critical fields are sufficiently initialized
     //   that the destructor can run cleanly.
@@ -208,11 +208,20 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         break;
 
 
+    case doNoChain:
+        // Scanned a '^' while on the rule start state.
+        fNoChainInRule = TRUE;
+        break;
+
+
     case doExprOrOperator:
         {
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             orNode->fLeftChild     = operandNode;
             operandNode->fParent   = orNode;
         }
@@ -227,6 +236,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
             fixOpStack(RBBINode::precOpCat);
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *catNode     = pushNewNode(RBBINode::opCat);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             catNode->fLeftChild    = operandNode;
             operandNode->fParent   = catNode;
         }
@@ -315,21 +327,40 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
 #endif
         U_ASSERT(fNodeStackPtr == 1);
+        RBBINode *thisRule = fNodeStack[fNodeStackPtr];
 
         // If this rule includes a look-ahead '/', add a endMark node to the
         //   expression tree.
         if (fLookAheadRule) {
-            RBBINode  *thisRule       = fNodeStack[fNodeStackPtr];
             RBBINode  *endNode        = pushNewNode(RBBINode::endMark);
             RBBINode  *catNode        = pushNewNode(RBBINode::opCat);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             fNodeStackPtr -= 2;
             catNode->fLeftChild       = thisRule;
             catNode->fRightChild      = endNode;
             fNodeStack[fNodeStackPtr] = catNode;
             endNode->fVal             = fRuleNum;
             endNode->fLookAheadEnd    = TRUE;
+            thisRule                  = catNode;
+
+            // TODO: Disable chaining out of look-ahead (hard break) rules.
+            //   The break on rule match is forced, so there is no point in building up
+            //   the state table to chain into another rule for a longer match.
         }
 
+        // Mark this node as being the root of a rule.
+        thisRule->fRuleRoot = TRUE;
+
+        // Flag if chaining into this rule is wanted.
+        //    
+        if (fRB->fChainRules &&         // If rule chaining is enabled globally via !!chain
+                !fNoChainInRule) {      //     and no '^' chain-in inhibit was on this rule
+            thisRule->fChainIn = TRUE;
+        }
+
+
         // All rule expressions are ORed together.
         // The ';' that terminates an expression really just functions as a '|' with
         //   a low operator prededence.
@@ -349,6 +380,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
             RBBINode  *thisRule    = fNodeStack[fNodeStackPtr];
             RBBINode  *prevRules   = *destRules;
             RBBINode  *orNode      = pushNewNode(RBBINode::opOr);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             orNode->fLeftChild     = prevRules;
             prevRules->fParent     = orNode;
             orNode->fRightChild    = thisRule;
@@ -363,6 +397,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         }
         fReverseRule   = FALSE;   // in preparation for the next rule.
         fLookAheadRule = FALSE;
+        fNoChainInRule = FALSE;
         fNodeStackPtr  = 0;
         }
         break;
@@ -389,6 +424,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *plusNode    = pushNewNode(RBBINode::opPlus);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             plusNode->fLeftChild   = operandNode;
             operandNode->fParent   = plusNode;
         }
@@ -398,6 +436,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *qNode       = pushNewNode(RBBINode::opQuestion);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             qNode->fLeftChild      = operandNode;
             operandNode->fParent   = qNode;
         }
@@ -407,6 +448,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         {
             RBBINode  *operandNode = fNodeStack[fNodeStackPtr--];
             RBBINode  *starNode    = pushNewNode(RBBINode::opStar);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             starNode->fLeftChild   = operandNode;
             operandNode->fParent   = starNode;
         }
@@ -420,6 +464,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         // sets that just happen to contain only one character.
         {
             n = pushNewNode(RBBINode::setRef);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             findSetFor(UnicodeString(fC.fChar), n);
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
@@ -431,6 +478,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         // scanned a ".", meaning match any single character.
         {
             n = pushNewNode(RBBINode::setRef);
+            if (U_FAILURE(*fRB->fStatus)) {
+                break;
+            }
             findSetFor(UnicodeString(TRUE, kAny, 3), n);
             n->fFirstPos = fScanIndex;
             n->fLastPos  = fNextIndex;
@@ -441,6 +491,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
     case doSlash:
         // Scanned a '/', which identifies a look-ahead break position in a rule.
         n = pushNewNode(RBBINode::lookAhead);
+        if (U_FAILURE(*fRB->fStatus)) {
+            break;
+        }
         n->fVal      = fRuleNum;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;
@@ -452,6 +505,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
     case doStartTagValue:
         // Scanned a '{', the opening delimiter for a tag value within a rule.
         n = pushNewNode(RBBINode::tag);
+        if (U_FAILURE(*fRB->fStatus)) {
+            break;
+        }
         n->fVal      = 0;
         n->fFirstPos = fScanIndex;
         n->fLastPos  = fNextIndex;
@@ -490,6 +546,8 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
                 fRB->fChainRules = TRUE;
             } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) {
                 fRB->fLBCMNoChain = TRUE;
+            } else if (opt == UNICODE_STRING("RINoChain", 9)) {
+                fRB->fRINoChain = TRUE;
             } else if (opt == UNICODE_STRING("forward", 7)) {
                 fRB->fDefaultTree   = &fRB->fForwardTree;
             } else if (opt == UNICODE_STRING("reverse", 7)) {
@@ -562,7 +620,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         returnVal = FALSE;
         break;
     }
-    return returnVal;
+    return returnVal && U_SUCCESS(*fRB->fStatus);
 }
 
 
@@ -581,7 +639,7 @@ void RBBIRuleScanner::error(UErrorCode e) {
             fRB->fParseError->line  = fLineNum;
             fRB->fParseError->offset = fCharNum;
             fRB->fParseError->preContext[0] = 0;
-            fRB->fParseError->preContext[0] = 0;
+            fRB->fParseError->postContext[0] = 0;
         }
     }
 }
@@ -964,7 +1022,7 @@ void RBBIRuleScanner::parse() {
 
         for (;;) {
             #ifdef RBBI_DEBUG
-                if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+                if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);}
             #endif
             if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE &&   tableEl->fCharClass == fC.fChar) {
                 // Table row specified an individual character, not a set, and
@@ -993,7 +1051,7 @@ void RBBIRuleScanner::parse() {
             if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 &&   // Table specs a char class &&
                 fC.fEscaped == FALSE &&                                      //   char is not escaped &&
                 fC.fChar != (UChar32)-1) {                                   //   char is not EOF
-                U_ASSERT((tableEl->fCharClass-128) < LENGTHOF(fRuleSets));
+                U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets));
                 if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
                     // Table row specified a character class, or set of characters,
                     //   and the current char matches it.
@@ -1053,6 +1111,9 @@ void RBBIRuleScanner::parse() {
     if (fRB->fReverseTree == NULL) {
         fRB->fReverseTree  = pushNewNode(RBBINode::opStar);
         RBBINode  *operand = pushNewNode(RBBINode::setRef);
+        if (U_FAILURE(*fRB->fStatus)) {
+            return;
+        }
         findSetFor(UnicodeString(TRUE, kAny, 3), operand);
         fRB->fReverseTree->fLeftChild = operand;
         operand->fParent              = fRB->fReverseTree;
@@ -1105,6 +1166,9 @@ void RBBIRuleScanner::printNodeStack(const char *title) {
 //
 //------------------------------------------------------------------------------
 RBBINode  *RBBIRuleScanner::pushNewNode(RBBINode::NodeType  t) {
+    if (U_FAILURE(*fRB->fStatus)) {
+        return NULL;
+    }
     fNodeStackPtr++;
     if (fNodeStackPtr >= kStackSize) {
         error(U_BRK_INTERNAL_ERROR);
@@ -1194,6 +1258,9 @@ void RBBIRuleScanner::scanSet() {
         RBBINode         *n;
 
         n = pushNewNode(RBBINode::setRef);
+        if (U_FAILURE(*fRB->fStatus)) {
+            return;
+        }
         n->fFirstPos = startPos;
         n->fLastPos  = fNextIndex;
         fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);