-
//
// file: rbbiscan.cpp
//
-// Copyright (C) 2002-2012, International Business Machines Corporation and others.
+// Copyright (C) 2002-2016, International Business Machines Corporation and others.
// All Rights Reserved.
//
// This file contains the Rule Based Break Iterator Rule Builder functions for
#include "uassert.h"
-#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
-
//------------------------------------------------------------------------------
//
// Unicode Set init strings for each of the character classes needed for parsing a rule file.
RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
{
fRB = rb;
+ fScanIndex = 0;
+ fNextIndex = 0;
+ fQuoteMode = FALSE;
+ fLineNum = 1;
+ fCharNum = 0;
+ fLastChar = 0;
+
+ fStateTable = NULL;
+ fStack[0] = 0;
fStackPtr = 0;
- fStack[fStackPtr] = 0;
- fNodeStackPtr = 0;
- fRuleNum = 0;
fNodeStack[0] = NULL;
-
- fSymbolTable = NULL;
- fSetTable = NULL;
-
- fScanIndex = 0;
- fNextIndex = 0;
+ fNodeStackPtr = 0;
fReverseRule = FALSE;
fLookAheadRule = FALSE;
+ fNoChainInRule = FALSE;
- fLineNum = 1;
- fCharNum = 0;
- fQuoteMode = FALSE;
+ fSymbolTable = NULL;
+ fSetTable = NULL;
+ fRuleNum = 0;
+ fOptionStart = 0;
// Do not check status until after all critical fields are sufficiently initialized
// that the destructor can run cleanly.
break;
+ case doNoChain:
+ // Scanned a '^' while on the rule start state.
+ fNoChainInRule = TRUE;
+ break;
+
+
case doExprOrOperator:
{
fixOpStack(RBBINode::precOpCat);
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *orNode = pushNewNode(RBBINode::opOr);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
orNode->fLeftChild = operandNode;
operandNode->fParent = orNode;
}
fixOpStack(RBBINode::precOpCat);
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *catNode = pushNewNode(RBBINode::opCat);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
catNode->fLeftChild = operandNode;
operandNode->fParent = catNode;
}
if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
#endif
U_ASSERT(fNodeStackPtr == 1);
+ RBBINode *thisRule = fNodeStack[fNodeStackPtr];
// If this rule includes a look-ahead '/', add a endMark node to the
// expression tree.
if (fLookAheadRule) {
- RBBINode *thisRule = fNodeStack[fNodeStackPtr];
RBBINode *endNode = pushNewNode(RBBINode::endMark);
RBBINode *catNode = pushNewNode(RBBINode::opCat);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
fNodeStackPtr -= 2;
catNode->fLeftChild = thisRule;
catNode->fRightChild = endNode;
fNodeStack[fNodeStackPtr] = catNode;
endNode->fVal = fRuleNum;
endNode->fLookAheadEnd = TRUE;
+ thisRule = catNode;
+
+ // TODO: Disable chaining out of look-ahead (hard break) rules.
+ // The break on rule match is forced, so there is no point in building up
+ // the state table to chain into another rule for a longer match.
}
+ // Mark this node as being the root of a rule.
+ thisRule->fRuleRoot = TRUE;
+
+ // Flag if chaining into this rule is wanted.
+ //
+ if (fRB->fChainRules && // If rule chaining is enabled globally via !!chain
+ !fNoChainInRule) { // and no '^' chain-in inhibit was on this rule
+ thisRule->fChainIn = TRUE;
+ }
+
+
// All rule expressions are ORed together.
// The ';' that terminates an expression really just functions as a '|' with
// a low operator prededence.
RBBINode *thisRule = fNodeStack[fNodeStackPtr];
RBBINode *prevRules = *destRules;
RBBINode *orNode = pushNewNode(RBBINode::opOr);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
orNode->fLeftChild = prevRules;
prevRules->fParent = orNode;
orNode->fRightChild = thisRule;
}
fReverseRule = FALSE; // in preparation for the next rule.
fLookAheadRule = FALSE;
+ fNoChainInRule = FALSE;
fNodeStackPtr = 0;
}
break;
{
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *plusNode = pushNewNode(RBBINode::opPlus);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
plusNode->fLeftChild = operandNode;
operandNode->fParent = plusNode;
}
{
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *qNode = pushNewNode(RBBINode::opQuestion);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
qNode->fLeftChild = operandNode;
operandNode->fParent = qNode;
}
{
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *starNode = pushNewNode(RBBINode::opStar);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
starNode->fLeftChild = operandNode;
operandNode->fParent = starNode;
}
// sets that just happen to contain only one character.
{
n = pushNewNode(RBBINode::setRef);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
findSetFor(UnicodeString(fC.fChar), n);
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
// scanned a ".", meaning match any single character.
{
n = pushNewNode(RBBINode::setRef);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
findSetFor(UnicodeString(TRUE, kAny, 3), n);
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
case doSlash:
// Scanned a '/', which identifies a look-ahead break position in a rule.
n = pushNewNode(RBBINode::lookAhead);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
n->fVal = fRuleNum;
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
case doStartTagValue:
// Scanned a '{', the opening delimiter for a tag value within a rule.
n = pushNewNode(RBBINode::tag);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
n->fVal = 0;
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
fRB->fChainRules = TRUE;
} else if (opt == UNICODE_STRING("LBCMNoChain", 11)) {
fRB->fLBCMNoChain = TRUE;
+ } else if (opt == UNICODE_STRING("RINoChain", 9)) {
+ fRB->fRINoChain = TRUE;
} else if (opt == UNICODE_STRING("forward", 7)) {
fRB->fDefaultTree = &fRB->fForwardTree;
} else if (opt == UNICODE_STRING("reverse", 7)) {
returnVal = FALSE;
break;
}
- return returnVal;
+ return returnVal && U_SUCCESS(*fRB->fStatus);
}
fRB->fParseError->line = fLineNum;
fRB->fParseError->offset = fCharNum;
fRB->fParseError->preContext[0] = 0;
- fRB->fParseError->preContext[0] = 0;
+ fRB->fParseError->postContext[0] = 0;
}
}
}
for (;;) {
#ifdef RBBI_DEBUG
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+ if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);}
#endif
if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE && tableEl->fCharClass == fC.fChar) {
// Table row specified an individual character, not a set, and
if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class &&
fC.fEscaped == FALSE && // char is not escaped &&
fC.fChar != (UChar32)-1) { // char is not EOF
- U_ASSERT((tableEl->fCharClass-128) < LENGTHOF(fRuleSets));
+ U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets));
if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
// Table row specified a character class, or set of characters,
// and the current char matches it.
if (fRB->fReverseTree == NULL) {
fRB->fReverseTree = pushNewNode(RBBINode::opStar);
RBBINode *operand = pushNewNode(RBBINode::setRef);
+ if (U_FAILURE(*fRB->fStatus)) {
+ return;
+ }
findSetFor(UnicodeString(TRUE, kAny, 3), operand);
fRB->fReverseTree->fLeftChild = operand;
operand->fParent = fRB->fReverseTree;
//
//------------------------------------------------------------------------------
RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) {
+ if (U_FAILURE(*fRB->fStatus)) {
+ return NULL;
+ }
fNodeStackPtr++;
if (fNodeStackPtr >= kStackSize) {
error(U_BRK_INTERNAL_ERROR);
RBBINode *n;
n = pushNewNode(RBBINode::setRef);
+ if (U_FAILURE(*fRB->fStatus)) {
+ return;
+ }
n->fFirstPos = startPos;
n->fLastPos = fNextIndex;
fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);