-
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
//
// file: rbbiscan.cpp
//
-// Copyright (C) 2002-2003, International Business Machines Corporation and others.
+// Copyright (C) 2002-2016, International Business Machines Corporation and others.
// All Rights Reserved.
//
// This file contains the Rule Based Break Iterator Rule Builder functions for
#include "unicode/uchriter.h"
#include "unicode/parsepos.h"
#include "unicode/parseerr.h"
-#include "uprops.h"
#include "cmemory.h"
#include "cstring.h"
#include "rbbirb.h"
#include "rbbinode.h"
#include "rbbiscan.h"
+#include "rbbitblb.h"
#include "uassert.h"
-
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Unicode Set init strings for each of the character classes needed for parsing a rule file.
// (Initialized with hex values for portability to EBCDIC based machines.
// The sets are referred to by name in the rbbirpt.txt, which is the
// source form of the state transition table for the RBBI rule parser.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
static const UChar gRuleSet_rule_char_pattern[] = {
+ // Characters that may appear as literals in patterns without escaping or quoting.
// [ ^ [ \ p { Z } \ u 0 0 2 0
0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
// - \ u 0 0 7 f ] - [ \ p
U_CDECL_BEGIN
-static void U_EXPORT2 U_CALLCONV RBBISetTable_deleter(void *p) {
- RBBISetTableEl *px = (RBBISetTableEl *)p;
+static void U_CALLCONV RBBISetTable_deleter(void *p) {
+ icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p;
delete px->key;
// Note: px->val is owned by the linked list "fSetsListHead" in scanner.
// Don't delete the value nodes here.
U_NAMESPACE_BEGIN
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Constructor.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb)
{
fRB = rb;
+ fScanIndex = 0;
+ fNextIndex = 0;
+ fQuoteMode = FALSE;
+ fLineNum = 1;
+ fCharNum = 0;
+ fLastChar = 0;
+
+ fStateTable = NULL;
+ fStack[0] = 0;
fStackPtr = 0;
- fStack[fStackPtr] = 0;
- fNodeStackPtr = 0;
- fRuleNum = 0;
fNodeStack[0] = NULL;
-
- fRuleSets[kRuleSet_rule_char-128] = NULL;
- fRuleSets[kRuleSet_white_space-128] = NULL;
- fRuleSets[kRuleSet_name_char-128] = NULL;
- fRuleSets[kRuleSet_name_start_char-128] = NULL;
- fRuleSets[kRuleSet_digit_char-128] = NULL;
- fSymbolTable = NULL;
- fSetTable = NULL;
-
- fScanIndex = 0;
- fNextIndex = 0;
+ fNodeStackPtr = 0;
fReverseRule = FALSE;
fLookAheadRule = FALSE;
+ fNoChainInRule = FALSE;
- fLineNum = 1;
- fCharNum = 0;
- fQuoteMode = FALSE;
+ fSymbolTable = NULL;
+ fSetTable = NULL;
+ fRuleNum = 0;
+ fOptionStart = 0;
+ // Do not check status until after all critical fields are sufficiently initialized
+ // that the destructor can run cleanly.
if (U_FAILURE(*rb->fStatus)) {
return;
}
// all instances of RBBIRuleScanners. BUT this is quite a bit simpler,
// and the time to build these few sets should be small compared to a
// full break iterator build.
- fRuleSets[kRuleSet_rule_char-128] = new UnicodeSet(gRuleSet_rule_char_pattern, *rb->fStatus);
- fRuleSets[kRuleSet_white_space-128] = (UnicodeSet*) uprv_openRuleWhiteSpaceSet(rb->fStatus);
- fRuleSets[kRuleSet_name_char-128] = new UnicodeSet(gRuleSet_name_char_pattern, *rb->fStatus);
- fRuleSets[kRuleSet_name_start_char-128] = new UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus);
- fRuleSets[kRuleSet_digit_char-128] = new UnicodeSet(gRuleSet_digit_char_pattern, *rb->fStatus);
+ fRuleSets[kRuleSet_rule_char-128]
+ = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern), *rb->fStatus);
+ // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:]
+ fRuleSets[kRuleSet_white_space-128].
+ add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029);
+ fRuleSets[kRuleSet_name_char-128]
+ = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern), *rb->fStatus);
+ fRuleSets[kRuleSet_name_start_char-128]
+ = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus);
+ fRuleSets[kRuleSet_digit_char-128]
+ = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern), *rb->fStatus);
if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) {
// This case happens if ICU's data is missing. UnicodeSet tries to look up property
- // names from the init string, can't find them, and claims an illegal arguement.
+ // names from the init string, can't find them, and claims an illegal argument.
// Change the error so that the actual problem will be clearer to users.
*rb->fStatus = U_BRK_INIT_ERROR;
}
}
fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus);
- fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, rb->fStatus);
+ if (fSymbolTable == NULL) {
+ *rb->fStatus = U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
+ fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus);
+ if (U_FAILURE(*rb->fStatus)) {
+ return;
+ }
uhash_setValueDeleter(fSetTable, RBBISetTable_deleter);
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Destructor
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
RBBIRuleScanner::~RBBIRuleScanner() {
- delete fRuleSets[kRuleSet_rule_char-128];
- delete fRuleSets[kRuleSet_white_space-128];
- delete fRuleSets[kRuleSet_name_char-128];
- delete fRuleSets[kRuleSet_name_start_char-128];
- delete fRuleSets[kRuleSet_digit_char-128];
-
delete fSymbolTable;
if (fSetTable != NULL) {
uhash_close(fSetTable);
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// doParseAction Do some action during rule parsing.
// Called by the parse state machine.
// in some compilers, while at the same time avoiding multiple
// definitions problems. I'm sure that there's a better way.
//
-//----------------------------------------------------------------------------------------
-UBool RBBIRuleScanner::doParseActions(EParseAction action)
+//------------------------------------------------------------------------------
+UBool RBBIRuleScanner::doParseActions(int32_t action)
{
RBBINode *n = NULL;
UBool returnVal = TRUE;
- switch ((RBBI_RuleParseAction)action) {
+ switch (action) {
case doExprStart:
pushNewNode(RBBINode::opStart);
break;
+ case doNoChain:
+ // Scanned a '^' while on the rule start state.
+ fNoChainInRule = TRUE;
+ break;
+
+
case doExprOrOperator:
{
fixOpStack(RBBINode::precOpCat);
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *orNode = pushNewNode(RBBINode::opOr);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
orNode->fLeftChild = operandNode;
operandNode->fParent = orNode;
}
fixOpStack(RBBINode::precOpCat);
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *catNode = pushNewNode(RBBINode::opCat);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
catNode->fLeftChild = operandNode;
operandNode->fParent = catNode;
}
// Make a symbol table entry for the $variableRef node.
fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus);
+ if (U_FAILURE(*fRB->fStatus)) {
+ // This is a round-about way to get the parse position set
+ // so that duplicate symbols error messages include a line number.
+ UErrorCode t = *fRB->fStatus;
+ *fRB->fStatus = U_ZERO_ERROR;
+ error(t);
+ }
// Clean up the stack.
delete startExprNode;
if (U_FAILURE(*fRB->fStatus)) { // parse tree rooted in TOS node.
break;
}
+#ifdef RBBI_DEBUG
if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");}
+#endif
U_ASSERT(fNodeStackPtr == 1);
+ RBBINode *thisRule = fNodeStack[fNodeStackPtr];
// If this rule includes a look-ahead '/', add a endMark node to the
// expression tree.
if (fLookAheadRule) {
- RBBINode *thisRule = fNodeStack[fNodeStackPtr];
RBBINode *endNode = pushNewNode(RBBINode::endMark);
RBBINode *catNode = pushNewNode(RBBINode::opCat);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
fNodeStackPtr -= 2;
catNode->fLeftChild = thisRule;
catNode->fRightChild = endNode;
fNodeStack[fNodeStackPtr] = catNode;
endNode->fVal = fRuleNum;
endNode->fLookAheadEnd = TRUE;
+ thisRule = catNode;
+
+ // TODO: Disable chaining out of look-ahead (hard break) rules.
+ // The break on rule match is forced, so there is no point in building up
+ // the state table to chain into another rule for a longer match.
+ }
+
+ // Mark this node as being the root of a rule.
+ thisRule->fRuleRoot = TRUE;
+
+ // Flag if chaining into this rule is wanted.
+ //
+ if (fRB->fChainRules && // If rule chaining is enabled globally via !!chain
+ !fNoChainInRule) { // and no '^' chain-in inhibit was on this rule
+ thisRule->fChainIn = TRUE;
}
+
// All rule expressions are ORed together.
// The ';' that terminates an expression really just functions as a '|' with
// a low operator prededence.
//
- // Forward and reverse rules are collected separately. Or this rule into
- // the appropriate group of them.
+ // Each of the four sets of rules are collected separately.
+ // (forward, reverse, safe_forward, safe_reverse)
+ // OR this rule into the appropriate group of them.
//
- RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : &fRB->fForwardTree);
+ RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree);
if (*destRules != NULL) {
// This is not the first rule encounted.
// with the current rule expression (on the Node Stack)
// with the resulting OR expression going to *destRules
//
- RBBINode *thisRule = fNodeStack[fNodeStackPtr];
+ thisRule = fNodeStack[fNodeStackPtr];
RBBINode *prevRules = *destRules;
RBBINode *orNode = pushNewNode(RBBINode::opOr);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
orNode->fLeftChild = prevRules;
prevRules->fParent = orNode;
orNode->fRightChild = thisRule;
}
fReverseRule = FALSE; // in preparation for the next rule.
fLookAheadRule = FALSE;
+ fNoChainInRule = FALSE;
fNodeStackPtr = 0;
}
break;
{
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *plusNode = pushNewNode(RBBINode::opPlus);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
plusNode->fLeftChild = operandNode;
operandNode->fParent = plusNode;
}
{
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *qNode = pushNewNode(RBBINode::opQuestion);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
qNode->fLeftChild = operandNode;
operandNode->fParent = qNode;
}
{
RBBINode *operandNode = fNodeStack[fNodeStackPtr--];
RBBINode *starNode = pushNewNode(RBBINode::opStar);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
starNode->fLeftChild = operandNode;
operandNode->fParent = starNode;
}
// sets that just happen to contain only one character.
{
n = pushNewNode(RBBINode::setRef);
- findSetFor(fC.fChar, n);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
+ findSetFor(UnicodeString(fC.fChar), n);
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
// scanned a ".", meaning match any single character.
{
n = pushNewNode(RBBINode::setRef);
- findSetFor(kAny, n);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
+ findSetFor(UnicodeString(TRUE, kAny, 3), n);
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
break;
}
- break;
case doSlash:
// Scanned a '/', which identifies a look-ahead break position in a rule.
n = pushNewNode(RBBINode::lookAhead);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
n->fVal = fRuleNum;
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
case doStartTagValue:
// Scanned a '{', the opening delimiter for a tag value within a rule.
n = pushNewNode(RBBINode::tag);
+ if (U_FAILURE(*fRB->fStatus)) {
+ break;
+ }
n->fVal = 0;
n->fFirstPos = fScanIndex;
n->fLastPos = fNextIndex;
fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
break;
+ case doTagExpectedError:
+ error(U_BRK_MALFORMED_RULE_TAG);
+ returnVal = FALSE;
+ break;
+ case doOptionStart:
+ // Scanning a !!option. At the start of string.
+ fOptionStart = fScanIndex;
+ break;
+
+ case doOptionEnd:
+ {
+ UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart);
+ if (opt == UNICODE_STRING("chain", 5)) {
+ fRB->fChainRules = TRUE;
+ } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) {
+ fRB->fLBCMNoChain = TRUE;
+ } else if (opt == UNICODE_STRING("forward", 7)) {
+ fRB->fDefaultTree = &fRB->fForwardTree;
+ } else if (opt == UNICODE_STRING("reverse", 7)) {
+ fRB->fDefaultTree = &fRB->fReverseTree;
+ } else if (opt == UNICODE_STRING("safe_forward", 12)) {
+ fRB->fDefaultTree = &fRB->fSafeFwdTree;
+ } else if (opt == UNICODE_STRING("safe_reverse", 12)) {
+ fRB->fDefaultTree = &fRB->fSafeRevTree;
+ } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
+ fRB->fLookAheadHardBreak = TRUE;
+ } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) {
+ fRuleSets[kRuleSet_rule_char-128].clear();
+ } else if (opt == UNICODE_STRING("unquoted_literals", 17)) {
+ fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus);
+ } else {
+ error(U_BRK_UNRECOGNIZED_OPTION);
+ }
+ }
+ break;
case doReverseDir:
fReverseRule = TRUE;
returnVal = FALSE;
break;
}
- return returnVal;
+ return returnVal && U_SUCCESS(*fRB->fStatus);
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Error Report a rule parse error.
// Only report it if no previous error has been recorded.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RBBIRuleScanner::error(UErrorCode e) {
if (U_SUCCESS(*fRB->fStatus)) {
*fRB->fStatus = e;
- fRB->fParseError->line = fLineNum;
- fRB->fParseError->offset = fCharNum;
- fRB->fParseError->preContext[0] = 0;
- fRB->fParseError->preContext[0] = 0;
+ if (fRB->fParseError) {
+ fRB->fParseError->line = fLineNum;
+ fRB->fParseError->offset = fCharNum;
+ fRB->fParseError->preContext[0] = 0;
+ fRB->fParseError->postContext[0] = 0;
+ }
}
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// fixOpStack The parse stack holds partially assembled chunks of the parse tree.
// An entry on the stack may be as small as a single setRef node,
// the precedence of the current operator, binds the operand left,
// to the previously encountered operator.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) {
RBBINode *n;
// printNodeStack("entering fixOpStack()");
for (;;) {
n = fNodeStack[fNodeStackPtr-1]; // an operator node
if (n->fPrecedence == 0) {
- RBBIDebugPrintf("RBBIRuleScanner::fixOpStack, bad operator node\n");
+ RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node");
error(U_BRK_INTERNAL_ERROR);
return;
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// findSetFor given a UnicodeString,
// - find the corresponding Unicode Set (uset node)
// just one element which is the char in question.
// If the string is "any", return a set containing all chars.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) {
RBBISetTableEl *el;
// This new uset node becomes the child of the caller's setReference node.
//
RBBINode *usetNode = new RBBINode(RBBINode::uset);
+ if (usetNode == NULL) {
+ error(U_MEMORY_ALLOCATION_ERROR);
+ return;
+ }
usetNode->fInputSet = setToAdopt;
usetNode->fParent = node;
node->fLeftChild = usetNode;
el = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl));
UnicodeString *tkey = new UnicodeString(s);
if (tkey == NULL || el == NULL || setToAdopt == NULL) {
+ // Delete to avoid memory leak
+ delete tkey;
+ tkey = NULL;
+ uprv_free(el);
+ el = NULL;
+ delete setToAdopt;
+ setToAdopt = NULL;
+
error(U_MEMORY_ALLOCATION_ERROR);
return;
}
static const UChar chRParen = 0x29;
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
-// stripRules Return a rules string without unnecessary
-// characters.
+// stripRules Return a rules string without extra spaces.
+// (Comments are removed separately, during rule parsing.)
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
UnicodeString strippedRules;
- int rulesLength = rules.length();
- for (int idx = 0; idx < rulesLength; ) {
- UChar ch = rules[idx++];
- if (ch == chPound) {
- while (idx < rulesLength
- && ch != chCR && ch != chLF && ch != chNEL)
- {
- ch = rules[idx++];
- }
- }
- if (!u_isISOControl(ch)) {
- strippedRules.append(ch);
+ int32_t rulesLength = rules.length();
+ bool skippingSpaces = false;
+
+ for (int32_t idx=0; idx<rulesLength; idx = rules.moveIndex32(idx, 1)) {
+ UChar32 cp = rules.char32At(idx);
+ bool whiteSpace = u_hasBinaryProperty(cp, UCHAR_PATTERN_WHITE_SPACE);
+ if (skippingSpaces && whiteSpace) {
+ continue;
}
+ strippedRules.append(cp);
+ skippingSpaces = whiteSpace;
}
- // strippedRules = strippedRules.unescape();
return strippedRules;
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// nextCharLL Low Level Next Char from rule input source.
// Get a char from the input character iterator,
// keep track of input position for error reporting.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
UChar32 RBBIRuleScanner::nextCharLL() {
UChar32 ch;
if (ch == chCR ||
ch == chNEL ||
ch == chLS ||
- ch == chLF && fLastChar != chCR) {
+ (ch == chLF && fLastChar != chCR)) {
// Character is starting a new line. Bump up the line number, and
// reset the column to 0.
fLineNum++;
}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// nextChar for rules scanning. At this level, we handle stripping
// out comments and processing backslash character escapes.
// The rest of the rules grammar is handled at the next level up.
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RBBIRuleScanner::nextChar(RBBIRuleChar &c) {
// Unicode Character constants needed for the processing done by nextChar(),
// It will be treated as white-space, and serves to break up anything
// that might otherwise incorrectly clump together with a comment in
// the middle (a variable name, for example.)
+ int32_t commentStart = fScanIndex;
for (;;) {
c.fChar = nextCharLL();
if (c.fChar == (UChar32)-1 || // EOF
c.fChar == chNEL ||
c.fChar == chLS) {break;}
}
+ for (int32_t i=commentStart; i<fNextIndex-1; ++i) {
+ fRB->fStrippedRules.setCharAt(i, u' ');
+ }
}
if (c.fChar == (UChar32)-1) {
return;
// putc(c.fChar, stdout);
}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Parse RBBI rules. The state machine for rules parsing is here.
-// The state tables are hand-written in the file TODO.txt,
+// The state tables are hand-written in the file rbbirpt.txt,
// and converted to the form used here by a perl
// script rbbicst.pl
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RBBIRuleScanner::parse() {
uint16_t state;
const RBBIRuleTableEl *tableEl;
// the search will stop there, if not before.
//
tableEl = &gRuleParseStateTable[state];
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) {
- RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ",
- fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]);
- }
+ #ifdef RBBI_DEBUG
+ if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) {
+ RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ",
+ fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]);
+ }
+ #endif
for (;;) {
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");}
+ #ifdef RBBI_DEBUG
+ if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);}
+ #endif
if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE && tableEl->fCharClass == fC.fChar) {
// Table row specified an individual character, not a set, and
// the input character is not escaped, and
if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class &&
fC.fEscaped == FALSE && // char is not escaped &&
fC.fChar != (UChar32)-1) { // char is not EOF
- UnicodeSet *uniset = fRuleSets[tableEl->fCharClass-128];
- if (uniset->contains(fC.fChar)) {
+ U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets));
+ if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
// Table row specified a character class, or set of characters,
// and the current char matches it.
break;
// No match on this row, advance to the next row for this state,
tableEl++;
}
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("\n");}
+ if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");}
//
// We've found the row of the state table that matches the current input
// character from the rules string.
// Perform any action specified by this row in the state table.
- if (doParseActions((EParseAction)tableEl->fAction) == FALSE) {
+ if (doParseActions((int32_t)tableEl->fAction) == FALSE) {
// Break out of the state machine loop if the
// the action signalled some kind of error, or
// the action was to exit, occurs on normal end-of-rules-input.
fStackPtr++;
if (fStackPtr >= kStackSize) {
error(U_BRK_INTERNAL_ERROR);
- RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack overflow.\n");
+ RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow.");
fStackPtr--;
}
fStack[fStackPtr] = tableEl->fPushState;
fStackPtr--;
if (fStackPtr < 0) {
error(U_BRK_INTERNAL_ERROR);
- RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack underflow.\n");
+ RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow.");
fStackPtr++;
}
}
}
+ if (U_FAILURE(*fRB->fStatus)) {
+ return;
+ }
+
+ // If there are no forward rules set an error.
//
- // If there were NO user specified reverse rules, set up the equivalent of ".*;"
- //
- if (fRB->fReverseTree == NULL) {
- fRB->fReverseTree = pushNewNode(RBBINode::opStar);
- RBBINode *operand = pushNewNode(RBBINode::setRef);
- findSetFor(kAny, operand);
- fRB->fReverseTree->fLeftChild = operand;
- operand->fParent = fRB->fReverseTree;
- fNodeStackPtr -= 2;
+ if (fRB->fForwardTree == NULL) {
+ error(U_BRK_RULE_SYNTAX);
+ return;
}
-
//
// Parsing of the input RBBI rules is complete.
// We now have a parse tree for the rule expressions
// and a list of all UnicodeSets that are referenced.
//
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->print();}
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree"))
- {
+#ifdef RBBI_DEBUG
+ if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();}
+ if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) {
RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n");
- fRB->fForwardTree->printTree();
+ RBBINode::printTree(fRB->fForwardTree, TRUE);
RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n");
- fRB->fReverseTree->printTree();
+ RBBINode::printTree(fRB->fReverseTree, TRUE);
+ RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n");
+ RBBINode::printTree(fRB->fSafeFwdTree, TRUE);
+ RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n");
+ RBBINode::printTree(fRB->fSafeRevTree, TRUE);
}
-
+#endif
}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// printNodeStack for debugging...
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
+#ifdef RBBI_DEBUG
void RBBIRuleScanner::printNodeStack(const char *title) {
int i;
RBBIDebugPrintf("%s. Dumping node stack...\n", title);
- for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree();}
+ for (i=fNodeStackPtr; i>0; i--) {RBBINode::printTree(fNodeStack[i], TRUE);}
}
+#endif
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// pushNewNode create a new RBBINode of the specified type and push it
// onto the stack of nodes.
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) {
- fNodeStackPtr++;
- if (fNodeStackPtr >= kStackSize) {
- error(U_BRK_INTERNAL_ERROR);
- RBBIDebugPrintf("RBBIRuleScanner::pushNewNode - stack overflow.\n");
- *fRB->fStatus = U_BRK_INTERNAL_ERROR;
+ if (U_FAILURE(*fRB->fStatus)) {
+ return NULL;
+ }
+ if (fNodeStackPtr >= kStackSize - 1) {
+ error(U_BRK_RULE_SYNTAX);
+ RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow.");
return NULL;
}
+ fNodeStackPtr++;
fNodeStack[fNodeStackPtr] = new RBBINode(t);
if (fNodeStack[fNodeStackPtr] == NULL) {
*fRB->fStatus = U_MEMORY_ALLOCATION_ERROR;
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// scanSet Construct a UnicodeSet from the text at the current scan
// position. Advance the scan position to the first character
// that controls rule parsing. UnicodeSets, however, are parsed by
// the UnicodeSet constructor, not by the RBBI rule parser.
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RBBIRuleScanner::scanSet() {
UnicodeSet *uset;
ParsePosition pos;
pos.setIndex(fScanIndex);
startPos = fScanIndex;
UErrorCode localStatus = U_ZERO_ERROR;
- uset = new UnicodeSet(fRB->fRules, pos,
- *fSymbolTable,
- localStatus);
+ uset = new UnicodeSet();
+ if (uset == NULL) {
+ localStatus = U_MEMORY_ALLOCATION_ERROR;
+ } else {
+ uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus);
+ }
if (U_FAILURE(localStatus)) {
// TODO: Get more accurate position of the error from UnicodeSet's return info.
// UnicodeSet appears to not be reporting correctly at this time.
- RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex());
+ #ifdef RBBI_DEBUG
+ RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex());
+ #endif
error(localStatus);
delete uset;
return;
// Verify that the set contains at least one code point.
//
- if (uset->charAt(0) == -1) {
+ U_ASSERT(uset!=NULL);
+ if (uset->isEmpty()) {
// This set is empty.
// Make it an error, because it almost certainly is not what the user wanted.
// Also, avoids having to think about corner cases in the tree manipulation code
RBBINode *n;
n = pushNewNode(RBBINode::setRef);
+ if (U_FAILURE(*fRB->fStatus)) {
+ return;
+ }
n->fFirstPos = startPos;
n->fLastPos = fNextIndex;
fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);
projects / apple / icu.git / blobdiff