X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/b75a7d8f3b4adbae880cab104ce2c6a50eee4db2..3d1f044b704633e2e541231cd17ae9ecf9ad5c7a:/icuSources/common/rbbiscan.cpp?ds=sidebyside diff --git a/icuSources/common/rbbiscan.cpp b/icuSources/common/rbbiscan.cpp index c5f15635..f536ab5d 100644 --- a/icuSources/common/rbbiscan.cpp +++ b/icuSources/common/rbbiscan.cpp @@ -1,8 +1,9 @@ - +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html // // file: rbbiscan.cpp // -// Copyright (C) 2002-2003, International Business Machines Corporation and others. +// Copyright (C) 2002-2016, International Business Machines Corporation and others. // All Rights Reserved. // // This file contains the Rule Based Break Iterator Rule Builder functions for @@ -23,7 +24,6 @@ #include "unicode/uchriter.h" #include "unicode/parsepos.h" #include "unicode/parseerr.h" -#include "uprops.h" #include "cmemory.h" #include "cstring.h" @@ -32,11 +32,11 @@ #include "rbbirb.h" #include "rbbinode.h" #include "rbbiscan.h" +#include "rbbitblb.h" #include "uassert.h" - -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // Unicode Set init strings for each of the character classes needed for parsing a rule file. // (Initialized with hex values for portability to EBCDIC based machines. @@ -45,8 +45,9 @@ // The sets are referred to by name in the rbbirpt.txt, which is the // source form of the state transition table for the RBBI rule parser. // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ static const UChar gRuleSet_rule_char_pattern[] = { + // Characters that may appear as literals in patterns without escaping or quoting. // [ ^ [ \ p { Z } \ u 0 0 2 0 0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30, // - \ u 0 0 7 f ] - [ \ p @@ -70,8 +71,8 @@ static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00}; // "any" U_CDECL_BEGIN -static void U_EXPORT2 U_CALLCONV RBBISetTable_deleter(void *p) { - RBBISetTableEl *px = (RBBISetTableEl *)p; +static void U_CALLCONV RBBISetTable_deleter(void *p) { + icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p; delete px->key; // Note: px->val is owned by the linked list "fSetsListHead" in scanner. // Don't delete the value nodes here. @@ -81,38 +82,38 @@ U_CDECL_END U_NAMESPACE_BEGIN -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // Constructor. // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) { fRB = rb; + fScanIndex = 0; + fNextIndex = 0; + fQuoteMode = FALSE; + fLineNum = 1; + fCharNum = 0; + fLastChar = 0; + + fStateTable = NULL; + fStack[0] = 0; fStackPtr = 0; - fStack[fStackPtr] = 0; - fNodeStackPtr = 0; - fRuleNum = 0; fNodeStack[0] = NULL; - - fRuleSets[kRuleSet_rule_char-128] = NULL; - fRuleSets[kRuleSet_white_space-128] = NULL; - fRuleSets[kRuleSet_name_char-128] = NULL; - fRuleSets[kRuleSet_name_start_char-128] = NULL; - fRuleSets[kRuleSet_digit_char-128] = NULL; - fSymbolTable = NULL; - fSetTable = NULL; - - fScanIndex = 0; - fNextIndex = 0; + fNodeStackPtr = 0; fReverseRule = FALSE; fLookAheadRule = FALSE; + fNoChainInRule = FALSE; - fLineNum = 1; - fCharNum = 0; - fQuoteMode = FALSE; + fSymbolTable = NULL; + fSetTable = NULL; + fRuleNum = 0; + fOptionStart = 0; + // Do not check status until after all critical fields are sufficiently initialized + // that the destructor can run cleanly. if (U_FAILURE(*rb->fStatus)) { return; } @@ -123,14 +124,20 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) // all instances of RBBIRuleScanners. BUT this is quite a bit simpler, // and the time to build these few sets should be small compared to a // full break iterator build. - fRuleSets[kRuleSet_rule_char-128] = new UnicodeSet(gRuleSet_rule_char_pattern, *rb->fStatus); - fRuleSets[kRuleSet_white_space-128] = (UnicodeSet*) uprv_openRuleWhiteSpaceSet(rb->fStatus); - fRuleSets[kRuleSet_name_char-128] = new UnicodeSet(gRuleSet_name_char_pattern, *rb->fStatus); - fRuleSets[kRuleSet_name_start_char-128] = new UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus); - fRuleSets[kRuleSet_digit_char-128] = new UnicodeSet(gRuleSet_digit_char_pattern, *rb->fStatus); + fRuleSets[kRuleSet_rule_char-128] + = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern), *rb->fStatus); + // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:] + fRuleSets[kRuleSet_white_space-128]. + add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029); + fRuleSets[kRuleSet_name_char-128] + = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern), *rb->fStatus); + fRuleSets[kRuleSet_name_start_char-128] + = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus); + fRuleSets[kRuleSet_digit_char-128] + = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern), *rb->fStatus); if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) { // This case happens if ICU's data is missing. UnicodeSet tries to look up property - // names from the init string, can't find them, and claims an illegal arguement. + // names from the init string, can't find them, and claims an illegal argument. // Change the error so that the actual problem will be clearer to users. *rb->fStatus = U_BRK_INIT_ERROR; } @@ -139,24 +146,25 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) } fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus); - fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, rb->fStatus); + if (fSymbolTable == NULL) { + *rb->fStatus = U_MEMORY_ALLOCATION_ERROR; + return; + } + fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus); + if (U_FAILURE(*rb->fStatus)) { + return; + } uhash_setValueDeleter(fSetTable, RBBISetTable_deleter); } -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // Destructor // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ RBBIRuleScanner::~RBBIRuleScanner() { - delete fRuleSets[kRuleSet_rule_char-128]; - delete fRuleSets[kRuleSet_white_space-128]; - delete fRuleSets[kRuleSet_name_char-128]; - delete fRuleSets[kRuleSet_name_start_char-128]; - delete fRuleSets[kRuleSet_digit_char-128]; - delete fSymbolTable; if (fSetTable != NULL) { uhash_close(fSetTable); @@ -175,7 +183,7 @@ RBBIRuleScanner::~RBBIRuleScanner() { } -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // doParseAction Do some action during rule parsing. // Called by the parse state machine. @@ -188,14 +196,14 @@ RBBIRuleScanner::~RBBIRuleScanner() { // in some compilers, while at the same time avoiding multiple // definitions problems. I'm sure that there's a better way. // -//---------------------------------------------------------------------------------------- -UBool RBBIRuleScanner::doParseActions(EParseAction action) +//------------------------------------------------------------------------------ +UBool RBBIRuleScanner::doParseActions(int32_t action) { RBBINode *n = NULL; UBool returnVal = TRUE; - switch ((RBBI_RuleParseAction)action) { + switch (action) { case doExprStart: pushNewNode(RBBINode::opStart); @@ -203,11 +211,20 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) break; + case doNoChain: + // Scanned a '^' while on the rule start state. + fNoChainInRule = TRUE; + break; + + case doExprOrOperator: { fixOpStack(RBBINode::precOpCat); RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *orNode = pushNewNode(RBBINode::opOr); + if (U_FAILURE(*fRB->fStatus)) { + break; + } orNode->fLeftChild = operandNode; operandNode->fParent = orNode; } @@ -222,6 +239,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) fixOpStack(RBBINode::precOpCat); RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *catNode = pushNewNode(RBBINode::opCat); + if (U_FAILURE(*fRB->fStatus)) { + break; + } catNode->fLeftChild = operandNode; operandNode->fParent = catNode; } @@ -286,6 +306,13 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) // Make a symbol table entry for the $variableRef node. fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus); + if (U_FAILURE(*fRB->fStatus)) { + // This is a round-about way to get the parse position set + // so that duplicate symbols error messages include a line number. + UErrorCode t = *fRB->fStatus; + *fRB->fStatus = U_ZERO_ERROR; + error(t); + } // Clean up the stack. delete startExprNode; @@ -299,31 +326,53 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) if (U_FAILURE(*fRB->fStatus)) { // parse tree rooted in TOS node. break; } +#ifdef RBBI_DEBUG if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");} +#endif U_ASSERT(fNodeStackPtr == 1); + RBBINode *thisRule = fNodeStack[fNodeStackPtr]; // If this rule includes a look-ahead '/', add a endMark node to the // expression tree. if (fLookAheadRule) { - RBBINode *thisRule = fNodeStack[fNodeStackPtr]; RBBINode *endNode = pushNewNode(RBBINode::endMark); RBBINode *catNode = pushNewNode(RBBINode::opCat); + if (U_FAILURE(*fRB->fStatus)) { + break; + } fNodeStackPtr -= 2; catNode->fLeftChild = thisRule; catNode->fRightChild = endNode; fNodeStack[fNodeStackPtr] = catNode; endNode->fVal = fRuleNum; endNode->fLookAheadEnd = TRUE; + thisRule = catNode; + + // TODO: Disable chaining out of look-ahead (hard break) rules. + // The break on rule match is forced, so there is no point in building up + // the state table to chain into another rule for a longer match. + } + + // Mark this node as being the root of a rule. + thisRule->fRuleRoot = TRUE; + + // Flag if chaining into this rule is wanted. + // + if (fRB->fChainRules && // If rule chaining is enabled globally via !!chain + !fNoChainInRule) { // and no '^' chain-in inhibit was on this rule + thisRule->fChainIn = TRUE; } + // All rule expressions are ORed together. // The ';' that terminates an expression really just functions as a '|' with // a low operator prededence. // - // Forward and reverse rules are collected separately. Or this rule into - // the appropriate group of them. + // Each of the four sets of rules are collected separately. + // (forward, reverse, safe_forward, safe_reverse) + // OR this rule into the appropriate group of them. // - RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : &fRB->fForwardTree); + RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree); if (*destRules != NULL) { // This is not the first rule encounted. @@ -331,9 +380,12 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) // with the current rule expression (on the Node Stack) // with the resulting OR expression going to *destRules // - RBBINode *thisRule = fNodeStack[fNodeStackPtr]; + thisRule = fNodeStack[fNodeStackPtr]; RBBINode *prevRules = *destRules; RBBINode *orNode = pushNewNode(RBBINode::opOr); + if (U_FAILURE(*fRB->fStatus)) { + break; + } orNode->fLeftChild = prevRules; prevRules->fParent = orNode; orNode->fRightChild = thisRule; @@ -348,6 +400,7 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) } fReverseRule = FALSE; // in preparation for the next rule. fLookAheadRule = FALSE; + fNoChainInRule = FALSE; fNodeStackPtr = 0; } break; @@ -374,6 +427,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *plusNode = pushNewNode(RBBINode::opPlus); + if (U_FAILURE(*fRB->fStatus)) { + break; + } plusNode->fLeftChild = operandNode; operandNode->fParent = plusNode; } @@ -383,6 +439,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *qNode = pushNewNode(RBBINode::opQuestion); + if (U_FAILURE(*fRB->fStatus)) { + break; + } qNode->fLeftChild = operandNode; operandNode->fParent = qNode; } @@ -392,6 +451,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *starNode = pushNewNode(RBBINode::opStar); + if (U_FAILURE(*fRB->fStatus)) { + break; + } starNode->fLeftChild = operandNode; operandNode->fParent = starNode; } @@ -405,7 +467,10 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) // sets that just happen to contain only one character. { n = pushNewNode(RBBINode::setRef); - findSetFor(fC.fChar, n); + if (U_FAILURE(*fRB->fStatus)) { + break; + } + findSetFor(UnicodeString(fC.fChar), n); n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); @@ -416,17 +481,22 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) // scanned a ".", meaning match any single character. { n = pushNewNode(RBBINode::setRef); - findSetFor(kAny, n); + if (U_FAILURE(*fRB->fStatus)) { + break; + } + findSetFor(UnicodeString(TRUE, kAny, 3), n); n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); break; } - break; case doSlash: // Scanned a '/', which identifies a look-ahead break position in a rule. n = pushNewNode(RBBINode::lookAhead); + if (U_FAILURE(*fRB->fStatus)) { + break; + } n->fVal = fRuleNum; n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; @@ -438,6 +508,9 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) case doStartTagValue: // Scanned a '{', the opening delimiter for a tag value within a rule. n = pushNewNode(RBBINode::tag); + if (U_FAILURE(*fRB->fStatus)) { + break; + } n->fVal = 0; n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; @@ -459,7 +532,42 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); break; + case doTagExpectedError: + error(U_BRK_MALFORMED_RULE_TAG); + returnVal = FALSE; + break; + case doOptionStart: + // Scanning a !!option. At the start of string. + fOptionStart = fScanIndex; + break; + + case doOptionEnd: + { + UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart); + if (opt == UNICODE_STRING("chain", 5)) { + fRB->fChainRules = TRUE; + } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) { + fRB->fLBCMNoChain = TRUE; + } else if (opt == UNICODE_STRING("forward", 7)) { + fRB->fDefaultTree = &fRB->fForwardTree; + } else if (opt == UNICODE_STRING("reverse", 7)) { + fRB->fDefaultTree = &fRB->fReverseTree; + } else if (opt == UNICODE_STRING("safe_forward", 12)) { + fRB->fDefaultTree = &fRB->fSafeFwdTree; + } else if (opt == UNICODE_STRING("safe_reverse", 12)) { + fRB->fDefaultTree = &fRB->fSafeRevTree; + } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) { + fRB->fLookAheadHardBreak = TRUE; + } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) { + fRuleSets[kRuleSet_rule_char-128].clear(); + } else if (opt == UNICODE_STRING("unquoted_literals", 17)) { + fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus); + } else { + error(U_BRK_UNRECOGNIZED_OPTION); + } + } + break; case doReverseDir: fReverseRule = TRUE; @@ -517,32 +625,34 @@ UBool RBBIRuleScanner::doParseActions(EParseAction action) returnVal = FALSE; break; } - return returnVal; + return returnVal && U_SUCCESS(*fRB->fStatus); } -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // Error Report a rule parse error. // Only report it if no previous error has been recorded. // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ void RBBIRuleScanner::error(UErrorCode e) { if (U_SUCCESS(*fRB->fStatus)) { *fRB->fStatus = e; - fRB->fParseError->line = fLineNum; - fRB->fParseError->offset = fCharNum; - fRB->fParseError->preContext[0] = 0; - fRB->fParseError->preContext[0] = 0; + if (fRB->fParseError) { + fRB->fParseError->line = fLineNum; + fRB->fParseError->offset = fCharNum; + fRB->fParseError->preContext[0] = 0; + fRB->fParseError->postContext[0] = 0; + } } } -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // fixOpStack The parse stack holds partially assembled chunks of the parse tree. // An entry on the stack may be as small as a single setRef node, @@ -556,14 +666,14 @@ void RBBIRuleScanner::error(UErrorCode e) { // the precedence of the current operator, binds the operand left, // to the previously encountered operator. // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) { RBBINode *n; // printNodeStack("entering fixOpStack()"); for (;;) { n = fNodeStack[fNodeStackPtr-1]; // an operator node if (n->fPrecedence == 0) { - RBBIDebugPrintf("RBBIRuleScanner::fixOpStack, bad operator node\n"); + RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node"); error(U_BRK_INTERNAL_ERROR); return; } @@ -603,7 +713,7 @@ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) { -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // findSetFor given a UnicodeString, // - find the corresponding Unicode Set (uset node) @@ -618,7 +728,7 @@ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) { // just one element which is the char in question. // If the string is "any", return a set containing all chars. // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) { RBBISetTableEl *el; @@ -652,6 +762,10 @@ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, Unicode // This new uset node becomes the child of the caller's setReference node. // RBBINode *usetNode = new RBBINode(RBBINode::uset); + if (usetNode == NULL) { + error(U_MEMORY_ALLOCATION_ERROR); + return; + } usetNode->fInputSet = setToAdopt; usetNode->fParent = node; node->fLeftChild = usetNode; @@ -670,6 +784,14 @@ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, Unicode el = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl)); UnicodeString *tkey = new UnicodeString(s); if (tkey == NULL || el == NULL || setToAdopt == NULL) { + // Delete to avoid memory leak + delete tkey; + tkey = NULL; + uprv_free(el); + el = NULL; + delete setToAdopt; + setToAdopt = NULL; + error(U_MEMORY_ALLOCATION_ERROR); return; } @@ -698,40 +820,37 @@ static const UChar chLParen = 0x28; static const UChar chRParen = 0x29; -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // -// stripRules Return a rules string without unnecessary -// characters. +// stripRules Return a rules string without extra spaces. +// (Comments are removed separately, during rule parsing.) // -//---------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) { UnicodeString strippedRules; - int rulesLength = rules.length(); - for (int idx = 0; idx < rulesLength; ) { - UChar ch = rules[idx++]; - if (ch == chPound) { - while (idx < rulesLength - && ch != chCR && ch != chLF && ch != chNEL) - { - ch = rules[idx++]; - } - } - if (!u_isISOControl(ch)) { - strippedRules.append(ch); + int32_t rulesLength = rules.length(); + bool skippingSpaces = false; + + for (int32_t idx=0; idxfStrippedRules.setCharAt(i, u' '); + } } if (c.fChar == (UChar32)-1) { return; @@ -850,14 +973,14 @@ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) { // putc(c.fChar, stdout); } -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // Parse RBBI rules. The state machine for rules parsing is here. -// The state tables are hand-written in the file TODO.txt, +// The state tables are hand-written in the file rbbirpt.txt, // and converted to the form used here by a perl // script rbbicst.pl // -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ void RBBIRuleScanner::parse() { uint16_t state; const RBBIRuleTableEl *tableEl; @@ -896,13 +1019,17 @@ void RBBIRuleScanner::parse() { // the search will stop there, if not before. // tableEl = &gRuleParseStateTable[state]; - if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { - RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ", - fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]); - } + #ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { + RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ", + fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]); + } + #endif for (;;) { - if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");} + #ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);} + #endif if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE && tableEl->fCharClass == fC.fChar) { // Table row specified an individual character, not a set, and // the input character is not escaped, and @@ -930,8 +1057,8 @@ void RBBIRuleScanner::parse() { if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class && fC.fEscaped == FALSE && // char is not escaped && fC.fChar != (UChar32)-1) { // char is not EOF - UnicodeSet *uniset = fRuleSets[tableEl->fCharClass-128]; - if (uniset->contains(fC.fChar)) { + U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets)); + if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) { // Table row specified a character class, or set of characters, // and the current char matches it. break; @@ -941,13 +1068,13 @@ void RBBIRuleScanner::parse() { // No match on this row, advance to the next row for this state, tableEl++; } - if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("\n");} + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");} // // We've found the row of the state table that matches the current input // character from the rules string. // Perform any action specified by this row in the state table. - if (doParseActions((EParseAction)tableEl->fAction) == FALSE) { + if (doParseActions((int32_t)tableEl->fAction) == FALSE) { // Break out of the state machine loop if the // the action signalled some kind of error, or // the action was to exit, occurs on normal end-of-rules-input. @@ -958,7 +1085,7 @@ void RBBIRuleScanner::parse() { fStackPtr++; if (fStackPtr >= kStackSize) { error(U_BRK_INTERNAL_ERROR); - RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack overflow.\n"); + RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow."); fStackPtr--; } fStack[fStackPtr] = tableEl->fPushState; @@ -977,71 +1104,77 @@ void RBBIRuleScanner::parse() { fStackPtr--; if (fStackPtr < 0) { error(U_BRK_INTERNAL_ERROR); - RBBIDebugPrintf("RBBIRuleScanner::parse() - state stack underflow.\n"); + RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow."); fStackPtr++; } } } + if (U_FAILURE(*fRB->fStatus)) { + return; + } + + // If there are no forward rules set an error. // - // If there were NO user specified reverse rules, set up the equivalent of ".*;" - // - if (fRB->fReverseTree == NULL) { - fRB->fReverseTree = pushNewNode(RBBINode::opStar); - RBBINode *operand = pushNewNode(RBBINode::setRef); - findSetFor(kAny, operand); - fRB->fReverseTree->fLeftChild = operand; - operand->fParent = fRB->fReverseTree; - fNodeStackPtr -= 2; + if (fRB->fForwardTree == NULL) { + error(U_BRK_RULE_SYNTAX); + return; } - // // Parsing of the input RBBI rules is complete. // We now have a parse tree for the rule expressions // and a list of all UnicodeSets that are referenced. // - if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->print();} - if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) - { +#ifdef RBBI_DEBUG + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();} + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) { RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n"); - fRB->fForwardTree->printTree(); + RBBINode::printTree(fRB->fForwardTree, TRUE); RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n"); - fRB->fReverseTree->printTree(); + RBBINode::printTree(fRB->fReverseTree, TRUE); + RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n"); + RBBINode::printTree(fRB->fSafeFwdTree, TRUE); + RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n"); + RBBINode::printTree(fRB->fSafeRevTree, TRUE); } - +#endif } -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // printNodeStack for debugging... // -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ +#ifdef RBBI_DEBUG void RBBIRuleScanner::printNodeStack(const char *title) { int i; RBBIDebugPrintf("%s. Dumping node stack...\n", title); - for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree();} + for (i=fNodeStackPtr; i>0; i--) {RBBINode::printTree(fNodeStack[i], TRUE);} } +#endif -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // pushNewNode create a new RBBINode of the specified type and push it // onto the stack of nodes. // -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { - fNodeStackPtr++; - if (fNodeStackPtr >= kStackSize) { - error(U_BRK_INTERNAL_ERROR); - RBBIDebugPrintf("RBBIRuleScanner::pushNewNode - stack overflow.\n"); - *fRB->fStatus = U_BRK_INTERNAL_ERROR; + if (U_FAILURE(*fRB->fStatus)) { + return NULL; + } + if (fNodeStackPtr >= kStackSize - 1) { + error(U_BRK_RULE_SYNTAX); + RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow."); return NULL; } + fNodeStackPtr++; fNodeStack[fNodeStackPtr] = new RBBINode(t); if (fNodeStack[fNodeStackPtr] == NULL) { *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR; @@ -1051,7 +1184,7 @@ RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ // // scanSet Construct a UnicodeSet from the text at the current scan // position. Advance the scan position to the first character @@ -1064,7 +1197,7 @@ RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { // that controls rule parsing. UnicodeSets, however, are parsed by // the UnicodeSet constructor, not by the RBBI rule parser. // -//--------------------------------------------------------------------------------- +//------------------------------------------------------------------------------ void RBBIRuleScanner::scanSet() { UnicodeSet *uset; ParsePosition pos; @@ -1078,13 +1211,18 @@ void RBBIRuleScanner::scanSet() { pos.setIndex(fScanIndex); startPos = fScanIndex; UErrorCode localStatus = U_ZERO_ERROR; - uset = new UnicodeSet(fRB->fRules, pos, - *fSymbolTable, - localStatus); + uset = new UnicodeSet(); + if (uset == NULL) { + localStatus = U_MEMORY_ALLOCATION_ERROR; + } else { + uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus); + } if (U_FAILURE(localStatus)) { // TODO: Get more accurate position of the error from UnicodeSet's return info. // UnicodeSet appears to not be reporting correctly at this time. - RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex()); + #ifdef RBBI_DEBUG + RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex()); + #endif error(localStatus); delete uset; return; @@ -1092,7 +1230,8 @@ void RBBIRuleScanner::scanSet() { // Verify that the set contains at least one code point. // - if (uset->charAt(0) == -1) { + U_ASSERT(uset!=NULL); + if (uset->isEmpty()) { // This set is empty. // Make it an error, because it almost certainly is not what the user wanted. // Also, avoids having to think about corner cases in the tree manipulation code @@ -1118,6 +1257,9 @@ void RBBIRuleScanner::scanSet() { RBBINode *n; n = pushNewNode(RBBINode::setRef); + if (U_FAILURE(*fRB->fStatus)) { + return; + } n->fFirstPos = startPos; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);