X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/46f4442e9a5a4f3b98b7c1083586332f6a8a99a4..2ca993e82fb37b597a3c73ecd1586a139a6579c5:/icuSources/common/rbbiscan.cpp diff --git a/icuSources/common/rbbiscan.cpp b/icuSources/common/rbbiscan.cpp index 82147cd3..3a74834b 100644 --- a/icuSources/common/rbbiscan.cpp +++ b/icuSources/common/rbbiscan.cpp @@ -1,8 +1,7 @@ - // // file: rbbiscan.cpp // -// Copyright (C) 2002-2008, International Business Machines Corporation and others. +// Copyright (C) 2002-2016, International Business Machines Corporation and others. // All Rights Reserved. // // This file contains the Rule Based Break Iterator Rule Builder functions for @@ -23,7 +22,6 @@ #include "unicode/uchriter.h" #include "unicode/parsepos.h" #include "unicode/parseerr.h" -#include "util.h" #include "cmemory.h" #include "cstring.h" @@ -36,7 +34,6 @@ #include "uassert.h" - //------------------------------------------------------------------------------ // // Unicode Set init strings for each of the character classes needed for parsing a rule file. @@ -72,7 +69,7 @@ static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00}; // "any" U_CDECL_BEGIN static void U_CALLCONV RBBISetTable_deleter(void *p) { - U_NAMESPACE_QUALIFIER RBBISetTableEl *px = (U_NAMESPACE_QUALIFIER RBBISetTableEl *)p; + icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p; delete px->key; // Note: px->val is owned by the linked list "fSetsListHead" in scanner. // Don't delete the value nodes here. @@ -90,24 +87,27 @@ U_NAMESPACE_BEGIN RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) { fRB = rb; + fScanIndex = 0; + fNextIndex = 0; + fQuoteMode = FALSE; + fLineNum = 1; + fCharNum = 0; + fLastChar = 0; + + fStateTable = NULL; + fStack[0] = 0; fStackPtr = 0; - fStack[fStackPtr] = 0; - fNodeStackPtr = 0; - fRuleNum = 0; fNodeStack[0] = NULL; - - fSymbolTable = NULL; - fSetTable = NULL; - - fScanIndex = 0; - fNextIndex = 0; + fNodeStackPtr = 0; fReverseRule = FALSE; fLookAheadRule = FALSE; + fNoChainInRule = FALSE; - fLineNum = 1; - fCharNum = 0; - fQuoteMode = FALSE; + fSymbolTable = NULL; + fSetTable = NULL; + fRuleNum = 0; + fOptionStart = 0; // Do not check status until after all critical fields are sufficiently initialized // that the destructor can run cleanly. @@ -121,19 +121,20 @@ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) // all instances of RBBIRuleScanners. BUT this is quite a bit simpler, // and the time to build these few sets should be small compared to a // full break iterator build. - fRuleSets[kRuleSet_rule_char-128] = UnicodeSet(gRuleSet_rule_char_pattern, *rb->fStatus); - UnicodeSet *whitespaceSet = uprv_openRuleWhiteSpaceSet(rb->fStatus); - if (U_FAILURE(*rb->fStatus)) { - return; - } - fRuleSets[kRuleSet_white_space-128] = *whitespaceSet; - delete whitespaceSet; - fRuleSets[kRuleSet_name_char-128] = UnicodeSet(gRuleSet_name_char_pattern, *rb->fStatus); - fRuleSets[kRuleSet_name_start_char-128] = UnicodeSet(gRuleSet_name_start_char_pattern, *rb->fStatus); - fRuleSets[kRuleSet_digit_char-128] = UnicodeSet(gRuleSet_digit_char_pattern, *rb->fStatus); + fRuleSets[kRuleSet_rule_char-128] + = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern), *rb->fStatus); + // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:] + fRuleSets[kRuleSet_white_space-128]. + add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029); + fRuleSets[kRuleSet_name_char-128] + = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern), *rb->fStatus); + fRuleSets[kRuleSet_name_start_char-128] + = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus); + fRuleSets[kRuleSet_digit_char-128] + = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern), *rb->fStatus); if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) { // This case happens if ICU's data is missing. UnicodeSet tries to look up property - // names from the init string, can't find them, and claims an illegal arguement. + // names from the init string, can't find them, and claims an illegal argument. // Change the error so that the actual problem will be clearer to users. *rb->fStatus = U_BRK_INIT_ERROR; } @@ -207,11 +208,20 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) break; + case doNoChain: + // Scanned a '^' while on the rule start state. + fNoChainInRule = TRUE; + break; + + case doExprOrOperator: { fixOpStack(RBBINode::precOpCat); RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *orNode = pushNewNode(RBBINode::opOr); + if (U_FAILURE(*fRB->fStatus)) { + break; + } orNode->fLeftChild = operandNode; operandNode->fParent = orNode; } @@ -226,6 +236,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) fixOpStack(RBBINode::precOpCat); RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *catNode = pushNewNode(RBBINode::opCat); + if (U_FAILURE(*fRB->fStatus)) { + break; + } catNode->fLeftChild = operandNode; operandNode->fParent = catNode; } @@ -314,21 +327,40 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");} #endif U_ASSERT(fNodeStackPtr == 1); + RBBINode *thisRule = fNodeStack[fNodeStackPtr]; // If this rule includes a look-ahead '/', add a endMark node to the // expression tree. if (fLookAheadRule) { - RBBINode *thisRule = fNodeStack[fNodeStackPtr]; RBBINode *endNode = pushNewNode(RBBINode::endMark); RBBINode *catNode = pushNewNode(RBBINode::opCat); + if (U_FAILURE(*fRB->fStatus)) { + break; + } fNodeStackPtr -= 2; catNode->fLeftChild = thisRule; catNode->fRightChild = endNode; fNodeStack[fNodeStackPtr] = catNode; endNode->fVal = fRuleNum; endNode->fLookAheadEnd = TRUE; + thisRule = catNode; + + // TODO: Disable chaining out of look-ahead (hard break) rules. + // The break on rule match is forced, so there is no point in building up + // the state table to chain into another rule for a longer match. } + // Mark this node as being the root of a rule. + thisRule->fRuleRoot = TRUE; + + // Flag if chaining into this rule is wanted. + // + if (fRB->fChainRules && // If rule chaining is enabled globally via !!chain + !fNoChainInRule) { // and no '^' chain-in inhibit was on this rule + thisRule->fChainIn = TRUE; + } + + // All rule expressions are ORed together. // The ';' that terminates an expression really just functions as a '|' with // a low operator prededence. @@ -348,6 +380,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) RBBINode *thisRule = fNodeStack[fNodeStackPtr]; RBBINode *prevRules = *destRules; RBBINode *orNode = pushNewNode(RBBINode::opOr); + if (U_FAILURE(*fRB->fStatus)) { + break; + } orNode->fLeftChild = prevRules; prevRules->fParent = orNode; orNode->fRightChild = thisRule; @@ -362,6 +397,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) } fReverseRule = FALSE; // in preparation for the next rule. fLookAheadRule = FALSE; + fNoChainInRule = FALSE; fNodeStackPtr = 0; } break; @@ -388,6 +424,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *plusNode = pushNewNode(RBBINode::opPlus); + if (U_FAILURE(*fRB->fStatus)) { + break; + } plusNode->fLeftChild = operandNode; operandNode->fParent = plusNode; } @@ -397,6 +436,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *qNode = pushNewNode(RBBINode::opQuestion); + if (U_FAILURE(*fRB->fStatus)) { + break; + } qNode->fLeftChild = operandNode; operandNode->fParent = qNode; } @@ -406,6 +448,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *starNode = pushNewNode(RBBINode::opStar); + if (U_FAILURE(*fRB->fStatus)) { + break; + } starNode->fLeftChild = operandNode; operandNode->fParent = starNode; } @@ -419,7 +464,10 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) // sets that just happen to contain only one character. { n = pushNewNode(RBBINode::setRef); - findSetFor(fC.fChar, n); + if (U_FAILURE(*fRB->fStatus)) { + break; + } + findSetFor(UnicodeString(fC.fChar), n); n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); @@ -430,7 +478,10 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) // scanned a ".", meaning match any single character. { n = pushNewNode(RBBINode::setRef); - findSetFor(kAny, n); + if (U_FAILURE(*fRB->fStatus)) { + break; + } + findSetFor(UnicodeString(TRUE, kAny, 3), n); n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); @@ -440,6 +491,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) case doSlash: // Scanned a '/', which identifies a look-ahead break position in a rule. n = pushNewNode(RBBINode::lookAhead); + if (U_FAILURE(*fRB->fStatus)) { + break; + } n->fVal = fRuleNum; n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; @@ -451,6 +505,9 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) case doStartTagValue: // Scanned a '{', the opening delimiter for a tag value within a rule. n = pushNewNode(RBBINode::tag); + if (U_FAILURE(*fRB->fStatus)) { + break; + } n->fVal = 0; n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; @@ -489,6 +546,8 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) fRB->fChainRules = TRUE; } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) { fRB->fLBCMNoChain = TRUE; + } else if (opt == UNICODE_STRING("RINoChain", 9)) { + fRB->fRINoChain = TRUE; } else if (opt == UNICODE_STRING("forward", 7)) { fRB->fDefaultTree = &fRB->fForwardTree; } else if (opt == UNICODE_STRING("reverse", 7)) { @@ -561,7 +620,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action) returnVal = FALSE; break; } - return returnVal; + return returnVal && U_SUCCESS(*fRB->fStatus); } @@ -580,7 +639,7 @@ void RBBIRuleScanner::error(UErrorCode e) { fRB->fParseError->line = fLineNum; fRB->fParseError->offset = fCharNum; fRB->fParseError->preContext[0] = 0; - fRB->fParseError->preContext[0] = 0; + fRB->fParseError->postContext[0] = 0; } } } @@ -802,7 +861,7 @@ UChar32 RBBIRuleScanner::nextCharLL() { if (ch == chCR || ch == chNEL || ch == chLS || - ch == chLF && fLastChar != chCR) { + (ch == chLF && fLastChar != chCR)) { // Character is starting a new line. Bump up the line number, and // reset the column to 0. fLineNum++; @@ -963,7 +1022,7 @@ void RBBIRuleScanner::parse() { for (;;) { #ifdef RBBI_DEBUG - if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");} + if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);} #endif if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE && tableEl->fCharClass == fC.fChar) { // Table row specified an individual character, not a set, and @@ -992,6 +1051,7 @@ void RBBIRuleScanner::parse() { if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class && fC.fEscaped == FALSE && // char is not escaped && fC.fChar != (UChar32)-1) { // char is not EOF + U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets)); if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) { // Table row specified a character class, or set of characters, // and the current char matches it. @@ -1051,7 +1111,10 @@ void RBBIRuleScanner::parse() { if (fRB->fReverseTree == NULL) { fRB->fReverseTree = pushNewNode(RBBINode::opStar); RBBINode *operand = pushNewNode(RBBINode::setRef); - findSetFor(kAny, operand); + if (U_FAILURE(*fRB->fStatus)) { + return; + } + findSetFor(UnicodeString(TRUE, kAny, 3), operand); fRB->fReverseTree->fLeftChild = operand; operand->fParent = fRB->fReverseTree; fNodeStackPtr -= 2; @@ -1103,6 +1166,9 @@ void RBBIRuleScanner::printNodeStack(const char *title) { // //------------------------------------------------------------------------------ RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { + if (U_FAILURE(*fRB->fStatus)) { + return NULL; + } fNodeStackPtr++; if (fNodeStackPtr >= kStackSize) { error(U_BRK_INTERNAL_ERROR); @@ -1146,11 +1212,11 @@ void RBBIRuleScanner::scanSet() { pos.setIndex(fScanIndex); startPos = fScanIndex; UErrorCode localStatus = U_ZERO_ERROR; - uset = new UnicodeSet(fRB->fRules, pos, USET_IGNORE_SPACE, - fSymbolTable, - localStatus); + uset = new UnicodeSet(); if (uset == NULL) { localStatus = U_MEMORY_ALLOCATION_ERROR; + } else { + uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus); } if (U_FAILURE(localStatus)) { // TODO: Get more accurate position of the error from UnicodeSet's return info. @@ -1165,6 +1231,7 @@ void RBBIRuleScanner::scanSet() { // Verify that the set contains at least one code point. // + U_ASSERT(uset!=NULL); if (uset->isEmpty()) { // This set is empty. // Make it an error, because it almost certainly is not what the user wanted. @@ -1191,6 +1258,9 @@ void RBBIRuleScanner::scanSet() { RBBINode *n; n = pushNewNode(RBBINode::setRef); + if (U_FAILURE(*fRB->fStatus)) { + return; + } n->fFirstPos = startPos; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText);