//
/*
***************************************************************************
-* Copyright (C) 2002-2004 International Business Machines Corporation *
+* Copyright (C) 2002-2008 International Business Machines Corporation *
* and others. All rights reserved. *
***************************************************************************
*/
fTrie = 0;
fTrieSize = 0;
fGroupCount = 0;
+ fSawBOF = FALSE;
}
// that is in no sets.
//
fRangeList = new RangeDescriptor(*fStatus); // will check for status here
+ if (fRangeList == NULL) {
+ *fStatus = U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
fRangeList->fStartChar = 0;
fRangeList->fEndChar = 0x10ffff;
// Find the set of non-overlapping ranges of characters
//
int ni;
- for (ni=0; ; ni++) {
+ for (ni=0; ; ni++) { // Loop over each of the UnicodeSets encountered in the input rules
usetNode = (RBBINode *)this->fRB->fUSetNodes->elementAt(ni);
if (usetNode==NULL) {
break;
// The groups are numbered, and these group numbers are the set of
// input symbols recognized by the run-time state machine.
//
+ // Numbering: # 0 (state table column 0) is unused.
+ // # 1 is reserved - table column 1 is for end-of-input
+ // # 2 is reserved - table column 2 is for beginning-in-input
+ // # 3 is the first range list.
+ //
RangeDescriptor *rlSearchRange;
for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) {
for (rlSearchRange=fRangeList; rlSearchRange != rlRange; rlSearchRange=rlSearchRange->fNext) {
}
if (rlRange->fNum == 0) {
fGroupCount ++;
- rlRange->fNum = fGroupCount;
+ rlRange->fNum = fGroupCount+2;
rlRange->setDictionaryFlag();
- addValToSets(rlRange->fIncludesSets, fGroupCount);
+ addValToSets(rlRange->fIncludesSets, fGroupCount+2);
}
}
+ // Handle input sets that contain the special string {eof}.
+ // Column 1 of the state table is reserved for EOF on input.
+ // Column 2 is reserved for before-the-start-input.
+ // (This column can be optimized away later if there are no rule
+ // references to {bof}.)
+ // Add this column value (1 or 2) to the equivalent expression
+ // subtree for each UnicodeSet that contains the string {eof}
+ // Because {bof} and {eof} are not a characters in the normal sense,
+ // they doesn't affect the computation of ranges or TRIE.
+ static const UChar eofUString[] = {0x65, 0x6f, 0x66, 0};
+ static const UChar bofUString[] = {0x62, 0x6f, 0x66, 0};
+
+ UnicodeString eofString(eofUString);
+ UnicodeString bofString(bofUString);
+ for (ni=0; ; ni++) { // Loop over each of the UnicodeSets encountered in the input rules
+ usetNode = (RBBINode *)this->fRB->fUSetNodes->elementAt(ni);
+ if (usetNode==NULL) {
+ break;
+ }
+ UnicodeSet *inputSet = usetNode->fInputSet;
+ if (inputSet->contains(eofString)) {
+ addValToSet(usetNode, 1);
+ }
+ if (inputSet->contains(bofString)) {
+ addValToSet(usetNode, 2);
+ fSawBOF = TRUE;
+ }
+ }
+
+
if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rgroup")) {printRangeGroups();}
if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "esets")) {printSets();}
//------------------------------------------------------------------------
//
// addValToSets Add a runtime-mapped input value to each uset from a
-// list of uset nodes.
+// list of uset nodes. (val corresponds to a state table column.)
// For each of the original Unicode sets - which correspond
// directly to uset nodes - a logically equivalent expression
// is constructed in terms of the remapped runtime input
for (ix=0; ix<sets->size(); ix++) {
RBBINode *usetNode = (RBBINode *)sets->elementAt(ix);
- RBBINode *leafNode = new RBBINode(RBBINode::leafChar);
- leafNode->fVal = (unsigned short)val;
- if (usetNode->fLeftChild == NULL) {
- usetNode->fLeftChild = leafNode;
- leafNode->fParent = usetNode;
- } else {
- // There are already input symbols present for this set.
- // Set up an OR node, with the previous stuff as the left child
- // and the new value as the right child.
- RBBINode *orNode = new RBBINode(RBBINode::opOr);
- orNode->fLeftChild = usetNode->fLeftChild;
- orNode->fRightChild = leafNode;
- orNode->fLeftChild->fParent = orNode;
- orNode->fRightChild->fParent = orNode;
- usetNode->fLeftChild = orNode;
- orNode->fParent = usetNode;
- }
+ addValToSet(usetNode, val);
}
}
+void RBBISetBuilder::addValToSet(RBBINode *usetNode, uint32_t val) {
+ RBBINode *leafNode = new RBBINode(RBBINode::leafChar);
+ if (leafNode == NULL) {
+ *fStatus = U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
+ leafNode->fVal = (unsigned short)val;
+ if (usetNode->fLeftChild == NULL) {
+ usetNode->fLeftChild = leafNode;
+ leafNode->fParent = usetNode;
+ } else {
+ // There are already input symbols present for this set.
+ // Set up an OR node, with the previous stuff as the left child
+ // and the new value as the right child.
+ RBBINode *orNode = new RBBINode(RBBINode::opOr);
+ if (orNode == NULL) {
+ *fStatus = U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
+ orNode->fLeftChild = usetNode->fLeftChild;
+ orNode->fRightChild = leafNode;
+ orNode->fLeftChild->fParent = orNode;
+ orNode->fRightChild->fParent = orNode;
+ usetNode->fLeftChild = orNode;
+ orNode->fParent = usetNode;
+ }
+}
//------------------------------------------------------------------------
//
-// getNumOutputSets
+// getNumCharCategories
//
//------------------------------------------------------------------------
int32_t RBBISetBuilder::getNumCharCategories() const {
- return fGroupCount + 1;
+ return fGroupCount + 3;
}
+//------------------------------------------------------------------------
+//
+// sawBOF
+//
+//------------------------------------------------------------------------
+UBool RBBISetBuilder::sawBOF() const {
+ return fSawBOF;
+}
+
//------------------------------------------------------------------------
//
void RangeDescriptor::split(UChar32 where, UErrorCode &status) {
U_ASSERT(where>fStartChar && where<=fEndChar);
RangeDescriptor *nr = new RangeDescriptor(*this, status);
- if (U_FAILURE(status)) {
- return;
- }
- /* test for NULL */
if(nr == 0) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
-
+ if (U_FAILURE(status)) {
+ delete nr;
+ return;
+ }
// RangeDescriptor copy constructor copies all fields.
// Only need to update those that are different after the split.
nr->fStartChar = where;
projects / apple / icu.git / blobdiff