//
// file: regexcmp.cpp
//
-// Copyright (C) 2002-2004 International Business Machines Corporation and others.
+// Copyright (C) 2002-2008 International Business Machines Corporation and others.
// All Rights Reserved.
//
// This file contains the ICU regular expression compiler, which is responsible
#include "uvectr32.h"
#include "uassert.h"
#include "ucln_in.h"
-#include "mutex.h"
+#include "uinvchar.h"
#include "regeximp.h"
#include "regexcst.h" // Contains state table for the regex pattern parser.
U_NAMESPACE_BEGIN
-
-
-
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Constructor.
//
-//----------------------------------------------------------------------------------------
-RegexCompile::RegexCompile(RegexPattern *rxp, UErrorCode &status) : fParenStack(status)
+//------------------------------------------------------------------------------
+RegexCompile::RegexCompile(RegexPattern *rxp, UErrorCode &status) :
+ fParenStack(status), fSetStack(status), fSetOpStack(status)
{
fStatus = &status;
fCharNum = 0;
fQuoteMode = FALSE;
fInBackslashQuote = FALSE;
- fModeFlags = fRXPat->fFlags;
+ fModeFlags = fRXPat->fFlags | 0x80000000;
fEOLComments = TRUE;
fMatchOpenParen = -1;
}
}
+static const UChar chAmp = 0x26; // '&'
+static const UChar chDash = 0x2d; // '-'
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Destructor
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
RegexCompile::~RegexCompile() {
}
-//---------------------------------------------------------------------------------
+static inline void addCategory(UnicodeSet *set, int32_t value, UErrorCode& ec) {
+ set->addAll(UnicodeSet().applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, value, ec));
+}
+
+//------------------------------------------------------------------------------
//
// Compile regex pattern. The state machine for rexexp pattern parsing is here.
// The state tables are hand-written in the file regexcst.txt,
// and converted to the form used here by a perl
// script regexcst.pl
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::compile(
const UnicodeString &pat, // Source pat to be compiled.
UParseError &pp, // Error position info
U_ASSERT(fRXPat->fPattern.length() == 0);
// Prepare the RegexPattern object to receive the compiled pattern.
- // TODO: remove per-instance field, and just use globals directly. (But check perf)
fRXPat->fPattern = pat;
fRXPat->fStaticSets = RegexStaticSets::gStaticSets->fPropSets;
fRXPat->fStaticSets8 = RegexStaticSets::gStaticSets->fPropSets8;
// the search will stop there, if not before.
//
tableEl = &gRuleParseStateTable[state];
- REGEX_SCAN_DEBUG_PRINTF(("char, line, col = (\'%c\', %d, %d) state=%s ",
+ REGEX_SCAN_DEBUG_PRINTF(("char, line, col = (\'%c\', %d, %d) state=%s ",
fC.fChar, fLineNum, fCharNum, RegexStateNames[state]));
for (;;) { // loop through table rows belonging to this state, looking for one
if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class &&
fC.fQuoted == FALSE && // char is not escaped &&
fC.fChar != (UChar32)-1) { // char is not EOF
- UnicodeSet *uniset = RegexStaticSets::gStaticSets->fRuleSets[tableEl->fCharClass-128];
- if (uniset->contains(fC.fChar)) {
+ if (RegexStaticSets::gStaticSets->fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
// Table row specified a character class, or set of characters,
// and the current char matches it.
break;
// We've found the row of the state table that matches the current input
// character from the rules string.
// Perform any action specified by this row in the state table.
- if (doParseActions((EParseAction)tableEl->fAction) == FALSE) {
+ if (doParseActions(tableEl->fAction) == FALSE) {
// Break out of the state machine loop if the
// the action signalled some kind of error, or
// the action was to exit, occurs on normal end-of-rules-input.
// state stack underflow
// This will occur if the user pattern has mis-matched parentheses,
// with extra close parens.
- //
+ //
fStackPtr++;
error(U_REGEX_MISMATCHED_PAREN);
}
}
+ if (U_FAILURE(*fStatus)) {
+ // Bail out if the pattern had errors.
+ // Set stack cleanup: a successful compile would have left it empty,
+ // but errors can leave temporary sets hanging around.
+ while (!fSetStack.empty()) {
+ delete (UnicodeSet *)fSetStack.pop();
+ }
+ return;
+ }
+
//
// The pattern has now been read and processed, and the compiled code generated.
//
//
// Optimization passes
- //
- matchStartType();
- OptDotStar();
+ //
+ matchStartType();
stripNOPs();
//
//
int32_t numSets = fRXPat->fSets->size();
fRXPat->fSets8 = new Regex8BitSet[numSets];
+ // Null pointer check.
+ if (fRXPat->fSets8 == NULL) {
+ e = *fStatus = U_MEMORY_ALLOCATION_ERROR;
+ return;
+ }
int32_t i;
for (i=0; i<numSets; i++) {
UnicodeSet *s = (UnicodeSet *)fRXPat->fSets->elementAt(i);
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// doParseAction Do some action during regex pattern parsing.
// Called by the parse state machine.
// in functions called from the parse actions defined here.
//
//
-//----------------------------------------------------------------------------------------
-UBool RegexCompile::doParseActions(EParseAction action)
+//------------------------------------------------------------------------------
+UBool RegexCompile::doParseActions(int32_t action)
{
UBool returnVal = TRUE;
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_STATE_SAVE, 2), *fStatus);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_JMP, 3), *fStatus);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_FAIL, 0), *fStatus);
- fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
- fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
- fParenStack.push(-1, *fStatus); // Begin a Paren Stack Frame
- fParenStack.push( 3, *fStatus); // Push location of first NOP
+ // Standard open nonCapture paren action emits the two NOPs and
+ // sets up the paren stack frame.
+ doParseActions(doOpenNonCaptureParen);
break;
case doPatFinish:
case doOpenAtomicParen:
// Open Atomic Paren. (?>
// Compile to a
- // - NOP, which later may be replaced if the parenthesized group
+ // - NOP, which later may be replaced if the parenthesized group
// has a quantifier, followed by
// - STO_SP save state stack position, so it can be restored at the ")"
// - NOP, which may later be replaced by a save-state if there
case doOpenLookAhead:
// Positive Look-ahead (?= stuff )
+ //
+ // Note: Addition of transparent input regions, with the need to
+ // restore the original regions when failing out of a lookahead
+ // block, complicated this sequence. Some conbined opcodes
+ // might make sense - or might not, lookahead aren't that common.
+ //
+ // Caution: min match length optimization knows about this
+ // sequence; don't change without making updates there too.
+ //
// Compiles to
- // 1 START_LA dataLoc
- // 2. NOP reserved for use by quantifiers on the block.
+ // 1 START_LA dataLoc Saves SP, Input Pos
+ // 2. STATE_SAVE 4 on failure of lookahead, goto 4
+ // 3 JMP 6 continue ...
+ //
+ // 4. LA_END Look Ahead failed. Restore regions.
+ // 5. BACKTRACK and back track again.
+ //
+ // 6. NOP reserved for use by quantifiers on the block.
// Look-ahead can't have quantifiers, but paren stack
// compile time conventions require the slot anyhow.
- // 3. NOP may be replaced if there is are '|' ops in the block.
- // 4. code for parenthesized stuff.
- // 5. ENDLA
- //
+ // 7. NOP may be replaced if there is are '|' ops in the block.
+ // 8. code for parenthesized stuff.
+ // 9. LA_END
+ //
// Two data slots are reserved, for saving the stack ptr and the input position.
{
int32_t dataLoc = fRXPat->fDataSize;
- fRXPat->fDataSize += 2;
+ fRXPat->fDataSize += 2;
int32_t op = URX_BUILD(URX_LA_START, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
+ op = URX_BUILD(URX_STATE_SAVE, fRXPat->fCompiledPat->size()+ 2);
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+
+ op = URX_BUILD(URX_JMP, fRXPat->fCompiledPat->size()+ 3);
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+
+ op = URX_BUILD(URX_LA_END, dataLoc);
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+
+ op = URX_BUILD(URX_BACKTRACK, 0);
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+
op = URX_BUILD(URX_NOP, 0);
fRXPat->fCompiledPat->addElement(op, *fStatus);
fRXPat->fCompiledPat->addElement(op, *fStatus);
// On the Parentheses stack, start a new frame and add the postions
- // of the NOPs.
+ // of the NOPs.
fParenStack.push(fModeFlags, *fStatus); // Match mode state
fParenStack.push(lookAhead, *fStatus); // Frame type.
fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location
// 3. NOP // Std. Open Paren sequence, for possible '|'
// 4. code for parenthesized stuff.
// 5. END_LA // Cut back stack, remove saved state from step 2.
- // 6. FAIL // code in block succeeded, so neg. lookahead fails.
- // 7. ...
+ // 6. BACKTRACK // code in block succeeded, so neg. lookahead fails.
+ // 7. END_LA // Restore match region, in case look-ahead was using
+ // an alternate (transparent) region.
{
int32_t dataLoc = fRXPat->fDataSize;
- fRXPat->fDataSize += 2;
+ fRXPat->fDataSize += 2;
int32_t op = URX_BUILD(URX_LA_START, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
fRXPat->fCompiledPat->addElement(op, *fStatus);
// On the Parentheses stack, start a new frame and add the postions
- // of the StateSave and NOP.
+ // of the StateSave and NOP.
fParenStack.push(fModeFlags, *fStatus); // Match mode state
- fParenStack.push( negLookAhead, *fStatus); // Frame type
+ fParenStack.push(negLookAhead, *fStatus); // Frame type
fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The STATE_SAVE location
fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP location
-
+
// Instructions #5 and #6 will be added when the ')' is encountered.
}
break;
// 0: Stack ptr on entry
// 1: Input Index on entry
// 2: Start index of match current match attempt.
- // 3: Original Input String len.
+ // 3: Original Input String len.
// Allocate data space
int32_t dataLoc = fRXPat->fDataSize;
- fRXPat->fDataSize += 4;
-
+ fRXPat->fDataSize += 4;
+
// Emit URX_LB_START
int32_t op = URX_BUILD(URX_LB_START, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
-
+
// Emit URX_LB_CONT
op = URX_BUILD(URX_LB_CONT, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
fRXPat->fCompiledPat->addElement(0, *fStatus); // MinMatchLength. To be filled later.
fRXPat->fCompiledPat->addElement(0, *fStatus); // MaxMatchLength. To be filled later.
-
+
// Emit the NOP
op = URX_BUILD(URX_NOP, 0);
fRXPat->fCompiledPat->addElement(op, *fStatus);
fRXPat->fCompiledPat->addElement(op, *fStatus);
-
+
// On the Parentheses stack, start a new frame and add the postions
- // of the URX_LB_CONT and the NOP.
+ // of the URX_LB_CONT and the NOP.
fParenStack.push(fModeFlags, *fStatus); // Match mode state
fParenStack.push(lookBehind, *fStatus); // Frame type
fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location
fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The 2nd NOP location
-
+
// The final two instructions will be added when the ')' is encountered.
}
// 0: Stack ptr on entry
// 1: Input Index on entry
// 2: Start index of match current match attempt.
- // 3: Original Input String len.
+ // 3: Original Input String len.
// Allocate data space
int32_t dataLoc = fRXPat->fDataSize;
- fRXPat->fDataSize += 4;
-
+ fRXPat->fDataSize += 4;
+
// Emit URX_LB_START
int32_t op = URX_BUILD(URX_LB_START, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
-
+
// Emit URX_LBN_CONT
op = URX_BUILD(URX_LBN_CONT, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
fRXPat->fCompiledPat->addElement(0, *fStatus); // MinMatchLength. To be filled later.
fRXPat->fCompiledPat->addElement(0, *fStatus); // MaxMatchLength. To be filled later.
fRXPat->fCompiledPat->addElement(0, *fStatus); // Continue Loc. To be filled later.
-
+
// Emit the NOP
op = URX_BUILD(URX_NOP, 0);
fRXPat->fCompiledPat->addElement(op, *fStatus);
fRXPat->fCompiledPat->addElement(op, *fStatus);
-
+
// On the Parentheses stack, start a new frame and add the postions
- // of the URX_LB_CONT and the NOP.
+ // of the URX_LB_CONT and the NOP.
fParenStack.push(fModeFlags, *fStatus); // Match mode state
fParenStack.push(lookBehindN, *fStatus); // Frame type
fParenStack.push(fRXPat->fCompiledPat->size()-2, *fStatus); // The first NOP location
fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The 2nd NOP location
-
+
// The final two instructions will be added when the ')' is encountered.
}
break;
}
if (URX_TYPE(repeatedOp) == URX_DOTANY ||
- URX_TYPE(repeatedOp) == URX_DOTANY_ALL) {
+ URX_TYPE(repeatedOp) == URX_DOTANY_ALL ||
+ URX_TYPE(repeatedOp) == URX_DOTANY_UNIX) {
// Emit Optimized code for .+ operations.
int32_t loopOpI = URX_BUILD(URX_LOOP_DOT_I, 0);
if (URX_TYPE(repeatedOp) == URX_DOTANY_ALL) {
- // URX_LOOP_DOT_I operand is a flag indicating . matches any mode.
+ // URX_LOOP_DOT_I operand is a flag indicating ". matches any" mode.
loopOpI |= 1;
}
+ if (fModeFlags & UREGEX_UNIX_LINES) {
+ loopOpI |= 2;
+ }
fRXPat->fCompiledPat->addElement(loopOpI, *fStatus);
frameLoc = fRXPat->fFrameSize;
fRXPat->fFrameSize++;
// 2. LOOP_C stack location
// ...
//
- // Or if this is a .*
+ // Or if this is a .*
// 1. LOOP_DOT_I (. matches all mode flag)
// 2. LOOP_C stack location
//
int32_t repeatedOp = fRXPat->fCompiledPat->elementAti(topLoc);
if (URX_TYPE(repeatedOp) == URX_SETREF) {
- // Emit optimized code for a [char set]*
+ // Emit optimized code for a [char set]*
int32_t loopOpI = URX_BUILD(URX_LOOP_SR_I, URX_VAL(repeatedOp));
fRXPat->fCompiledPat->setElementAt(loopOpI, topLoc);
dataLoc = fRXPat->fFrameSize;
}
if (URX_TYPE(repeatedOp) == URX_DOTANY ||
- URX_TYPE(repeatedOp) == URX_DOTANY_ALL) {
+ URX_TYPE(repeatedOp) == URX_DOTANY_ALL ||
+ URX_TYPE(repeatedOp) == URX_DOTANY_UNIX) {
// Emit Optimized code for .* operations.
int32_t loopOpI = URX_BUILD(URX_LOOP_DOT_I, 0);
if (URX_TYPE(repeatedOp) == URX_DOTANY_ALL) {
// URX_LOOP_DOT_I operand is a flag indicating . matches any mode.
loopOpI |= 1;
}
+ if ((fModeFlags & UREGEX_UNIX_LINES) != 0) {
+ loopOpI |= 2;
+ }
fRXPat->fCompiledPat->setElementAt(loopOpI, topLoc);
dataLoc = fRXPat->fFrameSize;
fRXPat->fFrameSize++;
fRXPat->fCompiledPat->setElementAt(op, saveStateLoc+1);
jmpOp = URX_BUILD(URX_JMP_SAV_X, saveStateLoc+2);
}
-
+
// Locate the position in the compiled pattern where the match will continue
// after completing the *. (4 or 5 in the comment above)
int32_t continueLoc = fRXPat->fCompiledPat->size()+1;
// Finished scanning a Possessive {lower,upper}+ interval. Generate the code for it.
{
// Remember the loc for the top of the block being looped over.
- // (Can not reserve a slot in the compiled pattern at this time, becuase
- // compileInterval needs to reserve also, and blockTopLoc can only reserve
+ // (Can not reserve a slot in the compiled pattern at this time, because
+ // compileInterval needs to reserve also, and blockTopLoc can only reserve
// once per block.)
int32_t topLoc = blockTopLoc(FALSE);
break;
case doLiteralChar:
- // We've just scanned a "normal" character from the pattern,
+ // We've just scanned a "normal" character from the pattern,
literalChar(fC.fChar);
break;
+ case doEscapedLiteralChar:
+ // We've just scanned an backslashed escaped character with no
+ // special meaning. It represents itself.
+ if ((fModeFlags & UREGEX_ERROR_ON_UNKNOWN_ESCAPES) != 0 &&
+ ((fC.fChar >= 0x41 && fC.fChar<= 0x5A) || // in [A-Z]
+ (fC.fChar >= 0x61 && fC.fChar <= 0x7a))) { // in [a-z]
+ error(U_REGEX_BAD_ESCAPE_SEQUENCE);
+ }
+ literalChar(fC.fChar);
+ break;
+
case doDotAny:
// scanned a ".", match any single character.
int32_t op;
if (fModeFlags & UREGEX_DOTALL) {
op = URX_BUILD(URX_DOTANY_ALL, 0);
+ } else if (fModeFlags & UREGEX_UNIX_LINES) {
+ op = URX_BUILD(URX_DOTANY_UNIX, 0);
} else {
op = URX_BUILD(URX_DOTANY, 0);
}
}
break;
- case doCaret:
+ case doCaret:
{
- int32_t op = (fModeFlags & UREGEX_MULTILINE)? URX_CARET_M : URX_CARET;
+ int32_t op = 0;
+ if ( (fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) {
+ op = URX_CARET;
+ } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) {
+ op = URX_CARET_M;
+ } else if ((fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) {
+ op = URX_CARET; // Only testing true start of input.
+ } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) {
+ op = URX_CARET_M_UNIX;
+ }
fRXPat->fCompiledPat->addElement(URX_BUILD(op, 0), *fStatus);
}
break;
-
- case doDollar:
+ case doDollar:
{
- int32_t op = (fModeFlags & UREGEX_MULTILINE)? URX_DOLLAR_M : URX_DOLLAR;
+ int32_t op = 0;
+ if ( (fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) {
+ op = URX_DOLLAR;
+ } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) {
+ op = URX_DOLLAR_M;
+ } else if ((fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) {
+ op = URX_DOLLAR_D;
+ } else if ((fModeFlags & UREGEX_MULTILINE) != 0 && (fModeFlags & UREGEX_UNIX_LINES) != 0) {
+ op = URX_DOLLAR_MD;
+ }
fRXPat->fCompiledPat->addElement(URX_BUILD(op, 0), *fStatus);
}
break;
}
break;
-
- case doScanUnicodeSet:
+ case doNamedChar:
{
- UnicodeSet *theSet = scanSet();
- compileSet(theSet);
+ UChar32 c = scanNamedChar();
+ literalChar(c);
}
break;
-
- case doEnterQuoteMode:
- // Just scanned a \Q. Put character scanner into quote mode.
- fQuoteMode = TRUE;
- break;
+
case doBackRef:
// BackReference. Somewhat unusual in that the front-end can not completely parse
break;
}
c = peekCharLL();
- if (RegexStaticSets::gStaticSets->fRuleDigits->contains(c) == FALSE) {
+ if (RegexStaticSets::gStaticSets->fRuleDigitsAlias->contains(c) == FALSE) {
break;
}
nextCharLL();
}
// Scan of the back reference in the source regexp is complete. Now generate
- // the compiled code for it.
+ // the compiled code for it.
// Because capture groups can be forward-referenced by back-references,
// we fill the operand with the capture group number. At the end
// of compilation, it will be changed to the variable's location.
// 6. ...
//
// Note: TODO: This is pretty inefficient. A mass of saved state is built up
- // then unconditionally discarded. Perhaps introduce a new opcode
+ // then unconditionally discarded. Perhaps introduce a new opcode. Ticket 6056
//
{
// Emit the STO_SP
// Emit the STATE_SAVE
op = URX_BUILD(URX_STATE_SAVE, fRXPat->fCompiledPat->size()+2);
fRXPat->fCompiledPat->addElement(op, *fStatus);
-
+
// Emit the JMP
op = URX_BUILD(URX_JMP, topLoc+1);
fRXPat->fCompiledPat->addElement(op, *fStatus);
op = URX_BUILD(URX_STATE_SAVE, L7);
fRXPat->fCompiledPat->setElementAt(op, topLoc+1);
- // Append the JMP operation.
+ // Append the JMP operation.
op = URX_BUILD(URX_JMP, topLoc+1);
fRXPat->fCompiledPat->addElement(op, *fStatus);
int32_t bit = 0;
switch (fC.fChar) {
case 0x69: /* 'i' */ bit = UREGEX_CASE_INSENSITIVE; break;
+ case 0x64: /* 'd' */ bit = UREGEX_UNIX_LINES; break;
case 0x6d: /* 'm' */ bit = UREGEX_MULTILINE; break;
case 0x73: /* 's' */ bit = UREGEX_DOTALL; break;
+ case 0x75: /* 'u' */ bit = 0; /* Unicode casing */ break;
case 0x77: /* 'w' */ bit = UREGEX_UWORD; break;
case 0x78: /* 'x' */ bit = UREGEX_COMMENTS; break;
case 0x2d: /* '-' */ fSetModeFlag = FALSE; break;
case doSetMatchMode:
// We've got a (?i) or similar. The match mode is being changed, but
// the change is not scoped to a parenthesized block.
+ U_ASSERT(fNewModeFlags < 0);
fModeFlags = fNewModeFlags;
// Prevent any string from spanning across the change of match mode.
- // Otherwise the pattern "abc(?i)def" would make a single string of "abcdef"
- fixLiterals();
+ // Otherwise the pattern "abc(?i)def" would make a single string of "abcdef"
+ fixLiterals();
break;
fParenStack.push(fRXPat->fCompiledPat->size()-1, *fStatus); // The second NOP
// Set the current mode flags to the new values.
+ U_ASSERT(fNewModeFlags < 0);
fModeFlags = fNewModeFlags;
}
break;
break;
+ case doSetAddAmp:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ set->add(chAmp);
+ }
+ break;
+
+ case doSetAddDash:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ set->add(chDash);
+ }
+ break;
+
+ case doSetBackslash_s:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ set->addAll(*RegexStaticSets::gStaticSets->fPropSets[URX_ISSPACE_SET]);
+ break;
+ }
+
+ case doSetBackslash_S:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ UnicodeSet SSet(*RegexStaticSets::gStaticSets->fPropSets[URX_ISSPACE_SET]);
+ SSet.complement();
+ set->addAll(SSet);
+ break;
+ }
+
+ case doSetBackslash_d:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ // TODO - make a static set, ticket 6058.
+ addCategory(set, U_GC_ND_MASK, *fStatus);
+ break;
+ }
+
+ case doSetBackslash_D:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ UnicodeSet digits;
+ // TODO - make a static set, ticket 6058.
+ digits.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ND_MASK, *fStatus);
+ digits.complement();
+ set->addAll(digits);
+ break;
+ }
+
+ case doSetBackslash_w:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ set->addAll(*RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET]);
+ break;
+ }
+
+ case doSetBackslash_W:
+ {
+ UnicodeSet *set = (UnicodeSet *)fSetStack.peek();
+ UnicodeSet SSet(*RegexStaticSets::gStaticSets->fPropSets[URX_ISWORD_SET]);
+ SSet.complement();
+ set->addAll(SSet);
+ break;
+ }
+
+ case doSetBegin:
+ fSetStack.push(new UnicodeSet(), *fStatus);
+ fSetOpStack.push(setStart, *fStatus);
+ if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) {
+ fSetOpStack.push(setCaseClose, *fStatus);
+ }
+ break;
+
+ case doSetBeginDifference1:
+ // We have scanned something like [[abc]-[
+ // Set up a new UnicodeSet for the set beginning with the just-scanned '['
+ // Push a Difference operator, which will cause the new set to be subtracted from what
+ // went before once it is created.
+ setPushOp(setDifference1);
+ fSetOpStack.push(setStart, *fStatus);
+ if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) {
+ fSetOpStack.push(setCaseClose, *fStatus);
+ }
+ break;
+
+ case doSetBeginIntersection1:
+ // We have scanned something like [[abc]&[
+ // Need both the '&' operator and the open '[' operator.
+ setPushOp(setIntersection1);
+ fSetOpStack.push(setStart, *fStatus);
+ if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) {
+ fSetOpStack.push(setCaseClose, *fStatus);
+ }
+ break;
+
+ case doSetBeginUnion:
+ // We have scanned something like [[abc][
+ // Need to handle the union operation explicitly [[abc] | [
+ setPushOp(setUnion);
+ fSetOpStack.push(setStart, *fStatus);
+ if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) {
+ fSetOpStack.push(setCaseClose, *fStatus);
+ }
+ break;
+
+ case doSetDifference2:
+ // We have scanned something like [abc--
+ // Consider this to unambiguously be a set difference operator.
+ setPushOp(setDifference2);
+ break;
+
+ case doSetEnd:
+ // Have encountered the ']' that closes a set.
+ // Force the evaluation of any pending operations within this set,
+ // leave the completed set on the top of the set stack.
+ {
+ setEval(setEnd);
+ int32_t setOp = fSetOpStack.popi();
+ U_ASSERT(setOp==setStart);
+ break;
+ }
+
+ case doSetFinish:
+ {
+ // Finished a complete set expression, including all nested sets.
+ // The close bracket has already triggered clearing out pending set operators,
+ // the operator stack should be empty and the operand stack should have just
+ // one entry, the result set.
+ U_ASSERT(fSetOpStack.empty());
+ UnicodeSet *theSet = (UnicodeSet *)fSetStack.pop();
+ U_ASSERT(fSetStack.empty());
+ compileSet(theSet);
+ break;
+ }
+
+ case doSetIntersection2:
+ // Have scanned something like [abc&&
+ setPushOp(setIntersection2);
+ break;
+
+ case doSetLiteral:
+ // Union the just-scanned literal character into the set being built.
+ // This operation is the highest precedence set operation, so we can always do
+ // it immediately, without waiting to see what follows. It is necessary to perform
+ // any pending '-' or '&' operation first, because these have the same precedence
+ // as union-ing in a literal'
+ {
+ setEval(setUnion);
+ UnicodeSet *s = (UnicodeSet *)fSetStack.peek();
+ s->add(fC.fChar);
+ fLastSetLiteral = fC.fChar;
+ break;
+ }
+
+ case doSetLiteralEscaped:
+ // A back-slash escaped literal character was encountered.
+ // Processing is the same as with setLiteral, above, with the addition of
+ // the optional check for errors on escaped ASCII letters.
+ {
+ if ((fModeFlags & UREGEX_ERROR_ON_UNKNOWN_ESCAPES) != 0 &&
+ ((fC.fChar >= 0x41 && fC.fChar<= 0x5A) || // in [A-Z]
+ (fC.fChar >= 0x61 && fC.fChar <= 0x7a))) { // in [a-z]
+ error(U_REGEX_BAD_ESCAPE_SEQUENCE);
+ }
+ setEval(setUnion);
+ UnicodeSet *s = (UnicodeSet *)fSetStack.peek();
+ s->add(fC.fChar);
+ fLastSetLiteral = fC.fChar;
+ break;
+ }
+
+ case doSetNamedChar:
+ // Scanning a \N{UNICODE CHARACTER NAME}
+ // Aside from the source of the character, the processing is identical to doSetLiteral,
+ // above.
+ {
+ UChar32 c = scanNamedChar();
+ setEval(setUnion);
+ UnicodeSet *s = (UnicodeSet *)fSetStack.peek();
+ s->add(c);
+ fLastSetLiteral = c;
+ break;
+ }
+
+ case doSetNamedRange:
+ // We have scanned literal-\N{CHAR NAME}. Add the range to the set.
+ // The left character is already in the set, and is saved in fLastSetLiteral.
+ // The right side needs to be picked up, the scan is at the 'N'.
+ // Lower Limit > Upper limit being an error matches both Java
+ // and ICU UnicodeSet behavior.
+ {
+ UChar32 c = scanNamedChar();
+ if (U_SUCCESS(*fStatus) && fLastSetLiteral > c) {
+ error(U_REGEX_INVALID_RANGE);
+ }
+ UnicodeSet *s = (UnicodeSet *)fSetStack.peek();
+ s->add(fLastSetLiteral, c);
+ fLastSetLiteral = c;
+ break;
+ }
+
+
+ case doSetNegate:
+ // Scanned a '^' at the start of a set.
+ // Push the negation operator onto the set op stack.
+ // A twist for case-insensitive matching:
+ // the case closure operation must happen _before_ negation.
+ // But the case closure operation will already be on the stack if it's required.
+ // This requires checking for case closure, and swapping the stack order
+ // if it is present.
+ {
+ int32_t tosOp = fSetOpStack.peeki();
+ if (tosOp == setCaseClose) {
+ fSetOpStack.popi();
+ fSetOpStack.push(setNegation, *fStatus);
+ fSetOpStack.push(setCaseClose, *fStatus);
+ } else {
+ fSetOpStack.push(setNegation, *fStatus);
+ }
+ }
+ break;
+
+ case doSetNoCloseError:
+ error(U_REGEX_MISSING_CLOSE_BRACKET);
+ break;
+
+ case doSetOpError:
+ error(U_REGEX_RULE_SYNTAX); // -- or && at the end of a set. Illegal.
+ break;
+
+ case doSetPosixProp:
+ {
+ UnicodeSet *s = scanPosixProp();
+ if (s != NULL) {
+ UnicodeSet *tos = (UnicodeSet *)fSetStack.peek();
+ tos->addAll(*s);
+ delete s;
+ } // else error. scanProp() reported the error status already.
+ }
+ break;
+
+ case doSetProp:
+ // Scanned a \p \P within [brackets].
+ {
+ UnicodeSet *s = scanProp();
+ if (s != NULL) {
+ UnicodeSet *tos = (UnicodeSet *)fSetStack.peek();
+ tos->addAll(*s);
+ delete s;
+ } // else error. scanProp() reported the error status already.
+ }
+ break;
+
+
+ case doSetRange:
+ // We have scanned literal-literal. Add the range to the set.
+ // The left character is already in the set, and is saved in fLastSetLiteral.
+ // The right side is the current character.
+ // Lower Limit > Upper limit being an error matches both Java
+ // and ICU UnicodeSet behavior.
+ {
+ if (fLastSetLiteral > fC.fChar) {
+ error(U_REGEX_INVALID_RANGE);
+ }
+ UnicodeSet *s = (UnicodeSet *)fSetStack.peek();
+ s->add(fLastSetLiteral, fC.fChar);
+ break;
+ }
+
default:
U_ASSERT(FALSE);
}
return returnVal;
-};
+}
fRXPat->fLiteralText.append(c);
return;
}
-
+
// We are adding onto an existing string
fRXPat->fLiteralText.append(c);
opType = URX_TYPE(op);
U_ASSERT(opType == URX_ONECHAR || opType == URX_ONECHAR_I || opType == URX_STRING_LEN);
- // If the most recently emitted op is a URX_ONECHAR,
+ // If the most recently emitted op is a URX_ONECHAR,
if (opType == URX_ONECHAR || opType == URX_ONECHAR_I) {
if (U16_IS_TRAIL(c) && U16_IS_LEAD(URX_VAL(op))) {
// The most recently emitted op is a ONECHAR that was the first half
fRXPat->fCompiledPat->setElementAt(op, patternLoc);
return;
}
-
+
// The most recently emitted op is a ONECHAR.
// We've now received another adjacent char. Change the ONECHAR op
// to a string op.
op = URX_BUILD(URX_STRING_LEN, 0);
fRXPat->fCompiledPat->addElement(op, *fStatus);
}
-
+
// The pattern contains a URX_SRING / URX_STRING_LEN. Update the
// string length to reflect the new char we just added to the string.
stringLen = fRXPat->fLiteralText.length() - fStringOpStart;
UChar32 nextToLastChar;
int32_t stringLen;
- fStringOpStart = -1;
+ fStringOpStart = -1;
if (!split) {
return;
}
// separate the last char from the rest of the string.
// If the last operation from the compiled pattern is not a string,
- // nothing needs to be done
+ // nothing needs to be done
op = fRXPat->fCompiledPat->lastElementi();
opType = URX_TYPE(op);
if (opType != URX_STRING_LEN) {
// the compiled pattern. (Negative values are frame boundaries, and don't need fixing.)
for (loc=0; loc<fParenStack.size(); loc++) {
int32_t x = fParenStack.elementAti(loc);
+ U_ASSERT(x < code->size());
if (x>where) {
x++;
fParenStack.setElementAt(x, loc);
//
// parameter reserveLoc : TRUE - ensure that there is space to add an opcode
// at the returned location.
-// FALSE - just return the address,
+// FALSE - just return the address,
// do not reserve a location there.
//
//------------------------------------------------------------------------------
// This function is called both when encountering a
// real ) and at the end of the pattern.
//
-//-------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::handleCloseParen() {
int32_t patIdx;
int32_t patOp;
// At the close of any parenthesized block, restore the match mode flags to
// the value they had at the open paren. Saved value is
- // at the top of the paren stack.
+ // at the top of the paren stack.
fModeFlags = fParenStack.popi();
-
+ U_ASSERT(fModeFlags < 0);
+
// DO any additional fixups, depending on the specific kind of
// parentesized grouping this is
case lookAhead:
{
- int32_t startOp = fRXPat->fCompiledPat->elementAti(fMatchOpenParen-1);
+ int32_t startOp = fRXPat->fCompiledPat->elementAti(fMatchOpenParen-5);
U_ASSERT(URX_TYPE(startOp) == URX_LA_START);
int32_t dataLoc = URX_VAL(startOp);
int32_t op = URX_BUILD(URX_LA_END, dataLoc);
int32_t dataLoc = URX_VAL(startOp);
int32_t op = URX_BUILD(URX_LA_END, dataLoc);
fRXPat->fCompiledPat->addElement(op, *fStatus);
- op = URX_BUILD(URX_FAIL, 0);
+ op = URX_BUILD(URX_BACKTRACK, 0);
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+ op = URX_BUILD(URX_LA_END, 0);
fRXPat->fCompiledPat->addElement(op, *fStatus);
// Patch the URX_SAVE near the top of the block.
+ // The destination of the SAVE is the final LA_END that was just added.
int32_t saveOp = fRXPat->fCompiledPat->elementAti(fMatchOpenParen);
U_ASSERT(URX_TYPE(saveOp) == URX_STATE_SAVE);
- int32_t dest = fRXPat->fCompiledPat->size();
+ int32_t dest = fRXPat->fCompiledPat->size()-1;
saveOp = URX_BUILD(URX_STATE_SAVE, dest);
fRXPat->fCompiledPat->setElementAt(saveOp, fMatchOpenParen);
}
case lookBehind:
{
// See comment at doOpenLookBehind.
-
+
// Append the URX_LB_END and URX_LA_END to the compiled pattern.
int32_t startOp = fRXPat->fCompiledPat->elementAti(fMatchOpenParen-4);
U_ASSERT(URX_TYPE(startOp) == URX_LB_START);
case lookBehindN:
{
// See comment at doOpenLookBehindNeg.
-
+
// Append the URX_LBN_END to the compiled pattern.
int32_t startOp = fRXPat->fCompiledPat->elementAti(fMatchOpenParen-5);
U_ASSERT(URX_TYPE(startOp) == URX_LB_START);
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// compileSet Compile the pattern operations for a reference to a
// UnicodeSet.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::compileSet(UnicodeSet *theSet)
{
if (theSet == NULL) {
return;
}
+ // Remove any strings from the set.
+ // There shoudn't be any, but just in case.
+ // (Case Closure can add them; if we had a simple case closure avaialble that
+ // ignored strings, that would be better.)
+ theSet->removeAllStrings();
int32_t setSize = theSet->size();
UChar32 firstSetChar = theSet->charAt(0);
- if (firstSetChar == -1) {
- // Sets that contain only strings, but no individual chars,
- // will end up here.
- error(U_REGEX_SET_CONTAINS_STRING);
- setSize = 0;
- }
switch (setSize) {
- case 0:
+ case 0:
{
- // Set of no elements. Always fails to match.
+ // Set of no elements. Always fails to match.
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_BACKTRACK, 0), *fStatus);
delete theSet;
}
break;
-
+
case 1:
{
// The set contains only a single code point. Put it into
delete theSet;
}
break;
-
- default:
+
+ default:
{
// The set contains two or more chars. (the normal case)
// Put it into the compiled pattern as a set.
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// compileInterval Generate the code for a {min, max} style interval quantifier.
// Except for the specific opcodes used, the code is the same
// 2 min count
// 3 max count (-1 for unbounded)
// 4 ... block to be iterated over
-// 5 CTR_LOOP
-//
-// In
-//----------------------------------------------------------------------------------------
+// 5 CTR_LOOP
+//
+// In
+//------------------------------------------------------------------------------
void RegexCompile::compileInterval(int32_t InitOp, int32_t LoopOp)
{
// The CTR_INIT op at the top of the block with the {n,m} quantifier takes
//
int32_t op = fRXPat->fCompiledPat->elementAti(topOfBlock);
- // Compute the pattern location where the inline sequence
+ // Compute the pattern location where the inline sequence
// will end, and set up the state save op that will be needed.
- //
+ //
int32_t endOfSequenceLoc = fRXPat->fCompiledPat->size()-1
+ fIntervalUpper + (fIntervalUpper-fIntervalLow);
int32_t saveOp = URX_BUILD(URX_STATE_SAVE, endOfSequenceLoc);
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// matchStartType Determine how a match can start.
// Used to optimize find() operations.
// op where the min match coming in is zero, add that ops possible
// starting matches to the possible starts for the overall pattern.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::matchStartType() {
if (U_FAILURE(*fStatus)) {
return;
// If the op we are now at was the destination of a branch in the pattern,
// and that path has a shorter minimum length than the current accumulated value,
// replace the current accumulated value.
- U_ASSERT(currentLen>=0 && currentLen < INT32_MAX);
if (forwardedLength.elementAti(loc) < currentLen) {
currentLen = forwardedLength.elementAti(loc);
U_ASSERT(currentLen>=0 && currentLen < INT32_MAX);
// Ops that don't change the total length matched
case URX_RESERVED_OP:
case URX_END:
+ case URX_FAIL:
case URX_STRING_LEN:
case URX_NOP:
case URX_START_CAPTURE:
case URX_BACKSLASH_G:
case URX_BACKSLASH_Z:
case URX_DOLLAR:
+ case URX_DOLLAR_M:
+ case URX_DOLLAR_D:
+ case URX_DOLLAR_MD:
case URX_RELOC_OPRND:
case URX_STO_INP_LOC:
- case URX_DOLLAR_M:
- case URX_BACKTRACK:
case URX_BACKREF: // BackRef. Must assume that it might be a zero length match
case URX_BACKREF_I:
case URX_STO_SP: // Setup for atomic or possessive blocks. Doesn't change what can match.
case URX_LD_SP:
break;
-
+
case URX_CARET:
if (atStart) {
fRXPat->fStartType = START_START;
break;
case URX_CARET_M:
+ case URX_CARET_M_UNIX:
if (atStart) {
fRXPat->fStartType = START_LINE;
}
break;
-
+
case URX_ONECHAR:
if (currentLen == 0) {
// This character could appear at the start of a match.
currentLen++;
atStart = FALSE;
break;
-
- case URX_SETREF:
+
+ case URX_SETREF:
if (currentLen == 0) {
int32_t sn = URX_VAL(op);
U_ASSERT(sn > 0 && sn < fRXPat->fSets->size());
break;
- case URX_STATIC_SETREF:
+ case URX_STATIC_SETREF:
if (currentLen == 0) {
int32_t sn = URX_VAL(op);
U_ASSERT(sn>0 && sn<URX_LAST_SET);
- case URX_STAT_SETREF_N:
+ case URX_STAT_SETREF_N:
if (currentLen == 0) {
int32_t sn = URX_VAL(op);
const UnicodeSet *s = fRXPat->fStaticSets[sn];
case URX_BACKSLASH_D:
// Digit Char
if (currentLen == 0) {
- UnicodeSet s;
+ UnicodeSet s;
s.applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, U_GC_ND_MASK, *fStatus);
if (URX_VAL(op) != 0) {
s.complement();
// character may have distinct cased forms. Add all of them
// to the set of possible starting match chars.
UnicodeSet s(c, c);
- s.closeOver(USET_CASE);
+ s.closeOver(USET_CASE_INSENSITIVE);
fRXPat->fInitialChars->addAll(s);
} else {
// Char has no case variants. Just add it as-is to the
case URX_BACKSLASH_X: // Grahpeme Cluster. Minimum is 1, max unbounded.
case URX_DOTANY_ALL: // . matches one or two.
case URX_DOTANY:
- case URX_DOTANY_ALL_PL:
- case URX_DOTANY_PL:
+ case URX_DOTANY_UNIX:
if (currentLen == 0) {
// These constructs are all bad news when they appear at the start
// of a match. Any character can begin the match.
// Loop of some kind. Can safely ignore, the worst that will happen
// is that we understate the true minimum length
currentLen = forwardedLength.elementAti(loc+1);
-
+
} else {
// Forward jump. Propagate the current min length to the target loc of the jump.
U_ASSERT(jmpDest <= end+1);
atStart = FALSE;
break;
- case URX_FAIL:
+ case URX_BACKTRACK:
// Fails are kind of like a branch, except that the min length was
// propagated already, by the state save.
currentLen = forwardedLength.elementAti(loc+1);
if (currentLen < forwardedLength.elementAti(jmpDest)) {
forwardedLength.setElementAt(currentLen, jmpDest);
}
- }
+ }
}
atStart = FALSE;
break;
-
+
fRXPat->fInitialStringIdx = stringStartIdx;
fRXPat->fInitialStringLen = stringLen;
}
-
+
currentLen += stringLen;
atStart = FALSE;
}
int32_t stringStartIdx = URX_VAL(op);
UChar32 c = fRXPat->fLiteralText.char32At(stringStartIdx);
UnicodeSet s(c, c);
- s.closeOver(USET_CASE);
+ s.closeOver(USET_CASE_INSENSITIVE);
fRXPat->fInitialChars->addAll(s);
numInitialStrings += 2; // Matching on an initial string not possible.
}
{
// Loop Init Ops. These don't change the min length, but they are 4 word ops
// so location must be updated accordingly.
- // Loop Init Ops.
+ // Loop Init Ops.
// If the min loop count == 0
// move loc forwards to the end of the loop, skipping over the body.
- // If the min count is > 0,
+ // If the min count is > 0,
// continue normal processing of the body of the loop.
int32_t loopEndLoc = fRXPat->fCompiledPat->elementAti(loc+1);
loopEndLoc = URX_VAL(loopEndLoc);
if (forwardedLength.elementAti(loopEndLoc) > currentLen) {
forwardedLength.setElementAt(currentLen, loopEndLoc);
}
- }
+ }
loc+=3; // Skips over operands of CTR_INIT
}
atStart = FALSE;
case URX_CTR_LOOP:
case URX_CTR_LOOP_NG:
- // Loop ops.
+ // Loop ops.
// The jump is conditional, backwards only.
atStart = FALSE;
break;
-
+
case URX_LOOP_C:
// More loop ops. These state-save to themselves.
// don't change the minimum match
atStart = FALSE;
break;
-
+
case URX_LA_START:
case URX_LB_START:
{
// Look-around. Scan forward until the matching look-ahead end,
// without processing the look-around block. This is overly pessimistic.
- int32_t depth = 0;
+
+ // Keep track of the nesting depth of look-around blocks. Boilerplate code for
+ // lookahead contains two LA_END instructions, so count goes up by two
+ // for each LA_START.
+ int32_t depth = (opType == URX_LA_START? 2: 1);
for (;;) {
loc++;
op = fRXPat->fCompiledPat->elementAti(loc);
- if (URX_TYPE(op) == URX_LA_START || URX_TYPE(op) == URX_LB_START) {
+ if (URX_TYPE(op) == URX_LA_START) {
+ depth+=2;
+ }
+ if (URX_TYPE(op) == URX_LB_START) {
depth++;
}
if (URX_TYPE(op) == URX_LA_END || URX_TYPE(op)==URX_LBN_END) {
+ depth--;
if (depth == 0) {
break;
}
- depth--;
}
if (URX_TYPE(op) == URX_STATE_SAVE) {
// Need this because neg lookahead blocks will FAIL to outside
}
}
}
- U_ASSERT(loc <= end);
+ U_ASSERT(loc <= end);
}
}
break;
-
+
case URX_LA_END:
case URX_LB_CONT:
case URX_LB_END:
case URX_LBN_CONT:
case URX_LBN_END:
- U_ASSERT(FALSE); // Shouldn't get here. These ops should be
+ U_ASSERT(FALSE); // Shouldn't get here. These ops should be
// consumed by the scan in URX_LA_START and LB_START
break;
-
+
default:
U_ASSERT(FALSE);
}
-
+
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// minMatchLength Calculate the length of the shortest string that could
-// match the specified pattern.
+// match the specified pattern.
// Length is in 16 bit code units, not code points.
//
// The calculated length may not be exact. The returned
// start and end are the range of p-code operations to be
// examined. The endpoints are included in the range.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
int32_t RegexCompile::minMatchLength(int32_t start, int32_t end) {
if (U_FAILURE(*fStatus)) {
return 0;
// If the op we are now at was the destination of a branch in the pattern,
// and that path has a shorter minimum length than the current accumulated value,
// replace the current accumulated value.
- U_ASSERT(currentLen>=0 && currentLen < INT32_MAX);
+ // U_ASSERT(currentLen>=0 && currentLen < INT32_MAX); // MinLength == INT32_MAX for some
+ // no-match-possible cases.
if (forwardedLength.elementAti(loc) < currentLen) {
currentLen = forwardedLength.elementAti(loc);
U_ASSERT(currentLen>=0 && currentLen < INT32_MAX);
case URX_BACKSLASH_Z:
case URX_CARET:
case URX_DOLLAR:
+ case URX_DOLLAR_M:
+ case URX_DOLLAR_D:
+ case URX_DOLLAR_MD:
case URX_RELOC_OPRND:
case URX_STO_INP_LOC:
- case URX_DOLLAR_M:
case URX_CARET_M:
- case URX_BACKTRACK:
+ case URX_CARET_M_UNIX:
case URX_BACKREF: // BackRef. Must assume that it might be a zero length match
case URX_BACKREF_I:
case URX_JMP_SAV:
case URX_JMP_SAV_X:
break;
-
+
// Ops that match a minimum of one character (one or two 16 bit code units.)
- //
+ //
case URX_ONECHAR:
case URX_STATIC_SETREF:
case URX_STAT_SETREF_N:
case URX_BACKSLASH_X: // Grahpeme Cluster. Minimum is 1, max unbounded.
case URX_DOTANY_ALL: // . matches one or two.
case URX_DOTANY:
- case URX_DOTANY_PL:
- case URX_DOTANY_ALL_PL:
+ case URX_DOTANY_UNIX:
currentLen++;
break;
}
break;
- case URX_FAIL:
+ case URX_BACKTRACK:
{
- // Fails are kind of like a branch, except that the min length was
+ // Back-tracks are kind of like a branch, except that the min length was
// propagated already, by the state save.
currentLen = forwardedLength.elementAti(loc+1);
- U_ASSERT(currentLen>=0 && currentLen < INT32_MAX);
}
break;
if (currentLen < forwardedLength.elementAti(jmpDest)) {
forwardedLength.setElementAt(currentLen, jmpDest);
}
- }
+ }
}
break;
-
+
case URX_STRING:
case URX_STRING_I:
case URX_CTR_INIT:
case URX_CTR_INIT_NG:
{
- // Loop Init Ops.
+ // Loop Init Ops.
// If the min loop count == 0
// move loc forwards to the end of the loop, skipping over the body.
- // If the min count is > 0,
+ // If the min count is > 0,
// continue normal processing of the body of the loop.
int32_t loopEndLoc = fRXPat->fCompiledPat->elementAti(loc+1);
loopEndLoc = URX_VAL(loopEndLoc);
case URX_CTR_LOOP:
case URX_CTR_LOOP_NG:
- // Loop ops.
+ // Loop ops.
// The jump is conditional, backwards only.
break;
-
+
case URX_LOOP_SR_I:
case URX_LOOP_DOT_I:
case URX_LOOP_C:
// More loop ops. These state-save to themselves.
// don't change the minimum match - could match nothing at all.
break;
-
+
case URX_LA_START:
case URX_LB_START:
{
// Look-around. Scan forward until the matching look-ahead end,
- // without processing the look-around block. This is overly pessimistic.
+ // without processing the look-around block. This is overly pessimistic for look-ahead,
+ // it assumes that the look-ahead match might be zero-length.
// TODO: Positive lookahead could recursively do the block, then continue
- // with the longer of the block or the value coming in.
- int32_t depth = 0;
+ // with the longer of the block or the value coming in. Ticket 6060
+ int32_t depth = (opType == URX_LA_START? 2: 1);;
for (;;) {
loc++;
op = fRXPat->fCompiledPat->elementAti(loc);
- if (URX_TYPE(op) == URX_LA_START || URX_TYPE(op) == URX_LB_START) {
+ if (URX_TYPE(op) == URX_LA_START) {
+ // The boilerplate for look-ahead includes two LA_END insturctions,
+ // Depth will be decremented by each one when it is seen.
+ depth += 2;
+ }
+ if (URX_TYPE(op) == URX_LB_START) {
depth++;
}
- if (URX_TYPE(op) == URX_LA_END || URX_TYPE(op)==URX_LBN_END) {
+ if (URX_TYPE(op) == URX_LA_END) {
+ depth--;
if (depth == 0) {
break;
}
+ }
+ if (URX_TYPE(op)==URX_LBN_END) {
depth--;
+ if (depth == 0) {
+ break;
+ }
}
if (URX_TYPE(op) == URX_STATE_SAVE) {
// Need this because neg lookahead blocks will FAIL to outside
}
}
}
-
- U_ASSERT(loc <= end);
+ U_ASSERT(loc <= end);
}
}
break;
-
+
case URX_LA_END:
case URX_LB_CONT:
case URX_LB_END:
// Only come here if the matching URX_LA_START or URX_LB_START was not in the
// range being sized, which happens when measuring size of look-behind blocks.
break;
-
+
default:
U_ASSERT(FALSE);
}
-
+
}
// We have finished walking through the ops. Check whether some forward jump
currentLen = forwardedLength.elementAti(end+1);
U_ASSERT(currentLen>=0 && currentLen < INT32_MAX);
}
-
+
return currentLen;
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// maxMatchLength Calculate the length of the longest string that could
-// match the specified pattern.
+// match the specified pattern.
// Length is in 16 bit code units, not code points.
//
// The calculated length may not be exact. The returned
// value may be longer than the actual maximum; it must
// never be shorter.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
int32_t RegexCompile::maxMatchLength(int32_t start, int32_t end) {
if (U_FAILURE(*fStatus)) {
return 0;
case URX_BACKSLASH_Z:
case URX_CARET:
case URX_DOLLAR:
+ case URX_DOLLAR_M:
+ case URX_DOLLAR_D:
+ case URX_DOLLAR_MD:
case URX_RELOC_OPRND:
case URX_STO_INP_LOC:
- case URX_DOLLAR_M:
case URX_CARET_M:
- case URX_BACKTRACK:
+ case URX_CARET_M_UNIX:
case URX_STO_SP: // Setup for atomic or possessive blocks. Doesn't change what can match.
case URX_LD_SP:
case URX_LBN_CONT:
case URX_LBN_END:
break;
-
+
// Ops that increase that cause an unbounded increase in the length
// of a matched string, or that increase it a hard to characterize way.
case URX_BACKREF: // BackRef. Must assume that it might be a zero length match
case URX_BACKREF_I:
case URX_BACKSLASH_X: // Grahpeme Cluster. Minimum is 1, max unbounded.
- case URX_DOTANY_PL:
- case URX_DOTANY_ALL_PL:
currentLen = INT32_MAX;
break;
// Ops that match a max of one character (possibly two 16 bit code units.)
- //
+ //
case URX_STATIC_SETREF:
case URX_STAT_SETREF_N:
case URX_SETREF:
case URX_BACKSLASH_D:
case URX_ONECHAR_I:
- case URX_DOTANY_ALL:
+ case URX_DOTANY_ALL:
case URX_DOTANY:
+ case URX_DOTANY_UNIX:
currentLen+=2;
break;
}
break;
- // Jumps.
+ // Jumps.
//
case URX_JMP:
case URX_JMPX:
}
break;
- case URX_FAIL:
- // Fails are kind of like a branch, except that the max length was
+ case URX_BACKTRACK:
+ // back-tracks are kind of like a branch, except that the max length was
// propagated already, by the state save.
currentLen = forwardedLength.elementAti(loc+1);
break;
}
}
break;
-
+
// INT32_MAX length will stop the per-instruction loop.
currentLen = INT32_MAX;
break;
-
-
+
+
case URX_LA_START:
case URX_LA_END:
// it were normal pattern. Gives a too-long match length,
// but good enough for now.
break;
-
+
// End of look-ahead ops should always be consumed by the processing at
// the URX_LA_START op.
- U_ASSERT(FALSE);
- break;
-
+ // U_ASSERT(FALSE);
+ // break;
+
case URX_LB_START:
{
// Look-behind. Scan forward until the matching look-around end,
- // without processing the look-behind block.
+ // without processing the look-behind block.
int32_t depth = 0;
for (;;) {
loc++;
}
depth--;
}
- U_ASSERT(loc < end);
+ U_ASSERT(loc < end);
}
}
break;
U_ASSERT(FALSE);
}
-
+
if (currentLen == INT32_MAX) {
// The maximum length is unbounded.
// Stop further processing of the pattern.
break;
}
-
+
}
return currentLen;
-
+
}
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// stripNOPs Remove any NOP operations from the compiled pattern code.
// Extra NOPs are inserted for some constructs during the initial
// code generation to provide locations that may be patched later.
// Many end up unneeded, and are removed by this function.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::stripNOPs() {
if (U_FAILURE(*fStatus)) {
case URX_BACKSLASH_X:
case URX_BACKSLASH_Z:
case URX_DOTANY_ALL:
- case URX_DOTANY_ALL_PL:
- case URX_DOTANY_PL:
case URX_BACKSLASH_D:
case URX_CARET:
case URX_DOLLAR:
case URX_CTR_INIT:
case URX_CTR_INIT_NG:
+ case URX_DOTANY_UNIX:
case URX_STO_SP:
case URX_LD_SP:
case URX_BACKREF:
case URX_BACKREF_I:
case URX_DOLLAR_M:
case URX_CARET_M:
+ case URX_CARET_M_UNIX:
case URX_LB_START:
case URX_LB_CONT:
case URX_LB_END:
case URX_LOOP_SR_I:
case URX_LOOP_DOT_I:
case URX_LOOP_C:
+ case URX_DOLLAR_D:
+ case URX_DOLLAR_MD:
// These instructions are unaltered by the relocation.
fRXPat->fCompiledPat->setElementAt(op, dst);
dst++;
-//----------------------------------------------------------------------------------------
-//
-// OptDotStar Optimize patterns that end with a '.*' or '.+' to
-// just advance the input to the end.
-//
-// Transform this compiled sequence
-// [DOT_ANY | DOT_ANY_ALL]
-// JMP_SAV to previous instruction
-// [NOP | END_CAPTURE | DOLLAR | BACKSLASH_Z]*
-// END
-//
-// To
-// NOP
-// [DOT_ANY_PL | DOT_ANY_ALL_PL]
-// [NOP | END_CAPTURE | DOLLAR | BACKSLASH_Z]*
-// END
-//
-//----------------------------------------------------------------------------------------
-void RegexCompile::OptDotStar() {
- // Scan backwards in the pattern, looking for a JMP_SAV near the end.
- int32_t jmpLoc;
- int32_t op = 0;
- int32_t opType;
- for (jmpLoc=fRXPat->fCompiledPat->size(); jmpLoc--;) {
- U_ASSERT(jmpLoc>0);
- op = fRXPat->fCompiledPat->elementAti(jmpLoc);
- opType = URX_TYPE(op);
- switch(opType) {
-
-
- case URX_END:
- case URX_NOP:
- case URX_END_CAPTURE:
- case URX_DOLLAR_M:
- case URX_DOLLAR:
- case URX_BACKSLASH_Z:
- // These ops may follow the JMP_SAV without preventing us from
- // doing this optimization.
- continue;
-
- case URX_JMP_SAV:
- // Got a trailing JMP_SAV that's a candidate for optimization.
- break;
-
- default:
- // This optimization not possible.
- return;
- }
- break; // from the for loop.
- }
-
- // We found in URX_JMP_SAV near the end that is a candidate for optimizing.
- // Is the target address the previous instruction?
- // Is the previous instruction a flavor of URX_DOTANY
- int32_t loopTopLoc = URX_VAL(op);
- if (loopTopLoc != jmpLoc-1) {
- return;
- }
- int32_t newOp;
- int32_t oldOp = fRXPat->fCompiledPat->elementAti(loopTopLoc);
- int32_t oldOpType = opType = URX_TYPE(oldOp);
- if (oldOpType == URX_DOTANY) {
- newOp = URX_BUILD(URX_DOTANY_PL, 0);
- }
- else if (oldOpType == URX_DOTANY_ALL) {
- newOp = URX_BUILD(URX_DOTANY_ALL_PL, 0);
- } else {
- return; // Sequence we were looking for isn't there.
- }
-
- // Substitute the new instructions into the pattern.
- // The NOP will be removed in a later optimization step.
- fRXPat->fCompiledPat->setElementAt(URX_BUILD(URX_NOP, 0), loopTopLoc);
- fRXPat->fCompiledPat->setElementAt(newOp, jmpLoc);
-}
-
-
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// Error Report a rule parse error.
// Only report it if no previous error has been recorded.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::error(UErrorCode e) {
if (U_SUCCESS(*fStatus)) {
*fStatus = e;
// (Think EBCDIC).
//
static const UChar chCR = 0x0d; // New lines, for terminating comments.
-static const UChar chLF = 0x0a;
-static const UChar chNEL = 0x85; // NEL newline variant
-static const UChar chLS = 0x2028; // Unicode Line Separator
-static const UChar chApos = 0x27; // single quote, for quoted chars.
+static const UChar chLF = 0x0a; // Line Feed
static const UChar chPound = 0x23; // '#', introduces a comment.
+static const UChar chDigit0 = 0x30; // '0'
+static const UChar chDigit7 = 0x37; // '9'
+static const UChar chColon = 0x3A; // ':'
static const UChar chE = 0x45; // 'E'
+static const UChar chQ = 0x51; // 'Q'
+static const UChar chN = 0x4E; // 'N'
+static const UChar chP = 0x50; // 'P'
static const UChar chBackSlash = 0x5c; // '\' introduces a char escape
-static const UChar chLParen = 0x28;
-static const UChar chRParen = 0x29;
-static const UChar chLBracket = 0x5b;
-static const UChar chRBracket = 0x5d;
-static const UChar chRBrace = 0x7d;
-static const UChar chUpperN = 0x4E;
+static const UChar chLBracket = 0x5b; // '['
+static const UChar chRBracket = 0x5d; // ']'
+static const UChar chUp = 0x5e; // '^'
static const UChar chLowerP = 0x70;
-static const UChar chUpperP = 0x50;
+static const UChar chLBrace = 0x7b; // '{'
+static const UChar chRBrace = 0x7d; // '}'
+static const UChar chNEL = 0x85; // NEL newline variant
+static const UChar chLS = 0x2028; // Unicode Line Separator
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// nextCharLL Low Level Next Char from the regex pattern.
// Get a char from the string, keep track of input position
// for error reporting.
//
-//----------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
UChar32 RegexCompile::nextCharLL() {
UChar32 ch;
UnicodeString &pattern = fRXPat->fPattern;
// reset the column to 0.
fLineNum++;
fCharNum=0;
- if (fQuoteMode) {
- error(U_REGEX_RULE_SYNTAX);
- fQuoteMode = FALSE;
- }
}
else {
// Character is not starting a new line. Except in the case of a
return ch;
}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// peekCharLL Low Level Character Scanning, sneak a peek at the next
// character without actually getting it.
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
UChar32 RegexCompile::peekCharLL() {
if (fPeekChar == -1) {
fPeekChar = nextCharLL();
}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// nextChar for pattern scanning. At this level, we handle stripping
// out comments and processing some backslash character escapes.
// The rest of the pattern grammar is handled at the next level up.
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
void RegexCompile::nextChar(RegexPatternChar &c) {
fScanIndex = fNextIndex;
if (fModeFlags & UREGEX_COMMENTS) {
//
// We are in free-spacing and comments mode.
- // Scan through any white space and comments, until we
+ // Scan through any white space and comments, until we
// reach a significant character or the end of inut.
for (;;) {
if (c.fChar == (UChar32)-1) {
}
}
}
+ // TODO: check what Java & Perl do with non-ASCII white spaces. Ticket 6061.
if (uprv_isRuleWhiteSpace(c.fChar) == FALSE) {
break;
}
//
// check for backslash escaped characters.
//
- int32_t startX = fNextIndex; // start and end positions of the
- int32_t endX = fNextIndex; // sequence following the '\'
if (c.fChar == chBackSlash) {
- if (RegexStaticSets::gStaticSets->fUnescapeCharSet->contains(peekCharLL())) {
+ int32_t startX = fNextIndex; // start and end positions of the
+ int32_t endX = fNextIndex; // sequence following the '\'
+ if (RegexStaticSets::gStaticSets->fUnescapeCharSet.contains(peekCharLL())) {
//
// A '\' sequence that is handled by ICU's standard unescapeAt function.
// Includes \uxxxx, \n, \r, many others.
fCharNum += endX - startX;
fNextIndex = endX;
}
+ else if (peekCharLL() == chDigit0) {
+ // Octal Escape, using Java Regexp Conventions
+ // which are \0 followed by 1-3 octal digits.
+ // Different from ICU Unescape handling of Octal, which does not
+ // require the leading 0.
+ // Java also has the convention of only consuning 2 octal digits if
+ // the three digit number would be > 0xff
+ //
+ c.fChar = 0;
+ nextCharLL(); // Consume the initial 0.
+ int index;
+ for (index=0; index<3; index++) {
+ int32_t ch = peekCharLL();
+ if (ch<chDigit0 || ch>chDigit7) {
+ if (index==0) {
+ // \0 is not followed by any octal digits.
+ error(U_REGEX_BAD_ESCAPE_SEQUENCE);
+ }
+ break;
+ }
+ c.fChar <<= 3;
+ c.fChar += ch&7;
+ if (c.fChar <= 255) {
+ nextCharLL();
+ } else {
+ // The last digit made the number too big. Forget we saw it.
+ c.fChar >>= 3;
+ }
+ }
+ c.fQuoted = TRUE;
+ }
+ else if (peekCharLL() == chQ) {
+ // "\Q" enter quote mode, which will continue until "\E"
+ fQuoteMode = TRUE;
+ nextCharLL(); // discard the 'Q'.
+ nextChar(c); // recurse to get the real next char.
+ }
else
{
// We are in a '\' escape that will be handled by the state table scanner.
// Just return the backslash, but remember that the following char is to
- // be taken literally. TODO: this is awkward, think about alternatives.
+ // be taken literally.
fInBackslashQuote = TRUE;
}
}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
-// scanSet Construct a UnicodeSet from the text at the current scan
-// position. Advance the scan position to the first character
-// after the set.
+// scanNamedChar
+ // Get a UChar32 from a \N{UNICODE CHARACTER NAME} in the pattern.
//
-// The scan position is normally under the control of the state machine
-// that controls pattern parsing. UnicodeSets, however, are parsed by
-// the UnicodeSet constructor, not by the Regex pattern parser.
+// The scan position will be at the 'N'. On return
+// the scan position should be just after the '}'
//
-//---------------------------------------------------------------------------------
-UnicodeSet *RegexCompile::scanSet() {
- UnicodeSet *uset = NULL;
- ParsePosition pos;
- int startPos;
- int i;
-
+// Return the UChar32
+//
+//------------------------------------------------------------------------------
+UChar32 RegexCompile::scanNamedChar() {
if (U_FAILURE(*fStatus)) {
- return NULL;
- }
-
- pos.setIndex(fScanIndex);
- startPos = fScanIndex;
- UErrorCode localStatus = U_ZERO_ERROR;
- uint32_t usetFlags = 0;
- if (fModeFlags & UREGEX_CASE_INSENSITIVE) {
- usetFlags |= USET_CASE_INSENSITIVE;
- }
- if (fModeFlags & UREGEX_COMMENTS) {
- usetFlags |= USET_IGNORE_SPACE;
+ return 0;
}
- uset = new UnicodeSet(fRXPat->fPattern, pos,
- usetFlags, NULL, localStatus);
- if (U_FAILURE(localStatus)) {
- // TODO: Get more accurate position of the error from UnicodeSet's return info.
- // UnicodeSet appears to not be reporting correctly at this time.
- REGEX_SCAN_DEBUG_PRINTF(("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex()));
- error(localStatus);
- delete uset;
- return NULL;
+ nextChar(fC);
+ if (fC.fChar != chLBrace) {
+ error(U_REGEX_PROPERTY_SYNTAX);
+ return 0;
}
-
- // Advance the current scan postion over the UnicodeSet.
- // Don't just set fScanIndex because the line/char positions maintained
- // for error reporting would be thrown off.
- i = pos.getIndex();
+
+ UnicodeString charName;
for (;;) {
- if (fNextIndex >= i) {
+ nextChar(fC);
+ if (fC.fChar == chRBrace) {
break;
}
- nextCharLL();
+ if (fC.fChar == -1) {
+ error(U_REGEX_PROPERTY_SYNTAX);
+ return 0;
+ }
+ charName.append(fC.fChar);
+ }
+
+ char name[100];
+ if (!uprv_isInvariantUString(charName.getBuffer(), charName.length()) ||
+ (uint32_t)charName.length()>=sizeof(name)) {
+ // All Unicode character names have only invariant characters.
+ // The API to get a character, given a name, accepts only char *, forcing us to convert,
+ // which requires this error check
+ error(U_REGEX_PROPERTY_SYNTAX);
+ return 0;
}
+ charName.extract(0, charName.length(), name, sizeof(name), US_INV);
- return uset;
-};
+ UChar32 theChar = u_charFromName(U_UNICODE_CHAR_NAME, name, fStatus);
+ if (U_FAILURE(*fStatus)) {
+ error(U_REGEX_PROPERTY_SYNTAX);
+ }
+ nextChar(fC); // Continue overall regex pattern processing with char after the '}'
+ return theChar;
+}
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
//
// scanProp Construct a UnicodeSet from the text at the current scan
// position, which will be of the form \p{whaterver}
// Return a UnicodeSet, constructed from the \P pattern,
// or NULL if the pattern is invalid.
//
-//---------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
UnicodeSet *RegexCompile::scanProp() {
UnicodeSet *uset = NULL;
if (U_FAILURE(*fStatus)) {
return NULL;
}
+ U_ASSERT(fC.fChar == chLowerP || fC.fChar == chP);
+ UBool negated = (fC.fChar == chP);
- U_ASSERT(fC.fChar == chLowerP || fC.fChar == chUpperP || fC.fChar == chUpperN);
-
- // enclose the \p{property} from the regex pattern source in [brackets]
- UnicodeString setPattern;
- setPattern.append(chLBracket);
- setPattern.append(chBackSlash);
+ UnicodeString propertyName;
+ nextChar(fC);
+ if (fC.fChar != chLBrace) {
+ error(U_REGEX_PROPERTY_SYNTAX);
+ return NULL;
+ }
for (;;) {
- setPattern.append(fC.fChar);
+ nextChar(fC);
if (fC.fChar == chRBrace) {
break;
}
- nextChar(fC);
if (fC.fChar == -1) {
// Hit the end of the input string without finding the closing '}'
error(U_REGEX_PROPERTY_SYNTAX);
return NULL;
}
+ propertyName.append(fC.fChar);
+ }
+ uset = createSetForProperty(propertyName, negated);
+ nextChar(fC); // Move input scan to position following the closing '}'
+ return uset;
+}
+
+//------------------------------------------------------------------------------
+//
+// scanPosixProp Construct a UnicodeSet from the text at the current scan
+// position, which is expected be of the form [:property expression:]
+//
+// The scan position will be at the opening ':'. On return
+// the scan position must be on the closing ']'
+//
+// Return a UnicodeSet constructed from the pattern,
+// or NULL if this is not a valid POSIX-style set expression.
+// If not a property expression, restore the initial scan position
+// (to the opening ':')
+//
+// Note: the opening '[:' is not sufficient to guarantee that
+// this is a [:property:] expression.
+// [:'+=,] is a perfectly good ordinary set expression that
+// happens to include ':' as one of its characters.
+//
+//------------------------------------------------------------------------------
+UnicodeSet *RegexCompile::scanPosixProp() {
+ UnicodeSet *uset = NULL;
+
+ if (U_FAILURE(*fStatus)) {
+ return NULL;
+ }
+
+ U_ASSERT(fC.fChar == chColon);
+
+ // Save the scanner state.
+ // TODO: move this into the scanner, with the state encapsulated in some way. Ticket 6062
+ int32_t savedScanIndex = fScanIndex;
+ int32_t savedNextIndex = fNextIndex;
+ UBool savedQuoteMode = fQuoteMode;
+ UBool savedInBackslashQuote = fInBackslashQuote;
+ UBool savedEOLComments = fEOLComments;
+ int32_t savedLineNum = fLineNum;
+ int32_t savedCharNum = fCharNum;
+ UChar32 savedLastChar = fLastChar;
+ UChar32 savedPeekChar = fPeekChar;
+ RegexPatternChar savedfC = fC;
+
+ // Scan for a closing ]. A little tricky because there are some perverse
+ // edge cases possible. "[:abc\Qdef:] \E]" is a valid non-property expression,
+ // ending on the second closing ].
+
+ UnicodeString propName;
+ UBool negated = FALSE;
+
+ // Check for and consume the '^' in a negated POSIX property, e.g. [:^Letter:]
+ nextChar(fC);
+ if (fC.fChar == chUp) {
+ negated = TRUE;
+ nextChar(fC);
+ }
+
+ // Scan for the closing ":]", collecting the property name along the way.
+ UBool sawPropSetTerminator = FALSE;
+ for (;;) {
+ propName.append(fC.fChar);
+ nextChar(fC);
+ if (fC.fQuoted || fC.fChar == -1) {
+ // Escaped characters or end of input - either says this isn't a [:Property:]
+ break;
+ }
+ if (fC.fChar == chColon) {
+ nextChar(fC);
+ if (fC.fChar == chRBracket) {
+ sawPropSetTerminator = TRUE;
+ }
+ break;
+ }
+ }
+
+ if (sawPropSetTerminator) {
+ uset = createSetForProperty(propName, negated);
+ }
+ else
+ {
+ // No closing ":]".
+ // Restore the original scan position.
+ // The main scanner will retry the input as a normal set expression,
+ // not a [:Property:] expression.
+ fScanIndex = savedScanIndex;
+ fNextIndex = savedNextIndex;
+ fQuoteMode = savedQuoteMode;
+ fInBackslashQuote = savedInBackslashQuote;
+ fEOLComments = savedEOLComments;
+ fLineNum = savedLineNum;
+ fCharNum = savedCharNum;
+ fLastChar = savedLastChar;
+ fPeekChar = savedPeekChar;
+ fC = savedfC;
+ }
+ return uset;
+}
+
+static inline void addIdentifierIgnorable(UnicodeSet *set, UErrorCode& ec) {
+ set->add(0, 8).add(0x0e, 0x1b).add(0x7f, 0x9f);
+ addCategory(set, U_GC_CF_MASK, ec);
+}
+
+//
+// Create a Unicode Set from a Unicode Property expression.
+// This is common code underlying both \p{...} ane [:...:] expressions.
+// Includes trying the Java "properties" that aren't supported as
+// normal ICU UnicodeSet properties
+//
+static const UChar posSetPrefix[] = {0x5b, 0x5c, 0x70, 0x7b, 0}; // "[\p{"
+static const UChar negSetPrefix[] = {0x5b, 0x5c, 0x50, 0x7b, 0}; // "[\P{"
+UnicodeSet *RegexCompile::createSetForProperty(const UnicodeString &propName, UBool negated) {
+ UnicodeString setExpr;
+ UnicodeSet *set;
+ uint32_t usetFlags = 0;
+
+ if (U_FAILURE(*fStatus)) {
+ return NULL;
}
- setPattern.append(chRBracket);
- uint32_t usetFlags = 0;
+ //
+ // First try the property as we received it
+ //
+ if (negated) {
+ setExpr.append(negSetPrefix, -1);
+ } else {
+ setExpr.append(posSetPrefix, -1);
+ }
+ setExpr.append(propName);
+ setExpr.append(chRBrace);
+ setExpr.append(chRBracket);
if (fModeFlags & UREGEX_CASE_INSENSITIVE) {
usetFlags |= USET_CASE_INSENSITIVE;
}
- if (fModeFlags & UREGEX_COMMENTS) {
- usetFlags |= USET_IGNORE_SPACE;
+ set = new UnicodeSet(setExpr, usetFlags, NULL, *fStatus);
+ if (U_SUCCESS(*fStatus)) {
+ return set;
+ }
+ delete set;
+ set = NULL;
+
+ //
+ // The property as it was didn't work.
+ // Do emergency fixes -
+ // InGreek -> InGreek or Coptic, that being the official Unicode name for that block.
+ // InCombiningMarksforSymbols -> InCombiningDiacriticalMarksforSymbols.
+ //
+ // Note on Spaces: either "InCombiningMarksForSymbols" or "InCombining Marks for Symbols"
+ // is accepted by Java. The property part of the name is compared
+ // case-insenstively. The spaces must be exactly as shown, either
+ // all there, or all omitted, with exactly one at each position
+ // if they are present. From checking against JDK 1.6
+ //
+ // This code should be removed when ICU properties support the Java compatibility names
+ // (ICU 4.0?)
+ //
+ UnicodeString mPropName = propName;
+ if (mPropName.caseCompare(UNICODE_STRING_SIMPLE("InGreek"), 0) == 0) {
+ mPropName = UNICODE_STRING_SIMPLE("InGreek and Coptic");
+ }
+ if (mPropName.caseCompare(UNICODE_STRING_SIMPLE("InCombining Marks for Symbols"), 0) == 0 ||
+ mPropName.caseCompare(UNICODE_STRING_SIMPLE("InCombiningMarksforSymbols"), 0) == 0) {
+ mPropName = UNICODE_STRING_SIMPLE("InCombining Diacritical Marks for Symbols");
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("all")) == 0) {
+ mPropName = UNICODE_STRING_SIMPLE("javaValidCodePoint");
+ }
+
+ // See if the property looks like a Java "InBlockName", which
+ // we will recast as "Block=BlockName"
+ //
+ static const UChar IN[] = {0x49, 0x6E, 0}; // "In"
+ static const UChar BLOCK[] = {0x42, 0x6C, 0x6f, 0x63, 0x6b, 0x3d, 00}; // "Block="
+ if (mPropName.startsWith(IN, 2) && propName.length()>=3) {
+ setExpr.truncate(4); // Leaves "[\p{", or "[\P{"
+ setExpr.append(BLOCK, -1);
+ setExpr.append(UnicodeString(mPropName, 2)); // Property with the leading "In" removed.
+ setExpr.append(chRBrace);
+ setExpr.append(chRBracket);
+ *fStatus = U_ZERO_ERROR;
+ set = new UnicodeSet(setExpr, usetFlags, NULL, *fStatus);
+ if (U_SUCCESS(*fStatus)) {
+ return set;
+ }
+ delete set;
+ set = NULL;
}
- // Build the UnicodeSet from the set pattern we just built up in a string.
- uset = new UnicodeSet(setPattern, usetFlags, NULL, *fStatus);
- if (U_FAILURE(*fStatus)) {
- delete uset;
- uset = NULL;
+ if (propName.startsWith(UNICODE_STRING_SIMPLE("java")) ||
+ propName.compare(UNICODE_STRING_SIMPLE("all")) == 0)
+ {
+ UErrorCode localStatus = U_ZERO_ERROR;
+ //setExpr.remove();
+ set = new UnicodeSet();
+ //
+ // Try the various Java specific properties.
+ // These all begin with "java"
+ //
+ if (mPropName.compare(UNICODE_STRING_SIMPLE("javaDefined")) == 0) {
+ addCategory(set, U_GC_CN_MASK, localStatus);
+ set->complement();
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaDigit")) == 0) {
+ addCategory(set, U_GC_ND_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaIdentifierIgnorable")) == 0) {
+ addIdentifierIgnorable(set, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaISOControl")) == 0) {
+ set->add(0, 0x1F).add(0x7F, 0x9F);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaJavaIdentifierPart")) == 0) {
+ addCategory(set, U_GC_L_MASK, localStatus);
+ addCategory(set, U_GC_SC_MASK, localStatus);
+ addCategory(set, U_GC_PC_MASK, localStatus);
+ addCategory(set, U_GC_ND_MASK, localStatus);
+ addCategory(set, U_GC_NL_MASK, localStatus);
+ addCategory(set, U_GC_MC_MASK, localStatus);
+ addCategory(set, U_GC_MN_MASK, localStatus);
+ addIdentifierIgnorable(set, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaJavaIdentifierStart")) == 0) {
+ addCategory(set, U_GC_L_MASK, localStatus);
+ addCategory(set, U_GC_NL_MASK, localStatus);
+ addCategory(set, U_GC_SC_MASK, localStatus);
+ addCategory(set, U_GC_PC_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaLetter")) == 0) {
+ addCategory(set, U_GC_L_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaLetterOrDigit")) == 0) {
+ addCategory(set, U_GC_L_MASK, localStatus);
+ addCategory(set, U_GC_ND_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaLowerCase")) == 0) {
+ addCategory(set, U_GC_LL_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaMirrored")) == 0) {
+ set->applyIntPropertyValue(UCHAR_BIDI_MIRRORED, 1, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaSpaceChar")) == 0) {
+ addCategory(set, U_GC_Z_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaSupplementaryCodePoint")) == 0) {
+ set->add(0x10000, UnicodeSet::MAX_VALUE);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaTitleCase")) == 0) {
+ addCategory(set, U_GC_LT_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaUnicodeIdentifierStart")) == 0) {
+ addCategory(set, U_GC_L_MASK, localStatus);
+ addCategory(set, U_GC_NL_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaUnicodeIdentifierPart")) == 0) {
+ addCategory(set, U_GC_L_MASK, localStatus);
+ addCategory(set, U_GC_PC_MASK, localStatus);
+ addCategory(set, U_GC_ND_MASK, localStatus);
+ addCategory(set, U_GC_NL_MASK, localStatus);
+ addCategory(set, U_GC_MC_MASK, localStatus);
+ addCategory(set, U_GC_MN_MASK, localStatus);
+ addIdentifierIgnorable(set, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaUpperCase")) == 0) {
+ addCategory(set, U_GC_LU_MASK, localStatus);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaValidCodePoint")) == 0) {
+ set->add(0, UnicodeSet::MAX_VALUE);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("javaWhitespace")) == 0) {
+ addCategory(set, U_GC_Z_MASK, localStatus);
+ set->removeAll(UnicodeSet().add(0xa0).add(0x2007).add(0x202f));
+ set->add(9, 0x0d).add(0x1c, 0x1f);
+ }
+ else if (mPropName.compare(UNICODE_STRING_SIMPLE("all")) == 0) {
+ set->add(0, UnicodeSet::MAX_VALUE);
+ }
+
+ if (U_SUCCESS(localStatus) && !set->isEmpty()) {
+ *fStatus = U_ZERO_ERROR;
+ if (usetFlags & USET_CASE_INSENSITIVE) {
+ set->closeOver(USET_CASE_INSENSITIVE);
+ }
+ if (negated) {
+ set->complement();
+ }
+ return set;
+ }
+ delete set;
+ set = NULL;
}
+ error(*fStatus);
+ return NULL;
+}
+
- nextChar(fC); // Continue overall regex pattern processing with char after the '}'
- return uset;
-};
+
+//
+// SetEval Part of the evaluation of [set expressions].
+// Perform any pending (stacked) operations with precedence
+// equal or greater to that of the next operator encountered
+// in the expression.
+//
+void RegexCompile::setEval(int32_t nextOp) {
+ UnicodeSet *rightOperand = NULL;
+ UnicodeSet *leftOperand = NULL;
+ for (;;) {
+ U_ASSERT(fSetOpStack.empty()==FALSE);
+ int32_t pendingSetOperation = fSetOpStack.peeki();
+ if ((pendingSetOperation&0xffff0000) < (nextOp&0xffff0000)) {
+ break;
+ }
+ fSetOpStack.popi();
+ U_ASSERT(fSetStack.empty() == FALSE);
+ rightOperand = (UnicodeSet *)fSetStack.peek();
+ switch (pendingSetOperation) {
+ case setNegation:
+ rightOperand->complement();
+ break;
+ case setCaseClose:
+ // TODO: need a simple close function. Ticket 6065
+ rightOperand->closeOver(USET_CASE_INSENSITIVE);
+ rightOperand->removeAllStrings();
+ break;
+ case setDifference1:
+ case setDifference2:
+ fSetStack.pop();
+ leftOperand = (UnicodeSet *)fSetStack.peek();
+ leftOperand->removeAll(*rightOperand);
+ delete rightOperand;
+ break;
+ case setIntersection1:
+ case setIntersection2:
+ fSetStack.pop();
+ leftOperand = (UnicodeSet *)fSetStack.peek();
+ leftOperand->retainAll(*rightOperand);
+ delete rightOperand;
+ break;
+ case setUnion:
+ fSetStack.pop();
+ leftOperand = (UnicodeSet *)fSetStack.peek();
+ leftOperand->addAll(*rightOperand);
+ delete rightOperand;
+ break;
+ default:
+ U_ASSERT(FALSE);
+ break;
+ }
+ }
+ }
+
+void RegexCompile::setPushOp(int32_t op) {
+ setEval(op);
+ fSetOpStack.push(op, *fStatus);
+ fSetStack.push(new UnicodeSet(), *fStatus);
+}
U_NAMESPACE_END
#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS
+
projects / apple / icu.git / blobdiff