//
// file: regexcmp.cpp
//
-// Copyright (C) 2002-2010 International Business Machines Corporation and others.
+// Copyright (C) 2002-2012 International Business Machines Corporation and others.
// All Rights Reserved.
//
// This file contains the ICU regular expression compiler, which is responsible
#include "unicode/parsepos.h"
#include "unicode/parseerr.h"
#include "unicode/regex.h"
-#include "util.h"
+#include "unicode/utf.h"
+#include "unicode/utf16.h"
+#include "patternprops.h"
#include "putilimp.h"
#include "cmemory.h"
#include "cstring.h"
fMatchOpenParen = -1;
fMatchCloseParen = -1;
- fStringOpStart = -1;
if (U_SUCCESS(status) && U_FAILURE(rxp->fDeferredStatus)) {
status = rxp->fDeferredStatus;
UParseError &pp, // Error position info
UErrorCode &e) // Error Code
{
- fRXPat->fPatternString = new UnicodeString(pat);
+ fRXPat->fPatternString = new UnicodeString(pat);
UText patternText = UTEXT_INITIALIZER;
utext_openConstUnicodeString(&patternText, fRXPat->fPatternString, &e);
fPatternLength = utext_nativeLength(pat);
uint16_t state = 1;
const RegexTableEl *tableEl;
+
+ // UREGEX_LITERAL force entire pattern to be treated as a literal string.
+ if (fModeFlags & UREGEX_LITERAL) {
+ fQuoteMode = TRUE;
+ }
+
nextChar(fC); // Fetch the first char from the pattern string.
//
if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class &&
fC.fQuoted == FALSE && // char is not escaped &&
fC.fChar != (UChar32)-1) { // char is not EOF
+ U_ASSERT(tableEl->fCharClass <= 137);
if (RegexStaticSets::gStaticSets->fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) {
// Table row specified a character class, or set of characters,
// and the current char matches it.
case doOrOperator:
// Scanning a '|', as in (A|B)
{
+ // Generate code for any pending literals preceding the '|'
+ fixLiterals(FALSE);
+
// Insert a SAVE operation at the start of the pattern section preceding
// this OR at this level. This SAVE will branch the match forward
// to the right hand side of the OR in the event that the left hand
// encountered. This will be promoted to a completed capture when (and if) the corresponding
// END_CAPTURE is encountered.
{
+ fixLiterals();
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
int32_t varsLoc = fRXPat->fFrameSize; // Reserve three slots in match stack frame.
fRXPat->fFrameSize += 3;
// - NOP, which may later be replaced by a save-state if there
// is an '|' alternation within the parens.
{
+ fixLiterals();
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
// - NOP, which may later be replaced by a save-state if there
// is an '|' alternation within the parens.
{
+ fixLiterals();
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
int32_t varLoc = fRXPat->fDataSize; // Reserve a data location for saving the
fRXPat->fDataSize += 1; // state stack ptr.
//
// Two data slots are reserved, for saving the stack ptr and the input position.
{
+ fixLiterals();
int32_t dataLoc = fRXPat->fDataSize;
fRXPat->fDataSize += 2;
int32_t op = URX_BUILD(URX_LA_START, dataLoc);
// 7. END_LA // Restore match region, in case look-ahead was using
// an alternate (transparent) region.
{
+ fixLiterals();
int32_t dataLoc = fRXPat->fDataSize;
fRXPat->fDataSize += 2;
int32_t op = URX_BUILD(URX_LA_START, dataLoc);
// 2: Start index of match current match attempt.
// 3: Original Input String len.
+ // Generate match code for any pending literals.
+ fixLiterals();
+
// Allocate data space
int32_t dataLoc = fRXPat->fDataSize;
fRXPat->fDataSize += 4;
// 2: Start index of match current match attempt.
// 3: Original Input String len.
+ // Generate match code for any pending literals.
+ fixLiterals();
+
// Allocate data space
int32_t dataLoc = fRXPat->fDataSize;
fRXPat->fDataSize += 4;
case doDotAny:
// scanned a ".", match any single character.
{
+ fixLiterals(FALSE);
int32_t op;
if (fModeFlags & UREGEX_DOTALL) {
op = URX_BUILD(URX_DOTANY_ALL, 0);
case doCaret:
{
+ fixLiterals(FALSE);
int32_t op = 0;
if ( (fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) {
op = URX_CARET;
case doDollar:
{
+ fixLiterals(FALSE);
int32_t op = 0;
if ( (fModeFlags & UREGEX_MULTILINE) == 0 && (fModeFlags & UREGEX_UNIX_LINES) == 0) {
op = URX_DOLLAR;
break;
case doBackslashA:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_CARET, 0), *fStatus);
break;
error(U_UNSUPPORTED_ERROR);
}
#endif
+ fixLiterals(FALSE);
int32_t op = (fModeFlags & UREGEX_UWORD)? URX_BACKSLASH_BU : URX_BACKSLASH_B;
fRXPat->fCompiledPat->addElement(URX_BUILD(op, 1), *fStatus);
}
error(U_UNSUPPORTED_ERROR);
}
#endif
+ fixLiterals(FALSE);
int32_t op = (fModeFlags & UREGEX_UWORD)? URX_BACKSLASH_BU : URX_BACKSLASH_B;
fRXPat->fCompiledPat->addElement(URX_BUILD(op, 0), *fStatus);
}
break;
case doBackslashD:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_BACKSLASH_D, 1), *fStatus);
break;
case doBackslashd:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_BACKSLASH_D, 0), *fStatus);
break;
case doBackslashG:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_BACKSLASH_G, 0), *fStatus);
break;
case doBackslashS:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(
URX_BUILD(URX_STAT_SETREF_N, URX_ISSPACE_SET), *fStatus);
break;
case doBackslashs:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(
URX_BUILD(URX_STATIC_SETREF, URX_ISSPACE_SET), *fStatus);
break;
case doBackslashW:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(
URX_BUILD(URX_STAT_SETREF_N, URX_ISWORD_SET), *fStatus);
break;
case doBackslashw:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(
URX_BUILD(URX_STATIC_SETREF, URX_ISWORD_SET), *fStatus);
break;
case doBackslashX:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_BACKSLASH_X, 0), *fStatus);
break;
case doBackslashZ:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_DOLLAR, 0), *fStatus);
break;
case doBackslashz:
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_BACKSLASH_Z, 0), *fStatus);
break;
break;
case doExit:
+ fixLiterals(FALSE);
returnVal = FALSE;
break;
case doProperty:
{
+ fixLiterals(FALSE);
UnicodeSet *theSet = scanProp();
compileSet(theSet);
}
// Because capture groups can be forward-referenced by back-references,
// we fill the operand with the capture group number. At the end
// of compilation, it will be changed to the variable's location.
- U_ASSERT(groupNum > 0);
+ U_ASSERT(groupNum > 0); // Shouldn't happen. '\0' begins an octal escape sequence,
+ // and shouldn't enter this code path at all.
+ fixLiterals(FALSE);
int32_t op;
if (fModeFlags & UREGEX_CASE_INSENSITIVE) {
op = URX_BUILD(URX_BACKREF_I, groupNum);
break;
case doSetMatchMode:
+ // Emit code to match any pending literals, using the not-yet changed match mode.
+ fixLiterals();
+
// We've got a (?i) or similar. The match mode is being changed, but
// the change is not scoped to a parenthesized block.
U_ASSERT(fNewModeFlags < 0);
fModeFlags = fNewModeFlags;
- // Prevent any string from spanning across the change of match mode.
- // Otherwise the pattern "abc(?i)def" would make a single string of "abcdef"
- fixLiterals();
break;
// - NOP, which may later be replaced by a save-state if there
// is an '|' alternation within the parens.
{
+ fixLiterals(FALSE);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
fRXPat->fCompiledPat->addElement(URX_BUILD(URX_NOP, 0), *fStatus);
}
case doSetBegin:
+ fixLiterals(FALSE);
fSetStack.push(new UnicodeSet(), *fStatus);
fSetOpStack.push(setStart, *fStatus);
if ((fModeFlags & UREGEX_CASE_INSENSITIVE) != 0) {
break;
}
-
default:
U_ASSERT(FALSE);
error(U_REGEX_INTERNAL_ERROR);
// or an escape sequence that reduces to a character.
// Add it to the string containing all literal chars/strings from
// the pattern.
-// If we are in a pattern string already, add the new char to it.
-// If we aren't in a pattern string, begin one now.
//
//------------------------------------------------------------------------------
void RegexCompile::literalChar(UChar32 c) {
- int32_t op; // An operation in the compiled pattern.
- int32_t opType;
- int32_t patternLoc; // A position in the compiled pattern.
- int32_t stringLen;
-
-
- // If the last thing compiled into the pattern was not a literal char,
- // force this new literal char to begin a new string, and not append to the previous.
- op = (int32_t)fRXPat->fCompiledPat->lastElementi();
- opType = URX_TYPE(op);
- if (!(opType == URX_STRING_LEN || opType == URX_ONECHAR || opType == URX_ONECHAR_I)) {
- fixLiterals();
- }
-
- if (fStringOpStart == -1) {
- // First char of a string in the pattern.
- // Emit a OneChar op into the compiled pattern.
- emitONE_CHAR(c);
-
- // Mark that we might actually be starting a string here
- fStringOpStart = fRXPat->fLiteralText.length();
- return;
- }
-
- op = (int32_t)fRXPat->fCompiledPat->lastElementi();
- opType = URX_TYPE(op);
- U_ASSERT(opType == URX_ONECHAR || opType == URX_ONECHAR_I || opType == URX_STRING_LEN);
-
- // If the most recently emitted op is a URX_ONECHAR,
- if (opType == URX_ONECHAR || opType == URX_ONECHAR_I) {
- if (U16_IS_TRAIL(c) && U16_IS_LEAD(URX_VAL(op))) {
- // The most recently emitted op is a ONECHAR that was the first half
- // of a surrogate pair. Update the ONECHAR's operand to be the
- // supplementary code point resulting from both halves of the pair.
- c = U16_GET_SUPPLEMENTARY(URX_VAL(op), c);
- op = URX_BUILD(opType, c);
- patternLoc = fRXPat->fCompiledPat->size() - 1;
- fRXPat->fCompiledPat->setElementAt(op, patternLoc);
- return;
- }
-
- // The most recently emitted op is a ONECHAR.
- // We've now received another adjacent char. Change the ONECHAR op
- // to a string op.
- fRXPat->fLiteralText.append(URX_VAL(op));
-
- if (fModeFlags & UREGEX_CASE_INSENSITIVE) {
- op = URX_BUILD(URX_STRING_I, fStringOpStart);
- } else {
- op = URX_BUILD(URX_STRING, fStringOpStart);
- }
- patternLoc = fRXPat->fCompiledPat->size() - 1;
- fRXPat->fCompiledPat->setElementAt(op, patternLoc);
- op = URX_BUILD(URX_STRING_LEN, 0);
- fRXPat->fCompiledPat->addElement(op, *fStatus);
- }
-
- // We are adding onto an existing string
- fRXPat->fLiteralText.append(c);
-
- // The pattern contains a URX_SRING / URX_STRING_LEN. Update the
- // string length to reflect the new char we just added to the string.
- stringLen = fRXPat->fLiteralText.length() - fStringOpStart;
- op = URX_BUILD(URX_STRING_LEN, stringLen);
- patternLoc = fRXPat->fCompiledPat->size() - 1;
- fRXPat->fCompiledPat->setElementAt(op, patternLoc);
-}
-
-
-
-//------------------------------------------------------------------------------
-//
-// emitONE_CHAR emit a ONE_CHAR op into the generated code.
-// Choose cased or uncased version, depending on the
-// match mode and whether the character itself is cased.
-//
-//------------------------------------------------------------------------------
-void RegexCompile::emitONE_CHAR(UChar32 c) {
- int32_t op;
- if ((fModeFlags & UREGEX_CASE_INSENSITIVE) &&
- u_hasBinaryProperty(c, UCHAR_CASE_SENSITIVE)) {
- // We have a cased character, and are in case insensitive matching mode.
- //c = u_foldCase(c, U_FOLD_CASE_DEFAULT); // !!!: handled in stripNOPs() now
- op = URX_BUILD(URX_ONECHAR_I, c);
- } else {
- // Uncased char, or case sensitive match mode.
- // Either way, just generate a literal compare of the char.
- op = URX_BUILD(URX_ONECHAR, c);
- }
- fRXPat->fCompiledPat->addElement(op, *fStatus);
+ fLiteralChars.append(c);
}
//------------------------------------------------------------------------------
//
// fixLiterals When compiling something that can follow a literal
-// string in a pattern, we need to "fix" any preceding
-// string, which will cause any subsequent literals to
-// begin a new string, rather than appending to the
-// old one.
+// string in a pattern, emit the code to match the
+// accumulated literal string.
//
// Optionally, split the last char of the string off into
// a single "ONE_CHAR" operation, so that quantifiers can
//
//------------------------------------------------------------------------------
void RegexCompile::fixLiterals(UBool split) {
- int32_t stringStart = fStringOpStart; // start index of the current literal string
- int32_t op; // An op from/for the compiled pattern.
- int32_t opType; // An opcode type from the compiled pattern.
- int32_t stringLastCharIdx;
- UChar32 lastChar;
- int32_t stringNextToLastCharIdx;
- UChar32 nextToLastChar;
- int32_t stringLen;
-
- fStringOpStart = -1;
- if (!split) {
+ int32_t op = 0; // An op from/for the compiled pattern.
+
+ // If no literal characters have been scanned but not yet had code generated
+ // for them, nothing needs to be done.
+ if (fLiteralChars.length() == 0) {
return;
}
- // Split: We need to ensure that the last item in the compiled pattern does
- // not refer to a literal string of more than one char. If it does,
- // separate the last char from the rest of the string.
+ int32_t indexOfLastCodePoint = fLiteralChars.moveIndex32(fLiteralChars.length(), -1);
+ UChar32 lastCodePoint = fLiteralChars.char32At(indexOfLastCodePoint);
- // If the last operation from the compiled pattern is not a string,
- // nothing needs to be done
- op = (int32_t)fRXPat->fCompiledPat->lastElementi();
- opType = URX_TYPE(op);
- if (opType != URX_STRING_LEN) {
+ // Split: We need to ensure that the last item in the compiled pattern
+ // refers only to the last literal scanned in the pattern, so that
+ // quantifiers (*, +, etc.) affect only it, and not a longer string.
+ // Split before case folding for case insensitive matches.
+
+ if (split) {
+ fLiteralChars.truncate(indexOfLastCodePoint);
+ fixLiterals(FALSE); // Recursive call, emit code to match the first part of the string.
+ // Note that the truncated literal string may be empty, in which case
+ // nothing will be emitted.
+
+ literalChar(lastCodePoint); // Re-add the last code point as if it were a new literal.
+ fixLiterals(FALSE); // Second recursive call, code for the final code point.
return;
}
- stringLen = URX_VAL(op);
- //
- // Find the position of the last code point in the string (might be a surrogate pair)
- //
- stringLastCharIdx = fRXPat->fLiteralText.length();
- stringLastCharIdx = fRXPat->fLiteralText.moveIndex32(stringLastCharIdx, -1);
- lastChar = fRXPat->fLiteralText.char32At(stringLastCharIdx);
-
- // The string should always be at least two code points long, meaning that there
- // should be something before the last char position that we just found.
- U_ASSERT(stringLastCharIdx > stringStart);
- stringNextToLastCharIdx = fRXPat->fLiteralText.moveIndex32(stringLastCharIdx, -1);
- U_ASSERT(stringNextToLastCharIdx >= stringStart);
- nextToLastChar = fRXPat->fLiteralText.char32At(stringNextToLastCharIdx);
-
- if (stringNextToLastCharIdx > stringStart) {
- // The length of string remaining after removing one char is two or more.
- // Leave the string in the compiled pattern, shorten it by one char,
- // and append a URX_ONECHAR op for the last char.
- stringLen -= (fRXPat->fLiteralText.length() - stringLastCharIdx);
- op = URX_BUILD(URX_STRING_LEN, stringLen);
- fRXPat->fCompiledPat->setElementAt(op, fRXPat->fCompiledPat->size() -1);
- emitONE_CHAR(lastChar);
+ // If we are doing case-insensitive matching, case fold the string. This may expand
+ // the string, e.g. the German sharp-s turns into "ss"
+ if (fModeFlags & UREGEX_CASE_INSENSITIVE) {
+ fLiteralChars.foldCase();
+ indexOfLastCodePoint = fLiteralChars.moveIndex32(fLiteralChars.length(), -1);
+ lastCodePoint = fLiteralChars.char32At(indexOfLastCodePoint);
+ }
+
+ if (indexOfLastCodePoint == 0) {
+ // Single character, emit a URX_ONECHAR op to match it.
+ if ((fModeFlags & UREGEX_CASE_INSENSITIVE) &&
+ u_hasBinaryProperty(lastCodePoint, UCHAR_CASE_SENSITIVE)) {
+ op = URX_BUILD(URX_ONECHAR_I, lastCodePoint);
+ } else {
+ op = URX_BUILD(URX_ONECHAR, lastCodePoint);
+ }
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
} else {
- // The original string consisted of exactly two characters. Replace
- // the existing compiled URX_STRING/URX_STRING_LEN ops with a pair
- // of URX_ONECHARs.
- fRXPat->fCompiledPat->setSize(fRXPat->fCompiledPat->size() -2);
- emitONE_CHAR(nextToLastChar);
- emitONE_CHAR(lastChar);
+ // Two or more chars, emit a URX_STRING to match them.
+ if (fModeFlags & UREGEX_CASE_INSENSITIVE) {
+ op = URX_BUILD(URX_STRING_I, fRXPat->fLiteralText.length());
+ } else {
+ // TODO here: add optimization to split case sensitive strings of length two
+ // into two single char ops, for efficiency.
+ op = URX_BUILD(URX_STRING, fRXPat->fLiteralText.length());
+ }
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+ op = URX_BUILD(URX_STRING_LEN, fLiteralChars.length());
+ fRXPat->fCompiledPat->addElement(op, *fStatus);
+
+ // Add this string into the accumulated strings of the compiled pattern.
+ fRXPat->fLiteralText.append(fLiteralChars);
}
+
+ fLiteralChars.remove();
}
opType == URX_CTR_LOOP ||
opType == URX_CTR_LOOP_NG ||
opType == URX_JMP_SAV ||
+ opType == URX_JMP_SAV_X ||
opType == URX_RELOC_OPRND) && opValue > where) {
// Target location for this opcode is after the insertion point and
// needs to be incremented to adjust for the insertion.
//------------------------------------------------------------------------------
int32_t RegexCompile::blockTopLoc(UBool reserveLoc) {
int32_t theLoc;
+ fixLiterals(TRUE); // Emit code for any pending literals.
+ // If last item was a string, emit separate op for the its last char.
if (fRXPat->fCompiledPat->size() == fMatchCloseParen)
{
// The item just processed is a parenthesized block.
U_ASSERT(URX_TYPE(((uint32_t)fRXPat->fCompiledPat->elementAti(theLoc))) == URX_NOP);
}
else {
- // Item just compiled is a single thing, a ".", or a single char, or a set reference.
+ // Item just compiled is a single thing, a ".", or a single char, a string or a set reference.
// No slot for STATE_SAVE was pre-reserved in the compiled code.
// We need to make space now.
- fixLiterals(TRUE); // If last item was a string, separate the last char.
theLoc = fRXPat->fCompiledPat->size()-1;
+ int32_t opAtTheLoc = (int32_t)fRXPat->fCompiledPat->elementAti(theLoc);
+ if (URX_TYPE(opAtTheLoc) == URX_STRING_LEN) {
+ // Strings take two opcode, we want the position of the first one.
+ // We can have a string at this point if a single character case-folded to two.
+ theLoc--;
+ }
if (reserveLoc) {
- /*int32_t opAtTheLoc = fRXPat->fCompiledPat->elementAti(theLoc);*/
int32_t nop = URX_BUILD(URX_NOP, 0);
fRXPat->fCompiledPat->insertElementAt(nop, theLoc, *fStatus);
}
return;
}
- // Force any literal chars that may follow the close paren to start a new string,
- // and not attach to any preceding it.
+ // Emit code for any pending literals.
fixLiterals(FALSE);
// Fixup any operations within the just-closed parenthesized group
if (currentLen == 0) {
UChar32 c = URX_VAL(op);
if (u_hasBinaryProperty(c, UCHAR_CASE_SENSITIVE)) {
- // character may have distinct cased forms. Add all of them
- // to the set of possible starting match chars.
- UnicodeSet s(c, c);
- s.closeOver(USET_CASE_INSENSITIVE);
- fRXPat->fInitialChars->addAll(s);
+
+ // Disable optimizations on first char of match.
+ // TODO: Compute the set of chars that case fold to this char, or to
+ // a string that begins with this char.
+ // For simple case folding, this code worked:
+ // UnicodeSet s(c, c);
+ // s.closeOver(USET_CASE_INSENSITIVE);
+ // fRXPat->fInitialChars->addAll(s);
+
+ fRXPat->fInitialChars->clear();
+ fRXPat->fInitialChars->complement();
} else {
// Char has no case variants. Just add it as-is to the
// set of possible starting chars.
int32_t stringStartIdx = URX_VAL(op);
UChar32 c = fRXPat->fLiteralText.char32At(stringStartIdx);
UnicodeSet s(c, c);
- s.closeOver(USET_CASE_INSENSITIVE);
+
+ // TODO: compute correct set of starting chars for full case folding.
+ // For the moment, say any char can start.
+ // s.closeOver(USET_CASE_INSENSITIVE);
+ s.clear();
+ s.complement();
+
fRXPat->fInitialChars->addAll(s);
numInitialStrings += 2; // Matching on an initial string not possible.
}
case URX_STRING:
- case URX_STRING_I:
{
loc++;
int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc);
break;
+ case URX_STRING_I:
+ {
+ loc++;
+ // TODO: with full case folding, matching input text may be shorter than
+ // the string we have here. More smarts could put some bounds on it.
+ // Assume a min length of one for now. A min length of zero causes
+ // optimization failures for a pattern like "string"+
+ // currentLen += URX_VAL(stringLenOp);
+ currentLen += 1;
+ }
+ break;
+
case URX_CTR_INIT:
case URX_CTR_INIT_NG:
{
return currentLen;
}
+// Increment with overflow check.
+// val and delta will both be positive.
+
+static int32_t safeIncrement(int32_t val, int32_t delta) {
+ if (INT32_MAX - val > delta) {
+ return val + delta;
+ } else {
+ return INT32_MAX;
+ }
+}
//------------------------------------------------------------------------------
case URX_DOTANY_ALL:
case URX_DOTANY:
case URX_DOTANY_UNIX:
- currentLen+=2;
+ currentLen = safeIncrement(currentLen, 2);
break;
// Single literal character. Increase current max length by one or two,
// depending on whether the char is in the supplementary range.
case URX_ONECHAR:
- currentLen++;
+ currentLen = safeIncrement(currentLen, 1);
if (URX_VAL(op) > 0x10000) {
- currentLen++;
+ currentLen = safeIncrement(currentLen, 1);
}
break;
case URX_STRING:
+ {
+ loc++;
+ int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc);
+ currentLen = safeIncrement(currentLen, URX_VAL(stringLenOp));
+ break;
+ }
+
case URX_STRING_I:
+ // TODO: This code assumes that any user string that matches will be no longer
+ // than our compiled string, with case insensitive matching.
+ // Our compiled string has been case-folded already.
+ //
+ // Any matching user string will have no more code points than our
+ // compiled (folded) string. Folding may add code points, but
+ // not remove them.
+ //
+ // There is a potential problem if a supplemental code point
+ // case-folds to a BMP code point. In this case our compiled string
+ // could be shorter (in code units) than a matching user string.
+ //
+ // At this time (Unicode 6.1) there are no such characters, and this case
+ // is not being handled. A test, intltest regex/Bug9283, will fail if
+ // any problematic characters are added to Unicode.
+ //
+ // If this happens, we can make a set of the BMP chars that the
+ // troublesome supplementals fold to, scan our string, and bump the
+ // currentLen one extra for each that is found.
+ //
{
loc++;
int32_t stringLenOp = (int32_t)fRXPat->fCompiledPat->elementAti(loc);
- currentLen += URX_VAL(stringLenOp);
+ currentLen = safeIncrement(currentLen, URX_VAL(stringLenOp));
}
break;
-
case URX_CTR_INIT:
case URX_CTR_INIT_NG:
case URX_CTR_LOOP:
// back-reference fixup is also performed here (adjusting
// back-reference operands to point to the correct frame offsets).
//
-// In addition, case-insensitive character and string literals are
-// now case-folded here, rather than when first parsed or at match
-// time.
-//
//------------------------------------------------------------------------------
void RegexCompile::stripNOPs() {
}
UnicodeString caseStringBuffer;
- int32_t stringDelta = 0;
// Make a second pass over the code, removing the NOPs by moving following
// code up, and patching operands that refer to code locations that
break;
}
- case URX_ONECHAR_I:
- {
- UChar32 c = URX_VAL(op);
- if (u_hasBinaryProperty(c, UCHAR_CASE_SENSITIVE)) {
- // We have a cased character to fold
- c = u_foldCase(c, U_FOLD_CASE_DEFAULT);
- op = URX_BUILD(URX_ONECHAR_I, c);
- }
-
- fRXPat->fCompiledPat->setElementAt(op, dst);
- dst++;
- break;
- }
- case URX_STRING_I:
- {
- op = URX_BUILD(URX_STRING_I, URX_VAL(op)+stringDelta);
-
- src++;
- int32_t lengthOp = (int32_t)fRXPat->fCompiledPat->elementAti(src);
-
- caseStringBuffer.setTo(fRXPat->fLiteralText, URX_VAL(op), URX_VAL(lengthOp));
- caseStringBuffer.foldCase(U_FOLD_CASE_DEFAULT);
-
- int32_t newLen = caseStringBuffer.length();
- if (newLen <= URX_VAL(lengthOp)) {
- // don't shift if we don't have to, take the tiny memory hit of a smaller string
- fRXPat->fLiteralText.replace(URX_VAL(op), newLen, caseStringBuffer);
- } else {
- // shift other strings over...at least UnicodeString handles this for us!
- fRXPat->fLiteralText.replace(URX_VAL(op), URX_VAL(lengthOp), caseStringBuffer);
- stringDelta += newLen - URX_VAL(lengthOp);
- }
- lengthOp = URX_BUILD(URX_STRING_LEN, newLen);
-
- fRXPat->fCompiledPat->setElementAt(op, dst);
- fRXPat->fCompiledPat->setElementAt(lengthOp, dst+1);
- dst += 2;
- break;
- }
case URX_BACKREF:
case URX_BACKREF_I:
{
fRXPat->fNeedsAltInput = TRUE;
break;
}
- case URX_STRING:
- op = URX_BUILD(URX_STRING, URX_VAL(op)+stringDelta);
- // continue
case URX_RESERVED_OP:
case URX_RESERVED_OP_N:
case URX_BACKTRACK:
case URX_END:
case URX_ONECHAR:
+ case URX_STRING:
case URX_STRING_LEN:
case URX_START_CAPTURE:
case URX_END_CAPTURE:
case URX_STO_INP_LOC:
case URX_LA_START:
case URX_LA_END:
+ case URX_ONECHAR_I:
+ case URX_STRING_I:
case URX_DOLLAR_M:
case URX_CARET_M:
case URX_CARET_M_UNIX:
static const UChar chColon = 0x3A; // ':'
static const UChar chE = 0x45; // 'E'
static const UChar chQ = 0x51; // 'Q'
-static const UChar chN = 0x4E; // 'N'
+//static const UChar chN = 0x4E; // 'N'
static const UChar chP = 0x50; // 'P'
static const UChar chBackSlash = 0x5c; // '\' introduces a char escape
-static const UChar chLBracket = 0x5b; // '['
+//static const UChar chLBracket = 0x5b; // '['
static const UChar chRBracket = 0x5d; // ']'
static const UChar chUp = 0x5e; // '^'
static const UChar chLowerP = 0x70;
if (fQuoteMode) {
c.fQuoted = TRUE;
- if ((c.fChar==chBackSlash && peekCharLL()==chE) || c.fChar == (UChar32)-1) {
+ if ((c.fChar==chBackSlash && peekCharLL()==chE && ((fModeFlags & UREGEX_LITERAL) == 0)) ||
+ c.fChar == (UChar32)-1) {
fQuoteMode = FALSE; // Exit quote mode,
- nextCharLL(); // discard the E
- nextChar(c); // recurse to get the real next char
+ nextCharLL(); // discard the E
+ nextChar(c); // recurse to get the real next char
}
}
else if (fInBackslashQuote) {
}
}
// TODO: check what Java & Perl do with non-ASCII white spaces. Ticket 6061.
- if (uprv_isRuleWhiteSpace(c.fChar) == FALSE) {
+ if (PatternProps::isWhiteSpace(c.fChar) == FALSE) {
break;
}
c.fChar = nextCharLL();
projects / apple / icu.git / blobdiff