+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
*
*******************************************************************************
* file name: utext.cpp
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
U_CAPI void U_EXPORT2
utext_setNativeIndex(UText *ut, int64_t index) {
+ // Apple note, at entry ut->chunkContents may be 0, not necessarily a problem
+ // (CF funcs will have set chunkNativeStart/Limit to 0 forcing call to access)
if(index<ut->chunkNativeStart || index>=ut->chunkNativeLimit) {
// The desired position is outside of the current chunk.
// Access the new position. Assume a forward iteration from here,
// utf-16 indexing.
ut->chunkOffset=(int32_t)(index-ut->chunkNativeStart);
} else {
- ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index);
+ ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index);
}
+ // Apple note, it can still be valid to have ut->chunkContents==0 at this
+ // point (just not inside the next block), see <rdar://problem/53610517>
+
// The convention is that the index must always be on a code point boundary.
// Adjust the index position if it is in the middle of a surrogate pair.
if (ut->chunkOffset<ut->chunkLength) {
//------------------------------------------------------------------------------
// Chunk size.
-// Must be less than 42 (256/6), because of byte mapping from UChar indexes to native indexes.
-// Worst case there are six UTF-8 bytes per UChar.
-// obsolete 6 byte form fd + 5 trails maps to fffd
-// obsolete 5 byte form fc + 4 trails maps to fffd
-// non-shortest 4 byte forms maps to fffd
-// normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit
-// mapToUChars array size must allow for the worst case, 6.
-// This could be brought down to 4, by treating fd and fc as pure illegal,
-// rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros.
+// Must be less than 85 (256/3), because of byte mapping from UChar indexes to native indexes.
+// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes
+// to two UChars.)
+// The longest illegal byte sequence treated as a single error (and converted to U+FFFD)
+// is a three-byte sequence (truncated four-byte sequence).
//
enum { UTF8_TEXT_CHUNK_SIZE=32 };
-enum { MAP_TO_UCHARS_SIZE=UTF8_TEXT_CHUNK_SIZE*6+6 };
//
// UTF8Buf Two of these structs will be set up in the UText's extra allocated space.
// Requires two extra slots,
// one for a supplementary starting in the last normal position,
// and one for an entry for the buffer limit position.
- uint8_t mapToUChars[MAP_TO_UCHARS_SIZE]; // Map native offset from bufNativeStart to
- // corresponding offset in filled part of buf.
+ uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to
+ // correspoding offset in filled part of buf.
int32_t align;
};
// Requested index is in this buffer.
u8b = (UTF8Buf *)ut->p; // the current buffer
mapIndex = ix - u8b->toUCharsMapStart;
- U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE);
+ U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars));
ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
return TRUE;
// Swap the UText buffers.
// We want to fill what was previously the alternate buffer,
// and make what was the current buffer be the new alternate.
- UTF8Buf *u8b = (UTF8Buf *)ut->q;
+ UTF8Buf *u8b_swap = (UTF8Buf *)ut->q;
ut->q = ut->p;
- ut->p = u8b;
+ ut->p = u8b_swap;
int32_t strLen = ut->b;
UBool nulTerminated = FALSE;
nulTerminated = TRUE;
}
- UChar *buf = u8b->buf;
- uint8_t *mapToNative = u8b->mapToNative;
- uint8_t *mapToUChars = u8b->mapToUChars;
+ UChar *buf = u8b_swap->buf;
+ uint8_t *mapToNative = u8b_swap->mapToNative;
+ uint8_t *mapToUChars = u8b_swap->mapToUChars;
int32_t destIx = 0;
int32_t srcIx = ix;
UBool seenNonAscii = FALSE;
// General case, handle everything.
if (seenNonAscii == FALSE) {
seenNonAscii = TRUE;
- u8b->bufNILimit = destIx;
+ u8b_swap->bufNILimit = destIx;
}
int32_t cIx = srcIx;
mapToUChars[srcIx - ix] = (uint8_t)destIx;
// fill in Buffer descriptor
- u8b->bufNativeStart = ix;
- u8b->bufNativeLimit = srcIx;
- u8b->bufStartIdx = 0;
- u8b->bufLimitIdx = destIx;
+ u8b_swap->bufNativeStart = ix;
+ u8b_swap->bufNativeLimit = srcIx;
+ u8b_swap->bufStartIdx = 0;
+ u8b_swap->bufLimitIdx = destIx;
if (seenNonAscii == FALSE) {
- u8b->bufNILimit = destIx;
+ u8b_swap->bufNILimit = destIx;
}
- u8b->toUCharsMapStart = u8b->bufNativeStart;
+ u8b_swap->toUCharsMapStart = u8b_swap->bufNativeStart;
// Set UText chunk to refer to this buffer.
ut->chunkContents = buf;
ut->chunkOffset = 0;
- ut->chunkLength = u8b->bufLimitIdx;
- ut->chunkNativeStart = u8b->bufNativeStart;
- ut->chunkNativeLimit = u8b->bufNativeLimit;
- ut->nativeIndexingLimit = u8b->bufNILimit;
+ ut->chunkLength = u8b_swap->bufLimitIdx;
+ ut->chunkNativeStart = u8b_swap->bufNativeStart;
+ ut->chunkNativeLimit = u8b_swap->bufNativeLimit;
+ ut->nativeIndexingLimit = u8b_swap->bufNILimit;
// For zero terminated strings, keep track of the maximum point
// scanned so far.
// Swap the UText buffers.
// We want to fill what was previously the alternate buffer,
// and make what was the current buffer be the new alternate.
- UTF8Buf *u8b = (UTF8Buf *)ut->q;
+ UTF8Buf *u8b_swap = (UTF8Buf *)ut->q;
ut->q = ut->p;
- ut->p = u8b;
+ ut->p = u8b_swap;
- UChar *buf = u8b->buf;
- uint8_t *mapToNative = u8b->mapToNative;
- uint8_t *mapToUChars = u8b->mapToUChars;
- int32_t toUCharsMapStart = ix - MAP_TO_UCHARS_SIZE + 1;
+ UChar *buf = u8b_swap->buf;
+ uint8_t *mapToNative = u8b_swap->mapToNative;
+ uint8_t *mapToUChars = u8b_swap->mapToUChars;
+ int32_t toUCharsMapStart = ix - sizeof(UTF8Buf::mapToUChars) + 1;
// Note that toUCharsMapStart can be negative. Happens when the remaining
// text from current position to the beginning is less than the buffer size.
// + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry.
bufNILimit = destIx;
}
}
- u8b->bufNativeStart = srcIx;
- u8b->bufNativeLimit = ix;
- u8b->bufStartIdx = destIx;
- u8b->bufLimitIdx = UTF8_TEXT_CHUNK_SIZE+2;
- u8b->bufNILimit = bufNILimit - u8b->bufStartIdx;
- u8b->toUCharsMapStart = toUCharsMapStart;
-
- ut->chunkContents = &buf[u8b->bufStartIdx];
- ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx;
+ u8b_swap->bufNativeStart = srcIx;
+ u8b_swap->bufNativeLimit = ix;
+ u8b_swap->bufStartIdx = destIx;
+ u8b_swap->bufLimitIdx = UTF8_TEXT_CHUNK_SIZE+2;
+ u8b_swap->bufNILimit = bufNILimit - u8b_swap->bufStartIdx;
+ u8b_swap->toUCharsMapStart = toUCharsMapStart;
+
+ ut->chunkContents = &buf[u8b_swap->bufStartIdx];
+ ut->chunkLength = u8b_swap->bufLimitIdx - u8b_swap->bufStartIdx;
ut->chunkOffset = ut->chunkLength;
- ut->chunkNativeStart = u8b->bufNativeStart;
- ut->chunkNativeLimit = u8b->bufNativeLimit;
- ut->nativeIndexingLimit = u8b->bufNILimit;
+ ut->chunkNativeStart = u8b_swap->bufNativeStart;
+ ut->chunkNativeLimit = u8b_swap->bufNativeLimit;
+ ut->nativeIndexingLimit = u8b_swap->bufNILimit;
return TRUE;
}
U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
U_ASSERT(index<=ut->chunkNativeLimit);
int32_t mapIndex = index - u8b->toUCharsMapStart;
- U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE);
+ U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars));
int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
U_ASSERT(offset>=0 && offset<=ut->chunkLength);
return offset;
}
if(move) {
- // move: copy to destIndex, then replace original with nothing
+ // move: copy to destIndex, then remove original
int32_t segLength=limit32-start32;
us->copy(start32, limit32, destIndex32);
if(destIndex32<start32) {
start32+=segLength;
}
- us->replace(start32, segLength, NULL, 0);
+ us->remove(start32, segLength);
} else {
// copy
us->copy(start32, limit32, destIndex32);
projects / apple / icu.git / blobdiff