X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/f59164e3d128c7675a4d3934206346a3384e53a5..a01113dcd0f39d5da295ef82785beff9ed86fe38:/icuSources/common/utext.cpp?ds=sidebyside diff --git a/icuSources/common/utext.cpp b/icuSources/common/utext.cpp index de279eba..cd969e73 100644 --- a/icuSources/common/utext.cpp +++ b/icuSources/common/utext.cpp @@ -1,3 +1,5 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html /* ******************************************************************************* * @@ -6,7 +8,7 @@ * ******************************************************************************* * file name: utext.cpp -* encoding: US-ASCII +* encoding: UTF-8 * tab size: 8 (not used) * indentation:4 * @@ -105,6 +107,8 @@ utext_getNativeIndex(const UText *ut) { U_CAPI void U_EXPORT2 utext_setNativeIndex(UText *ut, int64_t index) { + // Apple note, at entry ut->chunkContents may be 0, not necessarily a problem + // (CF funcs will have set chunkNativeStart/Limit to 0 forcing call to access) if(indexchunkNativeStart || index>=ut->chunkNativeLimit) { // The desired position is outside of the current chunk. // Access the new position. Assume a forward iteration from here, @@ -115,8 +119,11 @@ utext_setNativeIndex(UText *ut, int64_t index) { // utf-16 indexing. ut->chunkOffset=(int32_t)(index-ut->chunkNativeStart); } else { - ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index); + ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index); } + // Apple note, it can still be valid to have ut->chunkContents==0 at this + // point (just not inside the next block), see + // The convention is that the index must always be on a code point boundary. // Adjust the index position if it is in the middle of a surrogate pair. if (ut->chunkOffsetchunkLength) { @@ -845,18 +852,13 @@ U_CDECL_END //------------------------------------------------------------------------------ // Chunk size. -// Must be less than 42 (256/6), because of byte mapping from UChar indexes to native indexes. -// Worst case there are six UTF-8 bytes per UChar. -// obsolete 6 byte form fd + 5 trails maps to fffd -// obsolete 5 byte form fc + 4 trails maps to fffd -// non-shortest 4 byte forms maps to fffd -// normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit -// mapToUChars array size must allow for the worst case, 6. -// This could be brought down to 4, by treating fd and fc as pure illegal, -// rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros. +// Must be less than 85 (256/3), because of byte mapping from UChar indexes to native indexes. +// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes +// to two UChars.) +// The longest illegal byte sequence treated as a single error (and converted to U+FFFD) +// is a three-byte sequence (truncated four-byte sequence). // enum { UTF8_TEXT_CHUNK_SIZE=32 }; -enum { MAP_TO_UCHARS_SIZE=UTF8_TEXT_CHUNK_SIZE*6+6 }; // // UTF8Buf Two of these structs will be set up in the UText's extra allocated space. @@ -894,8 +896,8 @@ struct UTF8Buf { // Requires two extra slots, // one for a supplementary starting in the last normal position, // and one for an entry for the buffer limit position. - uint8_t mapToUChars[MAP_TO_UCHARS_SIZE]; // Map native offset from bufNativeStart to - // corresponding offset in filled part of buf. + uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to + // correspoding offset in filled part of buf. int32_t align; }; @@ -1037,7 +1039,7 @@ utf8TextAccess(UText *ut, int64_t index, UBool forward) { // Requested index is in this buffer. u8b = (UTF8Buf *)ut->p; // the current buffer mapIndex = ix - u8b->toUCharsMapStart; - U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE); + U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars)); ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; return TRUE; @@ -1199,9 +1201,9 @@ fillForward: // Swap the UText buffers. // We want to fill what was previously the alternate buffer, // and make what was the current buffer be the new alternate. - UTF8Buf *u8b = (UTF8Buf *)ut->q; + UTF8Buf *u8b_swap = (UTF8Buf *)ut->q; ut->q = ut->p; - ut->p = u8b; + ut->p = u8b_swap; int32_t strLen = ut->b; UBool nulTerminated = FALSE; @@ -1210,9 +1212,9 @@ fillForward: nulTerminated = TRUE; } - UChar *buf = u8b->buf; - uint8_t *mapToNative = u8b->mapToNative; - uint8_t *mapToUChars = u8b->mapToUChars; + UChar *buf = u8b_swap->buf; + uint8_t *mapToNative = u8b_swap->mapToNative; + uint8_t *mapToUChars = u8b_swap->mapToUChars; int32_t destIx = 0; int32_t srcIx = ix; UBool seenNonAscii = FALSE; @@ -1233,7 +1235,7 @@ fillForward: // General case, handle everything. if (seenNonAscii == FALSE) { seenNonAscii = TRUE; - u8b->bufNILimit = destIx; + u8b_swap->bufNILimit = destIx; } int32_t cIx = srcIx; @@ -1266,22 +1268,22 @@ fillForward: mapToUChars[srcIx - ix] = (uint8_t)destIx; // fill in Buffer descriptor - u8b->bufNativeStart = ix; - u8b->bufNativeLimit = srcIx; - u8b->bufStartIdx = 0; - u8b->bufLimitIdx = destIx; + u8b_swap->bufNativeStart = ix; + u8b_swap->bufNativeLimit = srcIx; + u8b_swap->bufStartIdx = 0; + u8b_swap->bufLimitIdx = destIx; if (seenNonAscii == FALSE) { - u8b->bufNILimit = destIx; + u8b_swap->bufNILimit = destIx; } - u8b->toUCharsMapStart = u8b->bufNativeStart; + u8b_swap->toUCharsMapStart = u8b_swap->bufNativeStart; // Set UText chunk to refer to this buffer. ut->chunkContents = buf; ut->chunkOffset = 0; - ut->chunkLength = u8b->bufLimitIdx; - ut->chunkNativeStart = u8b->bufNativeStart; - ut->chunkNativeLimit = u8b->bufNativeLimit; - ut->nativeIndexingLimit = u8b->bufNILimit; + ut->chunkLength = u8b_swap->bufLimitIdx; + ut->chunkNativeStart = u8b_swap->bufNativeStart; + ut->chunkNativeLimit = u8b_swap->bufNativeLimit; + ut->nativeIndexingLimit = u8b_swap->bufNILimit; // For zero terminated strings, keep track of the maximum point // scanned so far. @@ -1314,14 +1316,14 @@ fillReverse: // Swap the UText buffers. // We want to fill what was previously the alternate buffer, // and make what was the current buffer be the new alternate. - UTF8Buf *u8b = (UTF8Buf *)ut->q; + UTF8Buf *u8b_swap = (UTF8Buf *)ut->q; ut->q = ut->p; - ut->p = u8b; + ut->p = u8b_swap; - UChar *buf = u8b->buf; - uint8_t *mapToNative = u8b->mapToNative; - uint8_t *mapToUChars = u8b->mapToUChars; - int32_t toUCharsMapStart = ix - MAP_TO_UCHARS_SIZE + 1; + UChar *buf = u8b_swap->buf; + uint8_t *mapToNative = u8b_swap->mapToNative; + uint8_t *mapToUChars = u8b_swap->mapToUChars; + int32_t toUCharsMapStart = ix - sizeof(UTF8Buf::mapToUChars) + 1; // Note that toUCharsMapStart can be negative. Happens when the remaining // text from current position to the beginning is less than the buffer size. // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry. @@ -1390,19 +1392,19 @@ fillReverse: bufNILimit = destIx; } } - u8b->bufNativeStart = srcIx; - u8b->bufNativeLimit = ix; - u8b->bufStartIdx = destIx; - u8b->bufLimitIdx = UTF8_TEXT_CHUNK_SIZE+2; - u8b->bufNILimit = bufNILimit - u8b->bufStartIdx; - u8b->toUCharsMapStart = toUCharsMapStart; - - ut->chunkContents = &buf[u8b->bufStartIdx]; - ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx; + u8b_swap->bufNativeStart = srcIx; + u8b_swap->bufNativeLimit = ix; + u8b_swap->bufStartIdx = destIx; + u8b_swap->bufLimitIdx = UTF8_TEXT_CHUNK_SIZE+2; + u8b_swap->bufNILimit = bufNILimit - u8b_swap->bufStartIdx; + u8b_swap->toUCharsMapStart = toUCharsMapStart; + + ut->chunkContents = &buf[u8b_swap->bufStartIdx]; + ut->chunkLength = u8b_swap->bufLimitIdx - u8b_swap->bufStartIdx; ut->chunkOffset = ut->chunkLength; - ut->chunkNativeStart = u8b->bufNativeStart; - ut->chunkNativeLimit = u8b->bufNativeLimit; - ut->nativeIndexingLimit = u8b->bufNILimit; + ut->chunkNativeStart = u8b_swap->bufNativeStart; + ut->chunkNativeLimit = u8b_swap->bufNativeLimit; + ut->nativeIndexingLimit = u8b_swap->bufNILimit; return TRUE; } @@ -1556,7 +1558,7 @@ utf8TextMapIndexToUTF16(const UText *ut, int64_t index64) { U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit); U_ASSERT(index<=ut->chunkNativeLimit); int32_t mapIndex = index - u8b->toUCharsMapStart; - U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE); + U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars)); int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; U_ASSERT(offset>=0 && offset<=ut->chunkLength); return offset; @@ -2241,13 +2243,13 @@ unistrTextCopy(UText *ut, } if(move) { - // move: copy to destIndex, then replace original with nothing + // move: copy to destIndex, then remove original int32_t segLength=limit32-start32; us->copy(start32, limit32, destIndex32); if(destIndex32replace(start32, segLength, NULL, 0); + us->remove(start32, segLength); } else { // copy us->copy(start32, limit32, destIndex32);