ICU-64260.0.1.tar.gz

[apple/icu.git] / icuSources / common / utext.cpp

diff --git a/icuSources/common/utext.cpp b/icuSources/common/utext.cpp
index de279eba501f212057e3a1681ec37e620f5e1f54..cd969e73bbc797183c2110003fae8ce001ba0e9b 100644 (file)
--- a/icuSources/common/utext.cpp
+++ b/icuSources/common/utext.cpp
@@ -1,3 +1,5 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html

 /*
 *******************************************************************************
 *
 /*
 *******************************************************************************
 *


@@ -6,7 +8,7 @@
 *
 *******************************************************************************
 *   file name:  utext.cpp
 *
 *******************************************************************************
 *   file name:  utext.cpp

-*   encoding:   US-ASCII
+*   encoding:   UTF-8

 *   tab size:   8 (not used)
 *   indentation:4
 *
 *   tab size:   8 (not used)
 *   indentation:4
 *


@@ -105,6 +107,8 @@ utext_getNativeIndex(const UText *ut) {
 
 U_CAPI void U_EXPORT2
 utext_setNativeIndex(UText *ut, int64_t index) {
 
 U_CAPI void U_EXPORT2
 utext_setNativeIndex(UText *ut, int64_t index) {

+    // Apple note, at entry ut->chunkContents may be 0, not necessarily a problem
+    // (CF funcs will have set chunkNativeStart/Limit to 0 forcing call to access)

     if(index<ut->chunkNativeStart || index>=ut->chunkNativeLimit) {
         // The desired position is outside of the current chunk.
         // Access the new position.  Assume a forward iteration from here,
     if(index<ut->chunkNativeStart || index>=ut->chunkNativeLimit) {
         // The desired position is outside of the current chunk.
         // Access the new position.  Assume a forward iteration from here,


@@ -115,8 +119,11 @@ utext_setNativeIndex(UText *ut, int64_t index) {
         // utf-16 indexing.
         ut->chunkOffset=(int32_t)(index-ut->chunkNativeStart);
     } else {
         // utf-16 indexing.
         ut->chunkOffset=(int32_t)(index-ut->chunkNativeStart);
     } else {

-         ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index);
+        ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index);

     }
     }

+    // Apple note, it can still be valid to have ut->chunkContents==0 at this
+    // point (just not inside the next block), see <rdar://problem/53610517>
+

     // The convention is that the index must always be on a code point boundary.
     // Adjust the index position if it is in the middle of a surrogate pair.
     if (ut->chunkOffset<ut->chunkLength) {
     // The convention is that the index must always be on a code point boundary.
     // Adjust the index position if it is in the middle of a surrogate pair.
     if (ut->chunkOffset<ut->chunkLength) {


@@ -845,18 +852,13 @@ U_CDECL_END
 //------------------------------------------------------------------------------
 
 // Chunk size.
 //------------------------------------------------------------------------------
 
 // Chunk size.

-//     Must be less than 42  (256/6), because of byte mapping from UChar indexes to native indexes.
-//     Worst case there are six UTF-8 bytes per UChar.
-//         obsolete 6 byte form fd + 5 trails maps to fffd
-//         obsolete 5 byte form fc + 4 trails maps to fffd
-//         non-shortest 4 byte forms maps to fffd
-//         normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit
-//     mapToUChars array size must allow for the worst case, 6.
-//     This could be brought down to 4, by treating fd and fc as pure illegal,
-//     rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros.
+//     Must be less than 85 (256/3), because of byte mapping from UChar indexes to native indexes.
+//     Worst case is three native bytes to one UChar.  (Supplemenaries are 4 native bytes
+//     to two UChars.)
+//     The longest illegal byte sequence treated as a single error (and converted to U+FFFD)
+//     is a three-byte sequence (truncated four-byte sequence).

 //
 enum { UTF8_TEXT_CHUNK_SIZE=32 };
 //
 enum { UTF8_TEXT_CHUNK_SIZE=32 };

-enum { MAP_TO_UCHARS_SIZE=UTF8_TEXT_CHUNK_SIZE*6+6 };

 
 //
 // UTF8Buf  Two of these structs will be set up in the UText's extra allocated space.
 
 //
 // UTF8Buf  Two of these structs will be set up in the UText's extra allocated space.


@@ -894,8 +896,8 @@ struct UTF8Buf {
                                                      //  Requires two extra slots,
                                                      //    one for a supplementary starting in the last normal position,
                                                      //    and one for an entry for the buffer limit position.
                                                      //  Requires two extra slots,
                                                      //    one for a supplementary starting in the last normal position,
                                                      //    and one for an entry for the buffer limit position.

-    uint8_t   mapToUChars[MAP_TO_UCHARS_SIZE];       // Map native offset from bufNativeStart to
-                                                     //   corresponding offset in filled part of buf.
+    uint8_t   mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to
+                                                     //   correspoding offset in filled part of buf.

     int32_t   align;
 };
 
     int32_t   align;
 };
 


@@ -1037,7 +1039,7 @@ utf8TextAccess(UText *ut, int64_t index, UBool forward) {
             // Requested index is in this buffer.
             u8b = (UTF8Buf *)ut->p;   // the current buffer
             mapIndex = ix - u8b->toUCharsMapStart;
             // Requested index is in this buffer.
             u8b = (UTF8Buf *)ut->p;   // the current buffer
             mapIndex = ix - u8b->toUCharsMapStart;

-            U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE);
+            U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars));

             ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
             return TRUE;
 
             ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
             return TRUE;
 


@@ -1199,9 +1201,9 @@ fillForward:
         // Swap the UText buffers.
         //  We want to fill what was previously the alternate buffer,
         //  and make what was the current buffer be the new alternate.
         // Swap the UText buffers.
         //  We want to fill what was previously the alternate buffer,
         //  and make what was the current buffer be the new alternate.

-        UTF8Buf *u8b = (UTF8Buf *)ut->q;
+        UTF8Buf *u8b_swap = (UTF8Buf *)ut->q;

         ut->q = ut->p;
         ut->q = ut->p;

-        ut->p = u8b;
+        ut->p = u8b_swap;

 
         int32_t strLen = ut->b;
         UBool   nulTerminated = FALSE;
 
         int32_t strLen = ut->b;
         UBool   nulTerminated = FALSE;


@@ -1210,9 +1212,9 @@ fillForward:
             nulTerminated = TRUE;
         }
 
             nulTerminated = TRUE;
         }
 

-        UChar   *buf = u8b->buf;
-        uint8_t *mapToNative  = u8b->mapToNative;
-        uint8_t *mapToUChars  = u8b->mapToUChars;
+        UChar   *buf = u8b_swap->buf;
+        uint8_t *mapToNative  = u8b_swap->mapToNative;
+        uint8_t *mapToUChars  = u8b_swap->mapToUChars;

         int32_t  destIx       = 0;
         int32_t  srcIx        = ix;
         UBool    seenNonAscii = FALSE;
         int32_t  destIx       = 0;
         int32_t  srcIx        = ix;
         UBool    seenNonAscii = FALSE;


@@ -1233,7 +1235,7 @@ fillForward:
                 // General case, handle everything.
                 if (seenNonAscii == FALSE) {
                     seenNonAscii = TRUE;
                 // General case, handle everything.
                 if (seenNonAscii == FALSE) {
                     seenNonAscii = TRUE;

-                    u8b->bufNILimit = destIx;
+                    u8b_swap->bufNILimit = destIx;

                 }
 
                 int32_t  cIx      = srcIx;
                 }
 
                 int32_t  cIx      = srcIx;


@@ -1266,22 +1268,22 @@ fillForward:
         mapToUChars[srcIx - ix] = (uint8_t)destIx;
 
         //  fill in Buffer descriptor
         mapToUChars[srcIx - ix] = (uint8_t)destIx;
 
         //  fill in Buffer descriptor

-        u8b->bufNativeStart     = ix;
-        u8b->bufNativeLimit     = srcIx;
-        u8b->bufStartIdx        = 0;
-        u8b->bufLimitIdx        = destIx;
+        u8b_swap->bufNativeStart     = ix;
+        u8b_swap->bufNativeLimit     = srcIx;
+        u8b_swap->bufStartIdx        = 0;
+        u8b_swap->bufLimitIdx        = destIx;

         if (seenNonAscii == FALSE) {
         if (seenNonAscii == FALSE) {

-            u8b->bufNILimit     = destIx;
+            u8b_swap->bufNILimit     = destIx;

         }
         }

-        u8b->toUCharsMapStart   = u8b->bufNativeStart;
+        u8b_swap->toUCharsMapStart   = u8b_swap->bufNativeStart;

 
         // Set UText chunk to refer to this buffer.
         ut->chunkContents       = buf;
         ut->chunkOffset         = 0;
 
         // Set UText chunk to refer to this buffer.
         ut->chunkContents       = buf;
         ut->chunkOffset         = 0;

-        ut->chunkLength         = u8b->bufLimitIdx;
-        ut->chunkNativeStart    = u8b->bufNativeStart;
-        ut->chunkNativeLimit    = u8b->bufNativeLimit;
-        ut->nativeIndexingLimit = u8b->bufNILimit;
+        ut->chunkLength         = u8b_swap->bufLimitIdx;
+        ut->chunkNativeStart    = u8b_swap->bufNativeStart;
+        ut->chunkNativeLimit    = u8b_swap->bufNativeLimit;
+        ut->nativeIndexingLimit = u8b_swap->bufNILimit;

 
         // For zero terminated strings, keep track of the maximum point
         //   scanned so far.
 
         // For zero terminated strings, keep track of the maximum point
         //   scanned so far.


@@ -1314,14 +1316,14 @@ fillReverse:
         // Swap the UText buffers.
         //  We want to fill what was previously the alternate buffer,
         //  and make what was the current buffer be the new alternate.
         // Swap the UText buffers.
         //  We want to fill what was previously the alternate buffer,
         //  and make what was the current buffer be the new alternate.

-        UTF8Buf *u8b = (UTF8Buf *)ut->q;
+        UTF8Buf *u8b_swap = (UTF8Buf *)ut->q;

         ut->q = ut->p;
         ut->q = ut->p;

-        ut->p = u8b;
+        ut->p = u8b_swap;

 
 

-        UChar   *buf = u8b->buf;
-        uint8_t *mapToNative = u8b->mapToNative;
-        uint8_t *mapToUChars = u8b->mapToUChars;
-        int32_t  toUCharsMapStart = ix - MAP_TO_UCHARS_SIZE + 1;
+        UChar   *buf = u8b_swap->buf;
+        uint8_t *mapToNative = u8b_swap->mapToNative;
+        uint8_t *mapToUChars = u8b_swap->mapToUChars;
+        int32_t  toUCharsMapStart = ix - sizeof(UTF8Buf::mapToUChars) + 1;

         // Note that toUCharsMapStart can be negative. Happens when the remaining
         // text from current position to the beginning is less than the buffer size.
         // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry.
         // Note that toUCharsMapStart can be negative. Happens when the remaining
         // text from current position to the beginning is less than the buffer size.
         // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry.


@@ -1390,19 +1392,19 @@ fillReverse:
                 bufNILimit = destIx;
             }
         }
                 bufNILimit = destIx;
             }
         }

-        u8b->bufNativeStart     = srcIx;
-        u8b->bufNativeLimit     = ix;
-        u8b->bufStartIdx        = destIx;
-        u8b->bufLimitIdx        = UTF8_TEXT_CHUNK_SIZE+2;
-        u8b->bufNILimit         = bufNILimit - u8b->bufStartIdx;
-        u8b->toUCharsMapStart   = toUCharsMapStart;
-
-        ut->chunkContents       = &buf[u8b->bufStartIdx];
-        ut->chunkLength         = u8b->bufLimitIdx - u8b->bufStartIdx;
+        u8b_swap->bufNativeStart     = srcIx;
+        u8b_swap->bufNativeLimit     = ix;
+        u8b_swap->bufStartIdx        = destIx;
+        u8b_swap->bufLimitIdx        = UTF8_TEXT_CHUNK_SIZE+2;
+        u8b_swap->bufNILimit         = bufNILimit - u8b_swap->bufStartIdx;
+        u8b_swap->toUCharsMapStart   = toUCharsMapStart;
+
+        ut->chunkContents       = &buf[u8b_swap->bufStartIdx];
+        ut->chunkLength         = u8b_swap->bufLimitIdx - u8b_swap->bufStartIdx;

         ut->chunkOffset         = ut->chunkLength;
         ut->chunkOffset         = ut->chunkLength;

-        ut->chunkNativeStart    = u8b->bufNativeStart;
-        ut->chunkNativeLimit    = u8b->bufNativeLimit;
-        ut->nativeIndexingLimit = u8b->bufNILimit;
+        ut->chunkNativeStart    = u8b_swap->bufNativeStart;
+        ut->chunkNativeLimit    = u8b_swap->bufNativeLimit;
+        ut->nativeIndexingLimit = u8b_swap->bufNILimit;

         return TRUE;
     }
 
         return TRUE;
     }
 


@@ -1556,7 +1558,7 @@ utf8TextMapIndexToUTF16(const UText *ut, int64_t index64) {
     U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
     U_ASSERT(index<=ut->chunkNativeLimit);
     int32_t mapIndex = index - u8b->toUCharsMapStart;
     U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
     U_ASSERT(index<=ut->chunkNativeLimit);
     int32_t mapIndex = index - u8b->toUCharsMapStart;

-    U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE);
+    U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars));

     int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
     U_ASSERT(offset>=0 && offset<=ut->chunkLength);
     return offset;
     int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
     U_ASSERT(offset>=0 && offset<=ut->chunkLength);
     return offset;


@@ -2241,13 +2243,13 @@ unistrTextCopy(UText *ut,
     }
 
     if(move) {
     }
 
     if(move) {

-        // move: copy to destIndex, then replace original with nothing
+        // move: copy to destIndex, then remove original

         int32_t segLength=limit32-start32;
         us->copy(start32, limit32, destIndex32);
         if(destIndex32<start32) {
             start32+=segLength;
         }
         int32_t segLength=limit32-start32;
         us->copy(start32, limit32, destIndex32);
         if(destIndex32<start32) {
             start32+=segLength;
         }

-        us->replace(start32, segLength, NULL, 0);
+        us->remove(start32, segLength);

     } else {
         // copy
         us->copy(start32, limit32, destIndex32);
     } else {
         // copy
         us->copy(start32, limit32, destIndex32);