ICU-57166.0.1.tar.gz

diff --git a/icuSources/common/utext.cpp b/icuSources/common/utext.cpp
index 20952a8459273a82a48e3a19a1c67e7dd3cf6ad4..de279eba501f212057e3a1681ec37e620f5e1f54 100644 (file)
--- a/icuSources/common/utext.cpp
+++ b/icuSources/common/utext.cpp
@@ -845,11 +845,18 @@ U_CDECL_END
 //------------------------------------------------------------------------------
 
 // Chunk size.
 //------------------------------------------------------------------------------
 
 // Chunk size.

-//     Must be less than 85, because of byte mapping from UChar indexes to native indexes.
-//     Worst case is three native bytes to one UChar.  (Supplemenaries are 4 native bytes
-//     to two UChars.)
+//     Must be less than 42  (256/6), because of byte mapping from UChar indexes to native indexes.
+//     Worst case there are six UTF-8 bytes per UChar.
+//         obsolete 6 byte form fd + 5 trails maps to fffd
+//         obsolete 5 byte form fc + 4 trails maps to fffd
+//         non-shortest 4 byte forms maps to fffd
+//         normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit
+//     mapToUChars array size must allow for the worst case, 6.
+//     This could be brought down to 4, by treating fd and fc as pure illegal,
+//     rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros.

 //
 enum { UTF8_TEXT_CHUNK_SIZE=32 };
 //
 enum { UTF8_TEXT_CHUNK_SIZE=32 };

+enum { MAP_TO_UCHARS_SIZE=UTF8_TEXT_CHUNK_SIZE*6+6 };

 
 //
 // UTF8Buf  Two of these structs will be set up in the UText's extra allocated space.
 
 //
 // UTF8Buf  Two of these structs will be set up in the UText's extra allocated space.


@@ -887,8 +894,8 @@ struct UTF8Buf {
                                                      //  Requires two extra slots,
                                                      //    one for a supplementary starting in the last normal position,
                                                      //    and one for an entry for the buffer limit position.
                                                      //  Requires two extra slots,
                                                      //    one for a supplementary starting in the last normal position,
                                                      //    and one for an entry for the buffer limit position.

-    uint8_t   mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to
-                                                     //   correspoding offset in filled part of buf.
+    uint8_t   mapToUChars[MAP_TO_UCHARS_SIZE];       // Map native offset from bufNativeStart to
+                                                     //   corresponding offset in filled part of buf.

     int32_t   align;
 };
 
     int32_t   align;
 };
 


@@ -1030,6 +1037,7 @@ utf8TextAccess(UText *ut, int64_t index, UBool forward) {
             // Requested index is in this buffer.
             u8b = (UTF8Buf *)ut->p;   // the current buffer
             mapIndex = ix - u8b->toUCharsMapStart;
             // Requested index is in this buffer.
             u8b = (UTF8Buf *)ut->p;   // the current buffer
             mapIndex = ix - u8b->toUCharsMapStart;

+            U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE);

             ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
             return TRUE;
 
             ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
             return TRUE;
 


@@ -1296,6 +1304,10 @@ fillReverse:
         // Can only do this if the incoming index is somewhere in the interior of the string.
         //   If index is at the end, there is no character there to look at.
         if (ix != ut->b) {
         // Can only do this if the incoming index is somewhere in the interior of the string.
         //   If index is at the end, there is no character there to look at.
         if (ix != ut->b) {

+            // Note: this function will only move the index back if it is on a trail byte
+            //       and there is a preceding lead byte and the sequence from the lead 
+            //       through this trail could be part of a valid UTF-8 sequence
+            //       Otherwise the index remains unchanged.

             U8_SET_CP_START(s8, 0, ix);
         }
 
             U8_SET_CP_START(s8, 0, ix);
         }
 


@@ -1309,7 +1321,10 @@ fillReverse:
         UChar   *buf = u8b->buf;
         uint8_t *mapToNative = u8b->mapToNative;
         uint8_t *mapToUChars = u8b->mapToUChars;
         UChar   *buf = u8b->buf;
         uint8_t *mapToNative = u8b->mapToNative;
         uint8_t *mapToUChars = u8b->mapToUChars;

-        int32_t  toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1);
+        int32_t  toUCharsMapStart = ix - MAP_TO_UCHARS_SIZE + 1;
+        // Note that toUCharsMapStart can be negative. Happens when the remaining
+        // text from current position to the beginning is less than the buffer size.
+        // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry.

         int32_t  destIx = UTF8_TEXT_CHUNK_SIZE+2;   // Start in the overflow region
                                                     //   at end of buffer to leave room
                                                     //   for a surrogate pair at the
         int32_t  destIx = UTF8_TEXT_CHUNK_SIZE+2;   // Start in the overflow region
                                                     //   at end of buffer to leave room
                                                     //   for a surrogate pair at the


@@ -1336,6 +1351,7 @@ fillReverse:
             if (c<0x80) {
                 // Special case ASCII range for speed.
                 buf[destIx] = (UChar)c;
             if (c<0x80) {
                 // Special case ASCII range for speed.
                 buf[destIx] = (UChar)c;

+                U_ASSERT(toUCharsMapStart <= srcIx);

                 mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx;
                 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
             } else {
                 mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx;
                 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
             } else {


@@ -1365,6 +1381,7 @@ fillReverse:
                 do {
                     mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx;
                 } while (sIx >= srcIx);
                 do {
                     mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx;
                 } while (sIx >= srcIx);

+                U_ASSERT(toUCharsMapStart <= (srcIx+1));

 
                 // Set native indexing limit to be the current position.
                 //   We are processing a non-ascii, non-native-indexing char now;
 
                 // Set native indexing limit to be the current position.
                 //   We are processing a non-ascii, non-native-indexing char now;


@@ -1539,6 +1556,7 @@ utf8TextMapIndexToUTF16(const UText *ut, int64_t index64) {
     U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
     U_ASSERT(index<=ut->chunkNativeLimit);
     int32_t mapIndex = index - u8b->toUCharsMapStart;
     U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
     U_ASSERT(index<=ut->chunkNativeLimit);
     int32_t mapIndex = index - u8b->toUCharsMapStart;

+    U_ASSERT(mapIndex < MAP_TO_UCHARS_SIZE);

     int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
     U_ASSERT(offset>=0 && offset<=ut->chunkLength);
     return offset;
     int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
     U_ASSERT(offset>=0 && offset<=ut->chunkLength);
     return offset;

diff --git a/icuSources/test/intltest/utxttest.cpp b/icuSources/test/intltest/utxttest.cpp
index 474b7b971609a363ce29190a738c2e0d48d8be7c..2fce43ed916cd1dfce77015b1f34dae594b838a5 100644 (file)
--- a/icuSources/test/intltest/utxttest.cpp
+++ b/icuSources/test/intltest/utxttest.cpp
@@ -65,6 +65,8 @@ UTextTest::runIndexedTest(int32_t index, UBool exec,
             if (exec) Ticket10983();  break;
         case 7: name = "Ticket12130";
             if (exec) Ticket12130(); break;
             if (exec) Ticket10983();  break;
         case 7: name = "Ticket12130";
             if (exec) Ticket12130(); break;

+        case 8: name = "Ticket12888";
+            if (exec) Ticket12888(); break;

         default: name = "";          break;
     }
 }
         default: name = "";          break;
     }
 }


@@ -1581,3 +1583,63 @@ void UTextTest::Ticket12130() {
     }
     utext_close(&ut);
 }
     }
     utext_close(&ut);
 }

+
+// Ticket 12888: bad handling of illegal utf-8 containing many instances of the archaic, now illegal,
+//               six byte utf-8 forms. Original implementation had an assumption that
+//               there would be at most three utf-8 bytes per UTF-16 code unit.
+//               The five and six byte sequences map to a single replacement character.
+
+void UTextTest::Ticket12888() {
+    const char *badString = 
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"
+            "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80";
+
+    UErrorCode status = U_ZERO_ERROR;
+    LocalUTextPointer ut(utext_openUTF8(NULL, badString, -1, &status));
+    TEST_SUCCESS(status);
+    for (;;) {
+        UChar32 c = utext_next32(ut.getAlias());
+        if (c == U_SENTINEL) {
+            break;
+        }
+    }
+    int32_t endIdx = utext_getNativeIndex(ut.getAlias());
+    if (endIdx != (int32_t)strlen(badString)) {
+        errln("%s:%d expected=%d, actual=%d", __FILE__, __LINE__, strlen(badString), endIdx);
+        return;
+    }
+
+    for (int32_t prevIndex = endIdx; prevIndex>0;) {
+        UChar32 c = utext_previous32(ut.getAlias());
+        int32_t currentIndex = utext_getNativeIndex(ut.getAlias());
+        if (c != 0xfffd) {
+            errln("%s:%d (expected, actual, index) = (%d, %d, %d)\n",
+                    __FILE__, __LINE__, 0xfffd, c, currentIndex);
+            break;
+        }
+        if (currentIndex != prevIndex - 6) {
+            errln("%s:%d: wrong index. Expected, actual = %d, %d",
+                    __FILE__, __LINE__, prevIndex - 6, currentIndex);
+            break;
+        }
+        prevIndex = currentIndex;
+    }
+}

diff --git a/icuSources/test/intltest/utxttest.h b/icuSources/test/intltest/utxttest.h
index 346563d5a4891697291ca6f02a2d0e1256a6204b..bcb516f4abaa286a3b206dd3f29da3604e215839 100644 (file)
--- a/icuSources/test/intltest/utxttest.h
+++ b/icuSources/test/intltest/utxttest.h
@@ -36,6 +36,7 @@ public:
     void Ticket10562();
     void Ticket10983();
     void Ticket12130();
     void Ticket10562();
     void Ticket10983();
     void Ticket12130();

+    void Ticket12888();

 
 private:
     struct m {                              // Map between native indices & code points.
 
 private:
     struct m {                              // Map between native indices & code points.