xnu-792.17.14.tar.gz

[apple/xnu.git] / bsd / vfs / vfs_utfconv.c

diff --git a/bsd/vfs/vfs_utfconv.c b/bsd/vfs/vfs_utfconv.c
index 44f7263552095e226f1c6264f0b972987abbd9a9..6651c4f38108b8d7dc4ae7bba8ac3648fc5d7a9c 100644 (file)
--- a/bsd/vfs/vfs_utfconv.c
+++ b/bsd/vfs/vfs_utfconv.c
@@ -1,23 +1,33 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+ /*
+       Includes Unicode 3.2 decomposition code derived from Core Foundation

  */
 
 #include <sys/param.h>
  */
 
 #include <sys/param.h>


@@ -25,48 +35,139 @@
 #include <sys/errno.h>
 #include <architecture/byte_order.h>
 
 #include <sys/errno.h>
 #include <architecture/byte_order.h>
 

-

 /*
 /*

- * UTF-8 (UCS Transformation Format)
+ * UTF-8 (Unicode Transformation Format)

  *
  *

- * The following subset of UTF-8 is used to encode UCS-2 filenames. It
- * requires a maximum of three 3 bytes per UCS-2 character.  Only the
- * shortest encoding required to represent the significant UCS-2 bits
- * is legal.
+ * UTF-8 is the Unicode Transformation Format that serializes a Unicode
+ * character as a sequence of one to four bytes. Only the shortest form
+ * required to represent the significant Unicode bits is legal.

  * 
  * UTF-8 Multibyte Codes
  *
  * 
  * UTF-8 Multibyte Codes
  *

- * Bytes   Bits   UCS-2 Min   UCS-2 Max   UTF-8 Byte Sequence (binary)
- * -------------------------------------------------------------------
- *   1       7     0x0000      0x007F      0xxxxxxx
- *   2      11     0x0080      0x07FF      110xxxxx 10xxxxxx
- *   3      16     0x0800      0xFFFF      1110xxxx 10xxxxxx 10xxxxxx
- * -------------------------------------------------------------------
+ * Bytes   Bits   Unicode Min  Unicode Max   UTF-8 Byte Sequence (binary)
+ * -----------------------------------------------------------------------------
+ *   1       7       0x0000        0x007F    0xxxxxxx
+ *   2      11       0x0080        0x07FF    110xxxxx 10xxxxxx
+ *   3      16       0x0800        0xFFFF    1110xxxx 10xxxxxx 10xxxxxx
+ *   4      21      0x10000      0x10FFFF    11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ * -----------------------------------------------------------------------------

  */
 
 
  */
 
 

-#define UCS_TO_UTF_LEN(c)      ((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : 3))
+#define UNICODE_TO_UTF8_LEN(c)  \
+       ((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : (((c) & 0xf800) == 0xd800 ? 2 : 3)))
+
+#define UCS_ALT_NULL   0x2400
+
+/* Surrogate Pair Constants */
+#define SP_HALF_SHIFT  10
+#define SP_HALF_BASE   0x0010000UL
+#define SP_HALF_MASK   0x3FFUL

 
 

+#define SP_HIGH_FIRST  0xD800UL
+#define SP_HIGH_LAST   0xDBFFUL
+#define SP_LOW_FIRST   0xDC00UL
+#define SP_LOW_LAST    0xDFFFUL

 
 

-static u_int16_t ucs_decompose __P((u_int16_t, u_int16_t *));
+
+#include "vfs_utfconvdata.h"

 
 
 /*
 
 
 /*

- * utf8_encodelen - Calculates the UTF-8 encoding length for a UCS-2 filename
+ * Test for a combining character.
+ *
+ * Similar to __CFUniCharIsNonBaseCharacter except that
+ * unicode_combinable also includes Hangul Jamo characters.
+ */
+static inline int
+unicode_combinable(u_int16_t character)
+{
+       const u_int8_t *bitmap = __CFUniCharCombiningBitmap;
+       u_int8_t value;
+
+       if (character < 0x0300)
+               return (0);
+
+       value = bitmap[(character >> 8) & 0xFF];
+
+       if (value == 0xFF) {
+               return (1);
+       } else if (value) {
+               bitmap = bitmap + ((value - 1) * 32) + 256;
+               return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0);
+       }
+       return (0);
+}
+
+/*
+ * Test for a precomposed character.
+ *
+ * Similar to __CFUniCharIsDecomposableCharacter.
+ */
+static inline int
+unicode_decomposeable(u_int16_t character) {
+       const u_int8_t *bitmap = __CFUniCharDecomposableBitmap;
+       u_int8_t value;
+       
+       if (character < 0x00C0)
+               return (0);
+
+       value = bitmap[(character >> 8) & 0xFF];
+
+       if (value == 0xFF) {
+               return (1);
+       } else if (value) {
+               bitmap = bitmap + ((value - 1) * 32) + 256;
+               return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0);
+       }
+       return (0);
+}
+
+
+/*
+ * Get the combing class.
+ *
+ * Similar to CFUniCharGetCombiningPropertyForCharacter.
+ */
+static inline u_int8_t
+get_combining_class(u_int16_t character) {
+       const u_int8_t *bitmap = __CFUniCharCombiningPropertyBitmap;
+
+       u_int8_t value = bitmap[(character >> 8)];
+
+       if (value) {
+               bitmap = bitmap + (value * 256);
+               return bitmap[character % 256];
+       }
+       return (0);
+}
+
+
+static int unicode_decompose(u_int16_t character, u_int16_t *convertedChars);
+
+static u_int16_t unicode_combine(u_int16_t base, u_int16_t combining);
+
+static void priortysort(u_int16_t* characters, int count);
+
+char utf_extrabytes[32] = {
+        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+       -1, -1, -1, -1, -1, -1, -1, -1,  1,  1,  1,  1,  2,  2,  3, -1
+};
+
+
+/*
+ * utf8_encodelen - Calculates the UTF-8 encoding length for a Unicode filename

  *
  * NOTES:
  *    If '/' chars are allowed on disk then an alternate
  *    (replacement) char must be provided in altslash.
  *
  * input flags:
  *
  * NOTES:
  *    If '/' chars are allowed on disk then an alternate
  *    (replacement) char must be provided in altslash.
  *
  * input flags:

- *    UTF_REVERSE_ENDIAN: UCS-2 byteorder is opposite current runtime
+ *    UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime

  */
 size_t
  */
 size_t

-utf8_encodelen(ucsp, ucslen, altslash, flags)
-       const u_int16_t * ucsp;
-       size_t ucslen;
-       u_int16_t altslash;
-       int flags;
+utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
+               int flags)

 {
        u_int16_t ucs_ch;
        int charcnt;
 {
        u_int16_t ucs_ch;
        int charcnt;


@@ -81,12 +182,12 @@ utf8_encodelen(ucsp, ucslen, altslash, flags)
 
                if (swapbytes)
                        ucs_ch = NXSwapShort(ucs_ch);
 
                if (swapbytes)
                        ucs_ch = NXSwapShort(ucs_ch);

-               if (altslash && ucs_ch == '/')
-                       ucs_ch = altslash;
-               if (ucs_ch == '\0')
-                       ucs_ch = 0xc080;
+               if (ucs_ch == '/')
+                       ucs_ch = altslash ? altslash : '_';
+               else if (ucs_ch == '\0')
+                       ucs_ch = UCS_ALT_NULL;

                
                

-               len += UCS_TO_UTF_LEN(ucs_ch);
+               len += UNICODE_TO_UTF8_LEN(ucs_ch);

        }
 
        return (len);
        }
 
        return (len);


@@ -94,33 +195,36 @@ utf8_encodelen(ucsp, ucslen, altslash, flags)
 
 
 /*
 
 
 /*

- * utf8_encodestr - Encodes a UCS-2 (Unicode) string to UTF-8
+ * utf8_encodestr - Encodes a Unicode string to UTF-8

  *
  * NOTES:
  *
  * NOTES:

- *    The resulting UTF-8 string is not null terminated.
+ *    The resulting UTF-8 string is NULL terminated.

  *
  *    If '/' chars are allowed on disk then an alternate
  *    (replacement) char must be provided in altslash.
  *
  * input flags:
  *
  *    If '/' chars are allowed on disk then an alternate
  *    (replacement) char must be provided in altslash.
  *
  * input flags:

- *    UTF_REVERSE_ENDIAN: UCS-2 byteorder is opposite current runtime
+ *    UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime

  *    UTF_NO_NULL_TERM:  don't add NULL termination to UTF-8 output
  *    UTF_NO_NULL_TERM:  don't add NULL termination to UTF-8 output

+ *
+ * result:
+ *    ENAMETOOLONG: Name didn't fit; only buflen bytes were encoded
+ *    EINVAL: Illegal char found; char was replaced by an '_'.

  */
  */

-int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags)
-       const u_int16_t * ucsp;
-       size_t ucslen;
-       u_int8_t * utf8p;
-       size_t * utf8len;
-       size_t buflen;
-       u_int16_t altslash;
-       int flags;
+int
+utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
+               size_t * utf8len, size_t buflen, u_int16_t altslash, int flags)

 {
        u_int8_t * bufstart;
        u_int8_t * bufend;
        u_int16_t ucs_ch;
 {
        u_int8_t * bufstart;
        u_int8_t * bufend;
        u_int16_t ucs_ch;

+       u_int16_t * chp = NULL;
+       u_int16_t sequence[8];
+       int extra = 0;

        int charcnt;
        int swapbytes = (flags & UTF_REVERSE_ENDIAN);
        int charcnt;
        int swapbytes = (flags & UTF_REVERSE_ENDIAN);

-       int nullterm = ((flags & UTF_NO_NULL_TERM) == 0);
+       int nullterm  = ((flags & UTF_NO_NULL_TERM) == 0);
+       int decompose = (flags & UTF_DECOMPOSED);

        int result = 0;
        
        bufstart = utf8p;
        int result = 0;
        
        bufstart = utf8p;


@@ -130,18 +234,37 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags)
        charcnt = ucslen / 2;
 
        while (charcnt-- > 0) {
        charcnt = ucslen / 2;
 
        while (charcnt-- > 0) {

-               ucs_ch = *ucsp++;
+               if (extra > 0) {
+                       --extra;
+                       ucs_ch = *chp++;
+               } else {
+                       ucs_ch = swapbytes ? NXSwapShort(*ucsp++) : *ucsp++;

 
 

-               if (swapbytes)
-                       ucs_ch = NXSwapShort(ucs_ch);
-               if (altslash && ucs_ch == '/')
-                       ucs_ch = altslash;
+                       if (decompose && unicode_decomposeable(ucs_ch)) {
+                               extra = unicode_decompose(ucs_ch, sequence) - 1;
+                               charcnt += extra;
+                               ucs_ch = sequence[0];
+                               chp = &sequence[1];
+                       }
+               }

 
 

-               if ((ucs_ch < 0x0080) && (ucs_ch != '\0')) {
+               /* Slash and NULL are not permitted */
+               if (ucs_ch == '/') {
+                       if (altslash)
+                               ucs_ch = altslash;
+                       else {
+                               ucs_ch = '_';
+                               result = EINVAL;
+                       }
+               } else if (ucs_ch == '\0') {
+                       ucs_ch = UCS_ALT_NULL;
+               }
+
+               if (ucs_ch < 0x0080) {

                        if (utf8p >= bufend) {
                                result = ENAMETOOLONG;
                                break;
                        if (utf8p >= bufend) {
                                result = ENAMETOOLONG;
                                break;

-                       }
+                       }                       

                        *utf8p++ = ucs_ch;
 
                } else if (ucs_ch < 0x800) {
                        *utf8p++ = ucs_ch;
 
                } else if (ucs_ch < 0x800) {


@@ -149,18 +272,40 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags)
                                result = ENAMETOOLONG;
                                break;
                        }
                                result = ENAMETOOLONG;
                                break;
                        }

-                       /* NOTE: NULL maps to 0xC080 */
-                       *utf8p++ = (ucs_ch >> 6) | 0xc0;
-                       *utf8p++ = (ucs_ch & 0x3f) | 0x80;
+                       *utf8p++ = 0xc0 | (ucs_ch >> 6);
+                       *utf8p++ = 0x80 | (0x3f & ucs_ch);

 
                } else {
 
                } else {

+                       /* Combine valid surrogate pairs */
+                       if (ucs_ch >= SP_HIGH_FIRST && ucs_ch <= SP_HIGH_LAST
+                               && charcnt > 0) {
+                               u_int16_t ch2;
+                               u_int32_t pair;
+
+                               ch2 = swapbytes ? NXSwapShort(*ucsp) : *ucsp;
+                               if (ch2 >= SP_LOW_FIRST && ch2 <= SP_LOW_LAST) {
+                                       pair = ((ucs_ch - SP_HIGH_FIRST) << SP_HALF_SHIFT)
+                                               + (ch2 - SP_LOW_FIRST) + SP_HALF_BASE;
+                                       if ((utf8p + 3) >= bufend) {
+                                               result = ENAMETOOLONG;
+                                               break;
+                                       }
+                                       --charcnt;
+                                       ++ucsp;                         
+                                       *utf8p++ = 0xf0 | (pair >> 18);
+                                       *utf8p++ = 0x80 | (0x3f & (pair >> 12));
+                                       *utf8p++ = 0x80 | (0x3f & (pair >> 6));
+                                       *utf8p++ = 0x80 | (0x3f & pair);
+                                       continue;
+                               }
+                       }

                        if ((utf8p + 2) >= bufend) {
                                result = ENAMETOOLONG;
                                break;
                        }
                        if ((utf8p + 2) >= bufend) {
                                result = ENAMETOOLONG;
                                break;
                        }

-                       *utf8p++ = (ucs_ch >> 12) | 0xe0;
-                       *utf8p++ = ((ucs_ch >> 6) & 0x3f) | 0x80;
-                       *utf8p++ = ((ucs_ch) & 0x3f) | 0x80;
+                       *utf8p++ = 0xe0 | (ucs_ch >> 12);
+                       *utf8p++ = 0x80 | (0x3f & (ucs_ch >> 6));
+                       *utf8p++ = 0x80 | (0x3f & ucs_ch);

                }       
        }
        
                }       
        }
        


@@ -173,7 +318,7 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags)
 
 
 /*
 
 
 /*

- * utf8_decodestr - Decodes a UTF-8 string back to UCS-2 (Unicode)
+ * utf8_decodestr - Decodes a UTF-8 string back to Unicode

  *
  * NOTES:
  *    The input UTF-8 string does not need to be null terminated
  *
  * NOTES:
  *    The input UTF-8 string does not need to be null terminated


@@ -183,234 +328,486 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags)
  *    (replacement) char must be provided in altslash.
  *
  * input flags:
  *    (replacement) char must be provided in altslash.
  *
  * input flags:

- *    UTF_REV_ENDIAN:   UCS-2 byteorder is oposite current runtime
- *    UTF_DECOMPOSED:   UCS-2 output string must be fully decompsed
+ *    UTF_REV_ENDIAN:   Unicode byteorder is oposite current runtime
+ *    UTF_DECOMPOSED:   Unicode output string must be fully decompsed
+ *
+ * result:
+ *    ENAMETOOLONG: Name didn't fit; only ucslen chars were decoded.
+ *    EINVAL: Illegal UTF-8 sequence found.

  */
 int
  */
 int

-utf8_decodestr(utf8p, utf8len, ucsp, ucslen, buflen, altslash, flags)
-       const u_int8_t* utf8p;
-       size_t utf8len;
-       u_int16_t* ucsp;
-       size_t *ucslen;
-       size_t buflen;
-       u_int16_t altslash;
-       int flags;
+utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
+               size_t *ucslen, size_t buflen, u_int16_t altslash, int flags)

 {
        u_int16_t* bufstart;
        u_int16_t* bufend;
 {
        u_int16_t* bufstart;
        u_int16_t* bufend;

-       u_int16_t ucs_ch;
-       u_int8_t byte;
+       unsigned int ucs_ch;
+       unsigned int byte;
+       int combcharcnt = 0;

        int result = 0;
        int result = 0;

-       int decompose, swapbytes;
+       int decompose, precompose, swapbytes;

 
 

-       decompose = (flags & UTF_DECOMPOSED);
-       swapbytes = (flags & UTF_REVERSE_ENDIAN);
+       decompose =  (flags & UTF_DECOMPOSED);
+       precompose = (flags & UTF_PRECOMPOSED);
+       swapbytes =  (flags & UTF_REVERSE_ENDIAN);

 
        bufstart = ucsp;
        bufend = (u_int16_t *)((u_int8_t *)ucsp + buflen);
 
        while (utf8len-- > 0 && (byte = *utf8p++) != '\0') {
 
        bufstart = ucsp;
        bufend = (u_int16_t *)((u_int8_t *)ucsp + buflen);
 
        while (utf8len-- > 0 && (byte = *utf8p++) != '\0') {

-               if (ucsp >= bufend) {
-                       result = ENAMETOOLONG;
-                       goto stop;
-               }
+               if (ucsp >= bufend)
+                       goto toolong;

 
                /* check for ascii */
                if (byte < 0x80) {
 
                /* check for ascii */
                if (byte < 0x80) {

-                       ucs_ch = byte;
+                       ucs_ch = byte;                 /* 1st byte */

                } else {
                } else {

-                       switch (byte & 0xf0) {
-                       /*  2 byte sequence*/
-                       case 0xc0:
-                       case 0xd0:
-                               /* extract bits 6 - 10 from first byte */
-                               ucs_ch = (byte & 0x1F) << 6;  
-                               if ((ucs_ch < 0x0080) && (*utf8p != 0x80)) {
-                                       result = EINVAL;  /* seq not minimal */
-                                       goto stop;
-                               }
-                               break;
-                       /* 3 byte sequence*/
-                       case 0xe0:
-                               /* extract bits 12 - 15 from first byte */
-                               ucs_ch = (byte & 0x0F) << 6;
-
-                               /* extract bits 6 - 11 from second byte */
-                               if (((byte = *utf8p++) & 0xc0) != 0x80) {
-                                       result = EINVAL;
-                                       goto stop;
-                               }
-                               utf8len--;
-
-                               ucs_ch += (byte & 0x3F);
-                               ucs_ch <<= 6;
-
-                               if (ucs_ch < 0x0800) {
-                                       result = EINVAL; /* seq not minimal */
-                                       goto stop;
+                       u_int32_t ch;
+                       int extrabytes = utf_extrabytes[byte >> 3];
+
+                       if (utf8len < extrabytes)
+                               goto invalid;
+                       utf8len -= extrabytes;
+
+                       switch (extrabytes) {
+                       case 1:
+                               ch = byte; ch <<= 6;   /* 1st byte */
+                               byte = *utf8p++;       /* 2nd byte */
+                               if ((byte >> 6) != 2)
+                                       goto invalid;
+                               ch += byte;
+                               ch -= 0x00003080UL;
+                               if (ch < 0x0080)
+                                       goto invalid;
+                               ucs_ch = ch;
+                               break;
+                       case 2:
+                               ch = byte; ch <<= 6;   /* 1st byte */
+                               byte = *utf8p++;       /* 2nd byte */
+                               if ((byte >> 6) != 2)
+                                       goto invalid;
+                               ch += byte; ch <<= 6;
+                               byte = *utf8p++;       /* 3rd byte */
+                               if ((byte >> 6) != 2)
+                                       goto invalid;
+                               ch += byte;
+                               ch -= 0x000E2080UL;
+                               if (ch < 0x0800)
+                                       goto invalid;
+                               if (ch >= 0xD800) {
+                                       if (ch <= 0xDFFF)
+                                               goto invalid;
+                                       if (ch == 0xFFFE || ch == 0xFFFF)
+                                               goto invalid;

                                }
                                }

+                               ucs_ch = ch;

                                break;
                                break;

+                       case 3:
+                               ch = byte; ch <<= 6;   /* 1st byte */
+                               byte = *utf8p++;       /* 2nd byte */
+                               if ((byte >> 6) != 2)
+                                       goto invalid;
+                               ch += byte; ch <<= 6;
+                               byte = *utf8p++;       /* 3rd byte */
+                               if ((byte >> 6) != 2)
+                                       goto invalid;
+                               ch += byte; ch <<= 6;
+                               byte = *utf8p++;       /* 4th byte */
+                               if ((byte >> 6) != 2)
+                                       goto invalid;
+                               ch += byte;
+                               ch -= 0x03C82080UL + SP_HALF_BASE;
+                               ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST;
+                               if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST)
+                                       goto invalid;
+                               *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+                               if (ucsp >= bufend)
+                                       goto toolong;
+                               ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST;
+                               if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST)
+                                       goto invalid;
+                               *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+                               continue;

                        default:
                        default:

-                               result = EINVAL;
-                               goto stop;
+                               goto invalid;

                        }
                        }

-
-                       /* extract bits 0 - 5 from final byte */
-                       if (((byte = *utf8p++) & 0xc0) != 0x80) {
-                               result = EINVAL;
-                               goto stop;
-                       }
-                       utf8len--;
-                       ucs_ch += (byte & 0x3F);  
-

                        if (decompose) {
                        if (decompose) {

-                               u_int16_t comb_ch;
+                               if (unicode_decomposeable(ucs_ch)) {
+                                       u_int16_t sequence[8];
+                                       int count, i;

 
 

-                               ucs_ch = ucs_decompose(ucs_ch, &comb_ch);
+                                       count = unicode_decompose(ucs_ch, sequence);

 
 

-                               if (comb_ch) {
-                                       if (swapbytes)
-                                               *ucsp++ = NXSwapShort(ucs_ch);
-                                       else
-                                               *ucsp++ = ucs_ch;
-
-                                       if (ucsp >= bufend) {
-                                               result = ENAMETOOLONG;
-                                               goto stop;
+                                       for (i = 0; i < count; ++i) {
+                                               ucs_ch = sequence[i];
+                                               *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+                                               if (ucsp >= bufend)
+                                                       goto toolong;
+                                       }
+                                       combcharcnt += count - 1;
+                                       continue;                       
+                               }
+                       } else if (precompose && (ucsp != bufstart)) {
+                               u_int16_t composite, base;
+
+                               if (unicode_combinable(ucs_ch)) {
+                                       base = swapbytes ? NXSwapShort(*(ucsp - 1)) : *(ucsp - 1);
+                                       composite = unicode_combine(base, ucs_ch);
+                                       if (composite) {
+                                               --ucsp;
+                                               ucs_ch = composite;

                                        }
                                        }

-
-                                       ucs_ch = comb_ch;

                                }
                        }
                                }
                        }

+                       if (ucs_ch == UCS_ALT_NULL)
+                               ucs_ch = '\0';

                }
                }

-

                if (ucs_ch == altslash)
                        ucs_ch = '/';
                if (ucs_ch == altslash)
                        ucs_ch = '/';

-               if (swapbytes)
-                       ucs_ch = NXSwapShort(ucs_ch);

 
 

-               *ucsp++ = ucs_ch;
+               /*
+                * Make multiple combining character sequences canonical
+                */
+               if (unicode_combinable(ucs_ch)) {
+                       ++combcharcnt;   /* start tracking a run */
+               } else if (combcharcnt) {
+                       if (combcharcnt > 1) {
+                               priortysort(ucsp - combcharcnt, combcharcnt);
+                       }
+                       combcharcnt = 0;  /* start over */
+               }
+               *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+       }
+       /*
+        * Make a previous combining sequence canonical
+        */
+       if (combcharcnt > 1) {
+               priortysort(ucsp - combcharcnt, combcharcnt);

        }
        }

-stop:
+
+exit:

        *ucslen = (u_int8_t*)ucsp - (u_int8_t*)bufstart;
 
        return (result);
        *ucslen = (u_int8_t*)ucsp - (u_int8_t*)bufstart;
 
        return (result);

+
+invalid:
+       result = EINVAL;
+       goto exit;
+
+toolong:
+       result = ENAMETOOLONG;
+       goto exit;

 }
 
 
 /*
 }
 
 
 /*

- * Lookup tables for Unicode chars 0x00C0 thru 0x00FF
- * primary_char yields first decomposed char. If this
- * char is an alpha char then get the combining char
- * from the combining_char table and add 0x0300 to it.
+ * utf8_validatestr - Check for a valid UTF-8 string.

  */
  */

+int
+utf8_validatestr(const u_int8_t* utf8p, size_t utf8len)
+{
+       unsigned int byte;
+       u_int32_t ch;
+       unsigned int ucs_ch;
+       size_t extrabytes;

 
 

-static unsigned char primary_char[64] = {
-       0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0xC6, 0x43,
-
-       0x45, 0x45, 0x45, 0x45, 0x49, 0x49, 0x49, 0x49,
-
-       0xD0, 0x4E, 0x4F, 0x4F, 0x4F, 0x4F, 0x4F, 0xD7,
-
-       0xD8, 0x55, 0x55, 0x55, 0x55, 0x59, 0xDE, 0xDF,
-
-       0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0xE6, 0x63,
-
-       0x65, 0x65, 0x65, 0x65, 0x69, 0x69, 0x69, 0x69,
-
-       0xF0, 0x6E, 0x6F, 0x6F, 0x6F, 0x6F, 0x6F, 0xF7,
-
-       0xF8, 0x75, 0x75, 0x75, 0x75, 0x79, 0xFE, 0x79,
-};
-
-static unsigned char combining_char[64] = {
-       0x00, 0x01, 0x02, 0x03, 0x08, 0x0A, 0xFF, 0x27,
+       while (utf8len-- > 0 && (byte = *utf8p++) != '\0') {
+               if (byte < 0x80)
+                       continue;  /* plain ascii */
+
+               extrabytes = utf_extrabytes[byte >> 3];
+
+               if (utf8len < extrabytes)
+                       goto invalid;
+               utf8len -= extrabytes;
+
+               switch (extrabytes) {
+               case 1:
+                       ch = byte; ch <<= 6;   /* 1st byte */
+                       byte = *utf8p++;       /* 2nd byte */
+                       if ((byte >> 6) != 2)
+                               goto invalid;
+                       ch += byte;
+                       ch -= 0x00003080UL;
+                       if (ch < 0x0080)
+                               goto invalid;
+                       break;
+               case 2:
+                       ch = byte; ch <<= 6;   /* 1st byte */
+                       byte = *utf8p++;       /* 2nd byte */
+                       if ((byte >> 6) != 2)
+                               goto invalid;
+                       ch += byte; ch <<= 6;
+                       byte = *utf8p++;       /* 3rd byte */
+                       if ((byte >> 6) != 2)
+                               goto invalid;
+                       ch += byte;
+                       ch -= 0x000E2080UL;
+                       if (ch < 0x0800)
+                               goto invalid;
+                       if (ch >= 0xD800) {
+                               if (ch <= 0xDFFF)
+                                       goto invalid;
+                               if (ch == 0xFFFE || ch == 0xFFFF)
+                                       goto invalid;
+                       }
+                       break;
+               case 3:
+                       ch = byte; ch <<= 6;   /* 1st byte */
+                       byte = *utf8p++;       /* 2nd byte */
+                       if ((byte >> 6) != 2)
+                               goto invalid;
+                       ch += byte; ch <<= 6;
+                       byte = *utf8p++;       /* 3rd byte */
+                       if ((byte >> 6) != 2)
+                               goto invalid;
+                       ch += byte; ch <<= 6;
+                       byte = *utf8p++;       /* 4th byte */
+                       if ((byte >> 6) != 2)
+                               goto invalid;
+                       ch += byte;
+                       ch -= 0x03C82080UL + SP_HALF_BASE;
+                       ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST;
+                       if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST)
+                               goto invalid;
+                       ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST;
+                       if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST)
+                               goto invalid;
+                       break;
+               default:
+                       goto invalid;
+               }
+               
+       }
+       return (0);
+invalid:
+       return (EINVAL);
+}

 
 

-       0x00, 0x01, 0x02, 0x08, 0x00, 0x01, 0x02, 0x08,

 
 

-       0xFF, 0x03, 0x00, 0x01, 0x02, 0x03, 0x08, 0xFF,
+ /*
+  * Unicode 3.2 decomposition code (derived from Core Foundation)
+  */

 
 

-       0xFF, 0x00, 0x01, 0x02, 0x08, 0x01, 0xFF, 0xFF,
+typedef struct {
+       u_int32_t _key;
+       u_int32_t _value;
+} unicode_mappings32;

 
 

-       0x00, 0x01, 0x02, 0x03, 0x08, 0x0A, 0xFF, 0x27,
+static inline u_int32_t
+getmappedvalue32(const unicode_mappings32 *theTable, u_int32_t numElem,
+               u_int16_t character)
+{
+       const unicode_mappings32 *p, *q, *divider;
+
+       if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key))
+               return (0);
+
+       p = theTable;
+       q = p + (numElem-1);
+       while (p <= q) {
+               divider = p + ((q - p) >> 1);   /* divide by 2 */
+               if (character < divider->_key) { q = divider - 1; }
+               else if (character > divider->_key) { p = divider + 1; }
+               else { return (divider->_value); }
+       }
+       return (0);
+}

 
 

-       0x00, 0x01, 0x02, 0x08, 0x00, 0x01, 0x02, 0x08,
+#define RECURSIVE_DECOMPOSITION        (1 << 15)
+#define EXTRACT_COUNT(value)   (((value) >> 12) & 0x0007)

 
 

-       0xFF, 0x03, 0x00, 0x01, 0x02, 0x03, 0x08, 0xFF,
+typedef struct {
+       u_int16_t _key;
+       u_int16_t _value;
+} unicode_mappings16;

 
 

-       0xFF, 0x00, 0x01, 0x02, 0x08, 0x01, 0xFF, 0x08
-};
+static inline u_int16_t
+getmappedvalue16(const unicode_mappings16 *theTable, u_int32_t numElem,
+               u_int16_t character)
+{
+       const unicode_mappings16 *p, *q, *divider;
+
+       if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key))
+               return (0);
+
+       p = theTable;
+       q = p + (numElem-1);
+       while (p <= q) {
+               divider = p + ((q - p) >> 1);   /* divide by 2 */
+               if (character < divider->_key)
+                       q = divider - 1;
+               else if (character > divider->_key)
+                       p = divider + 1;
+               else
+                       return (divider->_value);
+       }
+       return (0);
+}

 
 
 
 

-/* CJK codepoints 0x3000 ~ 0x30FF */
-static const unsigned long __CJKDecompBitmap[] = {
-    0x00000000, 0x00000000, 0x000AAAAA, 0xA540DB6C,    /* 0x3000 */
-    0x00000802, 0x000AAAAA, 0xA540DB6C, 0x000009E2,    /* 0x3080 */
-};
-#define IS_DECOMPOSABLE(table,unicodeVal) \
-       (table[(unicodeVal) / 32] & (1 << (31 - ((unicodeVal) % 32))))
+static u_int32_t
+unicode_recursive_decompose(u_int16_t character, u_int16_t *convertedChars)
+{
+       u_int16_t value;
+       u_int32_t length;
+       u_int16_t firstChar;
+       u_int16_t theChar;
+       const u_int16_t *bmpMappings;
+       u_int32_t usedLength;
+
+       value = getmappedvalue16(
+               (const unicode_mappings16 *)__CFUniCharDecompositionTable,
+               __UniCharDecompositionTableLength, character);
+       length = EXTRACT_COUNT(value);
+       firstChar = value & 0x0FFF;
+       theChar = firstChar;
+       bmpMappings = (length == 1 ? &theChar : __CFUniCharMultipleDecompositionTable + firstChar);
+       usedLength = 0;
+
+       if (value & RECURSIVE_DECOMPOSITION) {
+           usedLength = unicode_recursive_decompose((u_int16_t)*bmpMappings, convertedChars);
+       
+           --length;   /* Decrement for the first char */
+           if (!usedLength)
+               return 0;
+           ++bmpMappings;
+           convertedChars += usedLength;
+       }
+       
+       usedLength += length;
+       
+       while (length--)
+               *(convertedChars++) = *(bmpMappings++);
+       
+       return (usedLength);
+}
+    
+#define HANGUL_SBASE 0xAC00
+#define HANGUL_LBASE 0x1100
+#define HANGUL_VBASE 0x1161
+#define HANGUL_TBASE 0x11A7
+
+#define HANGUL_SCOUNT 11172
+#define HANGUL_LCOUNT 19
+#define HANGUL_VCOUNT 21
+#define HANGUL_TCOUNT 28
+#define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT)

 
 /*
 
 /*

- * ucs_decompose - decompose a composed UCS-2 char
+ * unicode_decompose - decompose a composed Unicode char

  *
  * Composed Unicode characters are forbidden on
  * HFS Plus volumes. ucs_decompose will convert a
  * composed character into its correct decomposed
  * sequence.
  *
  *
  * Composed Unicode characters are forbidden on
  * HFS Plus volumes. ucs_decompose will convert a
  * composed character into its correct decomposed
  * sequence.
  *

- * Currently only MacRoman and MacJapanese chars
- * are handled.  Other composed characters are
- * passed unchanged.
+ * Similar to CFUniCharDecomposeCharacter
+ */
+static int
+unicode_decompose(u_int16_t character, u_int16_t *convertedChars)
+{
+       if ((character >= HANGUL_SBASE) &&
+           (character <= (HANGUL_SBASE + HANGUL_SCOUNT))) {
+               u_int32_t length;
+
+               character -= HANGUL_SBASE;
+               length = (character % HANGUL_TCOUNT ? 3 : 2);
+
+               *(convertedChars++) =
+                       character / HANGUL_NCOUNT + HANGUL_LBASE;
+               *(convertedChars++) =
+                       (character % HANGUL_NCOUNT) / HANGUL_TCOUNT + HANGUL_VBASE;
+               if (length > 2)
+                       *convertedChars = (character % HANGUL_TCOUNT) + HANGUL_TBASE;
+               return (length);
+       } else {
+               return (unicode_recursive_decompose(character, convertedChars));
+       }
+}
+
+/*
+ * unicode_combine - generate a precomposed Unicode char
+ *
+ * Precomposed Unicode characters are required for some volume
+ * formats and network protocols.  unicode_combine will combine
+ * a decomposed character sequence into a single precomposed
+ * (composite) character.
+ *
+ * Similar toCFUniCharPrecomposeCharacter but unicode_combine
+ * also handles Hangul Jamo characters.

  */
 static u_int16_t
  */
 static u_int16_t

-ucs_decompose(register u_int16_t ch, u_int16_t *cmb)
+unicode_combine(u_int16_t base, u_int16_t combining)

 {
 {

-       u_int16_t base;
+       u_int32_t value;
+
+       /* Check HANGUL */
+       if ((combining >= HANGUL_VBASE) && (combining < (HANGUL_TBASE + HANGUL_TCOUNT))) {
+               /* 2 char Hangul sequences */
+               if ((combining < (HANGUL_VBASE + HANGUL_VCOUNT)) &&
+                   (base >= HANGUL_LBASE && base < (HANGUL_LBASE + HANGUL_LCOUNT))) {
+                   return (HANGUL_SBASE +
+                           ((base - HANGUL_LBASE)*(HANGUL_VCOUNT*HANGUL_TCOUNT)) +
+                           ((combining  - HANGUL_VBASE)*HANGUL_TCOUNT));
+               }

        
        

-       *cmb = 0;
+               /* 3 char Hangul sequences */
+               if ((combining > HANGUL_TBASE) &&
+                   (base >= HANGUL_SBASE && base < (HANGUL_SBASE + HANGUL_SCOUNT))) {
+                       if ((base - HANGUL_SBASE) % HANGUL_TCOUNT)
+                               return (0);
+                       else
+                               return (base + (combining - HANGUL_TBASE));
+               }
+       }

 
 

-       if ((ch <= 0x00FF) && (ch >= 0x00C0)) {
-               ch -= 0x00C0;
-               
-               base = (u_int16_t) primary_char[ch];
+       value = getmappedvalue32(
+               (const unicode_mappings32 *)__CFUniCharPrecompSourceTable,
+               __CFUniCharPrecompositionTableLength, combining);

 
 

-               if (base <= 'z') {
-                       *cmb = (u_int16_t)0x0300 + (u_int16_t)combining_char[ch];
-               }
-       } else if ((ch > 0x3000) && (ch < 0x3100) &&
-                  IS_DECOMPOSABLE(__CJKDecompBitmap, ch - 0x3000)) {
-
-               /* Handle HIRAGANA LETTERs */
-               switch(ch) {
-               case 0x3071: base = 0x306F; *cmb = 0x309A; break; /* PA */
-               case 0x3074: base = 0x3072; *cmb = 0x309A; break; /* PI */
-               case 0x3077: base = 0x3075; *cmb = 0x309A; break; /* PU */
-               case 0x307A: base = 0x3078; *cmb = 0x309A; break; /* PE */
-
-               case 0x307D: base = 0x307B; *cmb = 0x309A; break; /* PO */
-               case 0x3094: base = 0x3046; *cmb = 0x3099; break; /* VU */
-               case 0x30D1: base = 0x30CF; *cmb = 0x309A; break; /* PA */
-               case 0x30D4: base = 0x30D2; *cmb = 0x309A; break; /* PI */
-
-               case 0x30D7: base = 0x30D5; *cmb = 0x309A; break; /* PU */
-               case 0x30DA: base = 0x30D8; *cmb = 0x309A; break; /* PE */
-               case 0x30DD: base = 0x30DB; *cmb = 0x309A; break; /* PO */
-               case 0x30F4: base = 0x30A6; *cmb = 0x3099; break; /* VU */
-
-               case 0x30F7: base = 0x30EF; *cmb = 0x3099; break; /* VA */
-               case 0x30F8: base = 0x30F0; *cmb = 0x3099; break; /* VI */
-               case 0x30F9: base = 0x30F1; *cmb = 0x3099; break; /* VE */
-               case 0x30FA: base = 0x30F2; *cmb = 0x3099; break; /* VO */
-               
-               default:
-                       /* the rest (41 of them) have a simple conversion */
-                       base = ch - 1;
-                       *cmb = 0x3099;
-               }
-       } else {
-               base = ch;
+       if (value) {
+               value = getmappedvalue16(
+                       (const unicode_mappings16 *)
+                       ((u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)),
+                       (value >> 16), base);

        }
        }

-       
-       return (base);
+       return (value);

 }
 
 }
 

+
+/*
+ * priortysort - order combining chars into canonical order
+ *
+ * Similar to CFUniCharPrioritySort
+ */
+static void
+priortysort(u_int16_t* characters, int count)
+{
+       u_int32_t p1, p2;
+       u_int16_t *ch1, *ch2;
+       u_int16_t *end;
+       int changes = 1;
+
+       end = characters + count;
+       do {
+               changes = 0;
+               ch1 = characters;
+               ch2 = characters + 1;
+               p2 = get_combining_class(*ch1);
+               while (ch2 < end) {
+                       p1 = p2;
+                       p2 = get_combining_class(*ch2);
+                       if (p1 > p2) {
+                               u_int32_t tmp;
+
+                               tmp = *ch1;
+                               *ch1 = *ch2;
+                               *ch2 = tmp;
+                               changes = 1;
+                       }
+                       ++ch1;
+                       ++ch2;
+               }
+       } while (changes);
+}