X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..8f6c56a50524aa785f7e596d52dddfb331e18961:/bsd/vfs/vfs_utfconv.c?ds=inline diff --git a/bsd/vfs/vfs_utfconv.c b/bsd/vfs/vfs_utfconv.c index 44f726355..6651c4f38 100644 --- a/bsd/vfs/vfs_utfconv.c +++ b/bsd/vfs/vfs_utfconv.c @@ -1,23 +1,33 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + /* + Includes Unicode 3.2 decomposition code derived from Core Foundation */ #include @@ -25,48 +35,139 @@ #include #include - /* - * UTF-8 (UCS Transformation Format) + * UTF-8 (Unicode Transformation Format) * - * The following subset of UTF-8 is used to encode UCS-2 filenames. It - * requires a maximum of three 3 bytes per UCS-2 character. Only the - * shortest encoding required to represent the significant UCS-2 bits - * is legal. + * UTF-8 is the Unicode Transformation Format that serializes a Unicode + * character as a sequence of one to four bytes. Only the shortest form + * required to represent the significant Unicode bits is legal. * * UTF-8 Multibyte Codes * - * Bytes Bits UCS-2 Min UCS-2 Max UTF-8 Byte Sequence (binary) - * ------------------------------------------------------------------- - * 1 7 0x0000 0x007F 0xxxxxxx - * 2 11 0x0080 0x07FF 110xxxxx 10xxxxxx - * 3 16 0x0800 0xFFFF 1110xxxx 10xxxxxx 10xxxxxx - * ------------------------------------------------------------------- + * Bytes Bits Unicode Min Unicode Max UTF-8 Byte Sequence (binary) + * ----------------------------------------------------------------------------- + * 1 7 0x0000 0x007F 0xxxxxxx + * 2 11 0x0080 0x07FF 110xxxxx 10xxxxxx + * 3 16 0x0800 0xFFFF 1110xxxx 10xxxxxx 10xxxxxx + * 4 21 0x10000 0x10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * ----------------------------------------------------------------------------- */ -#define UCS_TO_UTF_LEN(c) ((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : 3)) +#define UNICODE_TO_UTF8_LEN(c) \ + ((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : (((c) & 0xf800) == 0xd800 ? 2 : 3))) + +#define UCS_ALT_NULL 0x2400 + +/* Surrogate Pair Constants */ +#define SP_HALF_SHIFT 10 +#define SP_HALF_BASE 0x0010000UL +#define SP_HALF_MASK 0x3FFUL +#define SP_HIGH_FIRST 0xD800UL +#define SP_HIGH_LAST 0xDBFFUL +#define SP_LOW_FIRST 0xDC00UL +#define SP_LOW_LAST 0xDFFFUL -static u_int16_t ucs_decompose __P((u_int16_t, u_int16_t *)); + +#include "vfs_utfconvdata.h" /* - * utf8_encodelen - Calculates the UTF-8 encoding length for a UCS-2 filename + * Test for a combining character. + * + * Similar to __CFUniCharIsNonBaseCharacter except that + * unicode_combinable also includes Hangul Jamo characters. + */ +static inline int +unicode_combinable(u_int16_t character) +{ + const u_int8_t *bitmap = __CFUniCharCombiningBitmap; + u_int8_t value; + + if (character < 0x0300) + return (0); + + value = bitmap[(character >> 8) & 0xFF]; + + if (value == 0xFF) { + return (1); + } else if (value) { + bitmap = bitmap + ((value - 1) * 32) + 256; + return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0); + } + return (0); +} + +/* + * Test for a precomposed character. + * + * Similar to __CFUniCharIsDecomposableCharacter. + */ +static inline int +unicode_decomposeable(u_int16_t character) { + const u_int8_t *bitmap = __CFUniCharDecomposableBitmap; + u_int8_t value; + + if (character < 0x00C0) + return (0); + + value = bitmap[(character >> 8) & 0xFF]; + + if (value == 0xFF) { + return (1); + } else if (value) { + bitmap = bitmap + ((value - 1) * 32) + 256; + return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0); + } + return (0); +} + + +/* + * Get the combing class. + * + * Similar to CFUniCharGetCombiningPropertyForCharacter. + */ +static inline u_int8_t +get_combining_class(u_int16_t character) { + const u_int8_t *bitmap = __CFUniCharCombiningPropertyBitmap; + + u_int8_t value = bitmap[(character >> 8)]; + + if (value) { + bitmap = bitmap + (value * 256); + return bitmap[character % 256]; + } + return (0); +} + + +static int unicode_decompose(u_int16_t character, u_int16_t *convertedChars); + +static u_int16_t unicode_combine(u_int16_t base, u_int16_t combining); + +static void priortysort(u_int16_t* characters, int count); + +char utf_extrabytes[32] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 2, 2, 3, -1 +}; + + +/* + * utf8_encodelen - Calculates the UTF-8 encoding length for a Unicode filename * * NOTES: * If '/' chars are allowed on disk then an alternate * (replacement) char must be provided in altslash. * * input flags: - * UTF_REVERSE_ENDIAN: UCS-2 byteorder is opposite current runtime + * UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime */ size_t -utf8_encodelen(ucsp, ucslen, altslash, flags) - const u_int16_t * ucsp; - size_t ucslen; - u_int16_t altslash; - int flags; +utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash, + int flags) { u_int16_t ucs_ch; int charcnt; @@ -81,12 +182,12 @@ utf8_encodelen(ucsp, ucslen, altslash, flags) if (swapbytes) ucs_ch = NXSwapShort(ucs_ch); - if (altslash && ucs_ch == '/') - ucs_ch = altslash; - if (ucs_ch == '\0') - ucs_ch = 0xc080; + if (ucs_ch == '/') + ucs_ch = altslash ? altslash : '_'; + else if (ucs_ch == '\0') + ucs_ch = UCS_ALT_NULL; - len += UCS_TO_UTF_LEN(ucs_ch); + len += UNICODE_TO_UTF8_LEN(ucs_ch); } return (len); @@ -94,33 +195,36 @@ utf8_encodelen(ucsp, ucslen, altslash, flags) /* - * utf8_encodestr - Encodes a UCS-2 (Unicode) string to UTF-8 + * utf8_encodestr - Encodes a Unicode string to UTF-8 * * NOTES: - * The resulting UTF-8 string is not null terminated. + * The resulting UTF-8 string is NULL terminated. * * If '/' chars are allowed on disk then an alternate * (replacement) char must be provided in altslash. * * input flags: - * UTF_REVERSE_ENDIAN: UCS-2 byteorder is opposite current runtime + * UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime * UTF_NO_NULL_TERM: don't add NULL termination to UTF-8 output + * + * result: + * ENAMETOOLONG: Name didn't fit; only buflen bytes were encoded + * EINVAL: Illegal char found; char was replaced by an '_'. */ -int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) - const u_int16_t * ucsp; - size_t ucslen; - u_int8_t * utf8p; - size_t * utf8len; - size_t buflen; - u_int16_t altslash; - int flags; +int +utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p, + size_t * utf8len, size_t buflen, u_int16_t altslash, int flags) { u_int8_t * bufstart; u_int8_t * bufend; u_int16_t ucs_ch; + u_int16_t * chp = NULL; + u_int16_t sequence[8]; + int extra = 0; int charcnt; int swapbytes = (flags & UTF_REVERSE_ENDIAN); - int nullterm = ((flags & UTF_NO_NULL_TERM) == 0); + int nullterm = ((flags & UTF_NO_NULL_TERM) == 0); + int decompose = (flags & UTF_DECOMPOSED); int result = 0; bufstart = utf8p; @@ -130,18 +234,37 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) charcnt = ucslen / 2; while (charcnt-- > 0) { - ucs_ch = *ucsp++; + if (extra > 0) { + --extra; + ucs_ch = *chp++; + } else { + ucs_ch = swapbytes ? NXSwapShort(*ucsp++) : *ucsp++; - if (swapbytes) - ucs_ch = NXSwapShort(ucs_ch); - if (altslash && ucs_ch == '/') - ucs_ch = altslash; + if (decompose && unicode_decomposeable(ucs_ch)) { + extra = unicode_decompose(ucs_ch, sequence) - 1; + charcnt += extra; + ucs_ch = sequence[0]; + chp = &sequence[1]; + } + } - if ((ucs_ch < 0x0080) && (ucs_ch != '\0')) { + /* Slash and NULL are not permitted */ + if (ucs_ch == '/') { + if (altslash) + ucs_ch = altslash; + else { + ucs_ch = '_'; + result = EINVAL; + } + } else if (ucs_ch == '\0') { + ucs_ch = UCS_ALT_NULL; + } + + if (ucs_ch < 0x0080) { if (utf8p >= bufend) { result = ENAMETOOLONG; break; - } + } *utf8p++ = ucs_ch; } else if (ucs_ch < 0x800) { @@ -149,18 +272,40 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) result = ENAMETOOLONG; break; } - /* NOTE: NULL maps to 0xC080 */ - *utf8p++ = (ucs_ch >> 6) | 0xc0; - *utf8p++ = (ucs_ch & 0x3f) | 0x80; + *utf8p++ = 0xc0 | (ucs_ch >> 6); + *utf8p++ = 0x80 | (0x3f & ucs_ch); } else { + /* Combine valid surrogate pairs */ + if (ucs_ch >= SP_HIGH_FIRST && ucs_ch <= SP_HIGH_LAST + && charcnt > 0) { + u_int16_t ch2; + u_int32_t pair; + + ch2 = swapbytes ? NXSwapShort(*ucsp) : *ucsp; + if (ch2 >= SP_LOW_FIRST && ch2 <= SP_LOW_LAST) { + pair = ((ucs_ch - SP_HIGH_FIRST) << SP_HALF_SHIFT) + + (ch2 - SP_LOW_FIRST) + SP_HALF_BASE; + if ((utf8p + 3) >= bufend) { + result = ENAMETOOLONG; + break; + } + --charcnt; + ++ucsp; + *utf8p++ = 0xf0 | (pair >> 18); + *utf8p++ = 0x80 | (0x3f & (pair >> 12)); + *utf8p++ = 0x80 | (0x3f & (pair >> 6)); + *utf8p++ = 0x80 | (0x3f & pair); + continue; + } + } if ((utf8p + 2) >= bufend) { result = ENAMETOOLONG; break; } - *utf8p++ = (ucs_ch >> 12) | 0xe0; - *utf8p++ = ((ucs_ch >> 6) & 0x3f) | 0x80; - *utf8p++ = ((ucs_ch) & 0x3f) | 0x80; + *utf8p++ = 0xe0 | (ucs_ch >> 12); + *utf8p++ = 0x80 | (0x3f & (ucs_ch >> 6)); + *utf8p++ = 0x80 | (0x3f & ucs_ch); } } @@ -173,7 +318,7 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) /* - * utf8_decodestr - Decodes a UTF-8 string back to UCS-2 (Unicode) + * utf8_decodestr - Decodes a UTF-8 string back to Unicode * * NOTES: * The input UTF-8 string does not need to be null terminated @@ -183,234 +328,486 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) * (replacement) char must be provided in altslash. * * input flags: - * UTF_REV_ENDIAN: UCS-2 byteorder is oposite current runtime - * UTF_DECOMPOSED: UCS-2 output string must be fully decompsed + * UTF_REV_ENDIAN: Unicode byteorder is oposite current runtime + * UTF_DECOMPOSED: Unicode output string must be fully decompsed + * + * result: + * ENAMETOOLONG: Name didn't fit; only ucslen chars were decoded. + * EINVAL: Illegal UTF-8 sequence found. */ int -utf8_decodestr(utf8p, utf8len, ucsp, ucslen, buflen, altslash, flags) - const u_int8_t* utf8p; - size_t utf8len; - u_int16_t* ucsp; - size_t *ucslen; - size_t buflen; - u_int16_t altslash; - int flags; +utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp, + size_t *ucslen, size_t buflen, u_int16_t altslash, int flags) { u_int16_t* bufstart; u_int16_t* bufend; - u_int16_t ucs_ch; - u_int8_t byte; + unsigned int ucs_ch; + unsigned int byte; + int combcharcnt = 0; int result = 0; - int decompose, swapbytes; + int decompose, precompose, swapbytes; - decompose = (flags & UTF_DECOMPOSED); - swapbytes = (flags & UTF_REVERSE_ENDIAN); + decompose = (flags & UTF_DECOMPOSED); + precompose = (flags & UTF_PRECOMPOSED); + swapbytes = (flags & UTF_REVERSE_ENDIAN); bufstart = ucsp; bufend = (u_int16_t *)((u_int8_t *)ucsp + buflen); while (utf8len-- > 0 && (byte = *utf8p++) != '\0') { - if (ucsp >= bufend) { - result = ENAMETOOLONG; - goto stop; - } + if (ucsp >= bufend) + goto toolong; /* check for ascii */ if (byte < 0x80) { - ucs_ch = byte; + ucs_ch = byte; /* 1st byte */ } else { - switch (byte & 0xf0) { - /* 2 byte sequence*/ - case 0xc0: - case 0xd0: - /* extract bits 6 - 10 from first byte */ - ucs_ch = (byte & 0x1F) << 6; - if ((ucs_ch < 0x0080) && (*utf8p != 0x80)) { - result = EINVAL; /* seq not minimal */ - goto stop; - } - break; - /* 3 byte sequence*/ - case 0xe0: - /* extract bits 12 - 15 from first byte */ - ucs_ch = (byte & 0x0F) << 6; - - /* extract bits 6 - 11 from second byte */ - if (((byte = *utf8p++) & 0xc0) != 0x80) { - result = EINVAL; - goto stop; - } - utf8len--; - - ucs_ch += (byte & 0x3F); - ucs_ch <<= 6; - - if (ucs_ch < 0x0800) { - result = EINVAL; /* seq not minimal */ - goto stop; + u_int32_t ch; + int extrabytes = utf_extrabytes[byte >> 3]; + + if (utf8len < extrabytes) + goto invalid; + utf8len -= extrabytes; + + switch (extrabytes) { + case 1: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x00003080UL; + if (ch < 0x0080) + goto invalid; + ucs_ch = ch; + break; + case 2: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x000E2080UL; + if (ch < 0x0800) + goto invalid; + if (ch >= 0xD800) { + if (ch <= 0xDFFF) + goto invalid; + if (ch == 0xFFFE || ch == 0xFFFF) + goto invalid; } + ucs_ch = ch; break; + case 3: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 4th byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x03C82080UL + SP_HALF_BASE; + ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST; + if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST) + goto invalid; + *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch; + if (ucsp >= bufend) + goto toolong; + ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST; + if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST) + goto invalid; + *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch; + continue; default: - result = EINVAL; - goto stop; + goto invalid; } - - /* extract bits 0 - 5 from final byte */ - if (((byte = *utf8p++) & 0xc0) != 0x80) { - result = EINVAL; - goto stop; - } - utf8len--; - ucs_ch += (byte & 0x3F); - if (decompose) { - u_int16_t comb_ch; + if (unicode_decomposeable(ucs_ch)) { + u_int16_t sequence[8]; + int count, i; - ucs_ch = ucs_decompose(ucs_ch, &comb_ch); + count = unicode_decompose(ucs_ch, sequence); - if (comb_ch) { - if (swapbytes) - *ucsp++ = NXSwapShort(ucs_ch); - else - *ucsp++ = ucs_ch; - - if (ucsp >= bufend) { - result = ENAMETOOLONG; - goto stop; + for (i = 0; i < count; ++i) { + ucs_ch = sequence[i]; + *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch; + if (ucsp >= bufend) + goto toolong; + } + combcharcnt += count - 1; + continue; + } + } else if (precompose && (ucsp != bufstart)) { + u_int16_t composite, base; + + if (unicode_combinable(ucs_ch)) { + base = swapbytes ? NXSwapShort(*(ucsp - 1)) : *(ucsp - 1); + composite = unicode_combine(base, ucs_ch); + if (composite) { + --ucsp; + ucs_ch = composite; } - - ucs_ch = comb_ch; } } + if (ucs_ch == UCS_ALT_NULL) + ucs_ch = '\0'; } - if (ucs_ch == altslash) ucs_ch = '/'; - if (swapbytes) - ucs_ch = NXSwapShort(ucs_ch); - *ucsp++ = ucs_ch; + /* + * Make multiple combining character sequences canonical + */ + if (unicode_combinable(ucs_ch)) { + ++combcharcnt; /* start tracking a run */ + } else if (combcharcnt) { + if (combcharcnt > 1) { + priortysort(ucsp - combcharcnt, combcharcnt); + } + combcharcnt = 0; /* start over */ + } + *ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch; + } + /* + * Make a previous combining sequence canonical + */ + if (combcharcnt > 1) { + priortysort(ucsp - combcharcnt, combcharcnt); } -stop: + +exit: *ucslen = (u_int8_t*)ucsp - (u_int8_t*)bufstart; return (result); + +invalid: + result = EINVAL; + goto exit; + +toolong: + result = ENAMETOOLONG; + goto exit; } /* - * Lookup tables for Unicode chars 0x00C0 thru 0x00FF - * primary_char yields first decomposed char. If this - * char is an alpha char then get the combining char - * from the combining_char table and add 0x0300 to it. + * utf8_validatestr - Check for a valid UTF-8 string. */ +int +utf8_validatestr(const u_int8_t* utf8p, size_t utf8len) +{ + unsigned int byte; + u_int32_t ch; + unsigned int ucs_ch; + size_t extrabytes; -static unsigned char primary_char[64] = { - 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0xC6, 0x43, - - 0x45, 0x45, 0x45, 0x45, 0x49, 0x49, 0x49, 0x49, - - 0xD0, 0x4E, 0x4F, 0x4F, 0x4F, 0x4F, 0x4F, 0xD7, - - 0xD8, 0x55, 0x55, 0x55, 0x55, 0x59, 0xDE, 0xDF, - - 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0xE6, 0x63, - - 0x65, 0x65, 0x65, 0x65, 0x69, 0x69, 0x69, 0x69, - - 0xF0, 0x6E, 0x6F, 0x6F, 0x6F, 0x6F, 0x6F, 0xF7, - - 0xF8, 0x75, 0x75, 0x75, 0x75, 0x79, 0xFE, 0x79, -}; - -static unsigned char combining_char[64] = { - 0x00, 0x01, 0x02, 0x03, 0x08, 0x0A, 0xFF, 0x27, + while (utf8len-- > 0 && (byte = *utf8p++) != '\0') { + if (byte < 0x80) + continue; /* plain ascii */ + + extrabytes = utf_extrabytes[byte >> 3]; + + if (utf8len < extrabytes) + goto invalid; + utf8len -= extrabytes; + + switch (extrabytes) { + case 1: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x00003080UL; + if (ch < 0x0080) + goto invalid; + break; + case 2: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x000E2080UL; + if (ch < 0x0800) + goto invalid; + if (ch >= 0xD800) { + if (ch <= 0xDFFF) + goto invalid; + if (ch == 0xFFFE || ch == 0xFFFF) + goto invalid; + } + break; + case 3: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 4th byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x03C82080UL + SP_HALF_BASE; + ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST; + if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST) + goto invalid; + ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST; + if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST) + goto invalid; + break; + default: + goto invalid; + } + + } + return (0); +invalid: + return (EINVAL); +} - 0x00, 0x01, 0x02, 0x08, 0x00, 0x01, 0x02, 0x08, - 0xFF, 0x03, 0x00, 0x01, 0x02, 0x03, 0x08, 0xFF, + /* + * Unicode 3.2 decomposition code (derived from Core Foundation) + */ - 0xFF, 0x00, 0x01, 0x02, 0x08, 0x01, 0xFF, 0xFF, +typedef struct { + u_int32_t _key; + u_int32_t _value; +} unicode_mappings32; - 0x00, 0x01, 0x02, 0x03, 0x08, 0x0A, 0xFF, 0x27, +static inline u_int32_t +getmappedvalue32(const unicode_mappings32 *theTable, u_int32_t numElem, + u_int16_t character) +{ + const unicode_mappings32 *p, *q, *divider; + + if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key)) + return (0); + + p = theTable; + q = p + (numElem-1); + while (p <= q) { + divider = p + ((q - p) >> 1); /* divide by 2 */ + if (character < divider->_key) { q = divider - 1; } + else if (character > divider->_key) { p = divider + 1; } + else { return (divider->_value); } + } + return (0); +} - 0x00, 0x01, 0x02, 0x08, 0x00, 0x01, 0x02, 0x08, +#define RECURSIVE_DECOMPOSITION (1 << 15) +#define EXTRACT_COUNT(value) (((value) >> 12) & 0x0007) - 0xFF, 0x03, 0x00, 0x01, 0x02, 0x03, 0x08, 0xFF, +typedef struct { + u_int16_t _key; + u_int16_t _value; +} unicode_mappings16; - 0xFF, 0x00, 0x01, 0x02, 0x08, 0x01, 0xFF, 0x08 -}; +static inline u_int16_t +getmappedvalue16(const unicode_mappings16 *theTable, u_int32_t numElem, + u_int16_t character) +{ + const unicode_mappings16 *p, *q, *divider; + + if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key)) + return (0); + + p = theTable; + q = p + (numElem-1); + while (p <= q) { + divider = p + ((q - p) >> 1); /* divide by 2 */ + if (character < divider->_key) + q = divider - 1; + else if (character > divider->_key) + p = divider + 1; + else + return (divider->_value); + } + return (0); +} -/* CJK codepoints 0x3000 ~ 0x30FF */ -static const unsigned long __CJKDecompBitmap[] = { - 0x00000000, 0x00000000, 0x000AAAAA, 0xA540DB6C, /* 0x3000 */ - 0x00000802, 0x000AAAAA, 0xA540DB6C, 0x000009E2, /* 0x3080 */ -}; -#define IS_DECOMPOSABLE(table,unicodeVal) \ - (table[(unicodeVal) / 32] & (1 << (31 - ((unicodeVal) % 32)))) +static u_int32_t +unicode_recursive_decompose(u_int16_t character, u_int16_t *convertedChars) +{ + u_int16_t value; + u_int32_t length; + u_int16_t firstChar; + u_int16_t theChar; + const u_int16_t *bmpMappings; + u_int32_t usedLength; + + value = getmappedvalue16( + (const unicode_mappings16 *)__CFUniCharDecompositionTable, + __UniCharDecompositionTableLength, character); + length = EXTRACT_COUNT(value); + firstChar = value & 0x0FFF; + theChar = firstChar; + bmpMappings = (length == 1 ? &theChar : __CFUniCharMultipleDecompositionTable + firstChar); + usedLength = 0; + + if (value & RECURSIVE_DECOMPOSITION) { + usedLength = unicode_recursive_decompose((u_int16_t)*bmpMappings, convertedChars); + + --length; /* Decrement for the first char */ + if (!usedLength) + return 0; + ++bmpMappings; + convertedChars += usedLength; + } + + usedLength += length; + + while (length--) + *(convertedChars++) = *(bmpMappings++); + + return (usedLength); +} + +#define HANGUL_SBASE 0xAC00 +#define HANGUL_LBASE 0x1100 +#define HANGUL_VBASE 0x1161 +#define HANGUL_TBASE 0x11A7 + +#define HANGUL_SCOUNT 11172 +#define HANGUL_LCOUNT 19 +#define HANGUL_VCOUNT 21 +#define HANGUL_TCOUNT 28 +#define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT) /* - * ucs_decompose - decompose a composed UCS-2 char + * unicode_decompose - decompose a composed Unicode char * * Composed Unicode characters are forbidden on * HFS Plus volumes. ucs_decompose will convert a * composed character into its correct decomposed * sequence. * - * Currently only MacRoman and MacJapanese chars - * are handled. Other composed characters are - * passed unchanged. + * Similar to CFUniCharDecomposeCharacter + */ +static int +unicode_decompose(u_int16_t character, u_int16_t *convertedChars) +{ + if ((character >= HANGUL_SBASE) && + (character <= (HANGUL_SBASE + HANGUL_SCOUNT))) { + u_int32_t length; + + character -= HANGUL_SBASE; + length = (character % HANGUL_TCOUNT ? 3 : 2); + + *(convertedChars++) = + character / HANGUL_NCOUNT + HANGUL_LBASE; + *(convertedChars++) = + (character % HANGUL_NCOUNT) / HANGUL_TCOUNT + HANGUL_VBASE; + if (length > 2) + *convertedChars = (character % HANGUL_TCOUNT) + HANGUL_TBASE; + return (length); + } else { + return (unicode_recursive_decompose(character, convertedChars)); + } +} + +/* + * unicode_combine - generate a precomposed Unicode char + * + * Precomposed Unicode characters are required for some volume + * formats and network protocols. unicode_combine will combine + * a decomposed character sequence into a single precomposed + * (composite) character. + * + * Similar toCFUniCharPrecomposeCharacter but unicode_combine + * also handles Hangul Jamo characters. */ static u_int16_t -ucs_decompose(register u_int16_t ch, u_int16_t *cmb) +unicode_combine(u_int16_t base, u_int16_t combining) { - u_int16_t base; + u_int32_t value; + + /* Check HANGUL */ + if ((combining >= HANGUL_VBASE) && (combining < (HANGUL_TBASE + HANGUL_TCOUNT))) { + /* 2 char Hangul sequences */ + if ((combining < (HANGUL_VBASE + HANGUL_VCOUNT)) && + (base >= HANGUL_LBASE && base < (HANGUL_LBASE + HANGUL_LCOUNT))) { + return (HANGUL_SBASE + + ((base - HANGUL_LBASE)*(HANGUL_VCOUNT*HANGUL_TCOUNT)) + + ((combining - HANGUL_VBASE)*HANGUL_TCOUNT)); + } - *cmb = 0; + /* 3 char Hangul sequences */ + if ((combining > HANGUL_TBASE) && + (base >= HANGUL_SBASE && base < (HANGUL_SBASE + HANGUL_SCOUNT))) { + if ((base - HANGUL_SBASE) % HANGUL_TCOUNT) + return (0); + else + return (base + (combining - HANGUL_TBASE)); + } + } - if ((ch <= 0x00FF) && (ch >= 0x00C0)) { - ch -= 0x00C0; - - base = (u_int16_t) primary_char[ch]; + value = getmappedvalue32( + (const unicode_mappings32 *)__CFUniCharPrecompSourceTable, + __CFUniCharPrecompositionTableLength, combining); - if (base <= 'z') { - *cmb = (u_int16_t)0x0300 + (u_int16_t)combining_char[ch]; - } - } else if ((ch > 0x3000) && (ch < 0x3100) && - IS_DECOMPOSABLE(__CJKDecompBitmap, ch - 0x3000)) { - - /* Handle HIRAGANA LETTERs */ - switch(ch) { - case 0x3071: base = 0x306F; *cmb = 0x309A; break; /* PA */ - case 0x3074: base = 0x3072; *cmb = 0x309A; break; /* PI */ - case 0x3077: base = 0x3075; *cmb = 0x309A; break; /* PU */ - case 0x307A: base = 0x3078; *cmb = 0x309A; break; /* PE */ - - case 0x307D: base = 0x307B; *cmb = 0x309A; break; /* PO */ - case 0x3094: base = 0x3046; *cmb = 0x3099; break; /* VU */ - case 0x30D1: base = 0x30CF; *cmb = 0x309A; break; /* PA */ - case 0x30D4: base = 0x30D2; *cmb = 0x309A; break; /* PI */ - - case 0x30D7: base = 0x30D5; *cmb = 0x309A; break; /* PU */ - case 0x30DA: base = 0x30D8; *cmb = 0x309A; break; /* PE */ - case 0x30DD: base = 0x30DB; *cmb = 0x309A; break; /* PO */ - case 0x30F4: base = 0x30A6; *cmb = 0x3099; break; /* VU */ - - case 0x30F7: base = 0x30EF; *cmb = 0x3099; break; /* VA */ - case 0x30F8: base = 0x30F0; *cmb = 0x3099; break; /* VI */ - case 0x30F9: base = 0x30F1; *cmb = 0x3099; break; /* VE */ - case 0x30FA: base = 0x30F2; *cmb = 0x3099; break; /* VO */ - - default: - /* the rest (41 of them) have a simple conversion */ - base = ch - 1; - *cmb = 0x3099; - } - } else { - base = ch; + if (value) { + value = getmappedvalue16( + (const unicode_mappings16 *) + ((u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)), + (value >> 16), base); } - - return (base); + return (value); } + +/* + * priortysort - order combining chars into canonical order + * + * Similar to CFUniCharPrioritySort + */ +static void +priortysort(u_int16_t* characters, int count) +{ + u_int32_t p1, p2; + u_int16_t *ch1, *ch2; + u_int16_t *end; + int changes = 1; + + end = characters + count; + do { + changes = 0; + ch1 = characters; + ch2 = characters + 1; + p2 = get_combining_class(*ch1); + while (ch2 < end) { + p1 = p2; + p2 = get_combining_class(*ch2); + if (p1 > p2) { + u_int32_t tmp; + + tmp = *ch1; + *ch1 = *ch2; + *ch2 = tmp; + changes = 1; + } + ++ch1; + ++ch2; + } + } while (changes); +}