X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/1c79356b52d46aa6b508fb032f5ae709b1f2897b..d190cdc3f5544636abb56dc1874be391d3e1b148:/bsd/vfs/vfs_utfconv.c diff --git a/bsd/vfs/vfs_utfconv.c b/bsd/vfs/vfs_utfconv.c index 44f726355..1f014aacf 100644 --- a/bsd/vfs/vfs_utfconv.c +++ b/bsd/vfs/vfs_utfconv.c @@ -1,92 +1,236 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + /* + Includes Unicode 3.2 decomposition code derived from Core Foundation */ #include #include #include -#include +#include +#include +#if defined(KERNEL) && !defined(VFS_UTF8_UNIT_TEST) +#include +#else +#include +#endif /* - * UTF-8 (UCS Transformation Format) + * UTF-8 (Unicode Transformation Format) * - * The following subset of UTF-8 is used to encode UCS-2 filenames. It - * requires a maximum of three 3 bytes per UCS-2 character. Only the - * shortest encoding required to represent the significant UCS-2 bits - * is legal. + * UTF-8 is the Unicode Transformation Format that serializes a Unicode + * character as a sequence of one to four bytes. Only the shortest form + * required to represent the significant Unicode bits is legal. * * UTF-8 Multibyte Codes * - * Bytes Bits UCS-2 Min UCS-2 Max UTF-8 Byte Sequence (binary) - * ------------------------------------------------------------------- - * 1 7 0x0000 0x007F 0xxxxxxx - * 2 11 0x0080 0x07FF 110xxxxx 10xxxxxx - * 3 16 0x0800 0xFFFF 1110xxxx 10xxxxxx 10xxxxxx - * ------------------------------------------------------------------- + * Bytes Bits Unicode Min Unicode Max UTF-8 Byte Sequence (binary) + * ----------------------------------------------------------------------------- + * 1 7 0x0000 0x007F 0xxxxxxx + * 2 11 0x0080 0x07FF 110xxxxx 10xxxxxx + * 3 16 0x0800 0xFFFF 1110xxxx 10xxxxxx 10xxxxxx + * 4 21 0x10000 0x10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * ----------------------------------------------------------------------------- */ -#define UCS_TO_UTF_LEN(c) ((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : 3)) +#define UNICODE_TO_UTF8_LEN(c) \ + ((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : (((c) & 0xf800) == 0xd800 ? 2 : 3))) + +#define UCS_ALT_NULL 0x2400 + +/* Surrogate Pair Constants */ +#define SP_HALF_SHIFT 10 +#define SP_HALF_BASE 0x0010000u +#define SP_HALF_MASK 0x3FFu +#define SP_HIGH_FIRST 0xD800u +#define SP_HIGH_LAST 0xDBFFu +#define SP_LOW_FIRST 0xDC00u +#define SP_LOW_LAST 0xDFFFu -static u_int16_t ucs_decompose __P((u_int16_t, u_int16_t *)); + +#include "vfs_utfconvdata.h" /* - * utf8_encodelen - Calculates the UTF-8 encoding length for a UCS-2 filename + * Test for a combining character. * - * NOTES: - * If '/' chars are allowed on disk then an alternate - * (replacement) char must be provided in altslash. + * Similar to __CFUniCharIsNonBaseCharacter except that + * unicode_combinable also includes Hangul Jamo characters. + */ +int +unicode_combinable(u_int16_t character) +{ + const u_int8_t *bitmap = __CFUniCharCombiningBitmap; + u_int8_t value; + + if (character < 0x0300) + return (0); + + value = bitmap[(character >> 8) & 0xFF]; + + if (value == 0xFF) { + return (1); + } else if (value) { + bitmap = bitmap + ((value - 1) * 32) + 256; + return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0); + } + return (0); +} + +/* + * Test for a precomposed character. * - * input flags: - * UTF_REVERSE_ENDIAN: UCS-2 byteorder is opposite current runtime + * Similar to __CFUniCharIsDecomposableCharacter. + */ +int +unicode_decomposeable(u_int16_t character) { + const u_int8_t *bitmap = __CFUniCharDecomposableBitmap; + u_int8_t value; + + if (character < 0x00C0) + return (0); + + value = bitmap[(character >> 8) & 0xFF]; + + if (value == 0xFF) { + return (1); + } else if (value) { + bitmap = bitmap + ((value - 1) * 32) + 256; + return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0); + } + return (0); +} + + +/* + * Get the combing class. + * + * Similar to CFUniCharGetCombiningPropertyForCharacter. + */ +static inline u_int8_t +get_combining_class(u_int16_t character) { + const u_int8_t *bitmap = __CFUniCharCombiningPropertyBitmap; + + u_int8_t value = bitmap[(character >> 8)]; + + if (value) { + bitmap = bitmap + (value * 256); + return bitmap[character % 256]; + } + return (0); +} + + +static int unicode_decompose(u_int16_t character, u_int16_t *convertedChars); + +static u_int16_t unicode_combine(u_int16_t base, u_int16_t combining); + +static void prioritysort(u_int16_t* characters, int count); + +static u_int16_t ucs_to_sfm(u_int16_t ucs_ch, int lastchar); + +static u_int16_t sfm_to_ucs(u_int16_t ucs_ch); + + +char utf_extrabytes[32] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 2, 2, 3, -1 +}; + +const char hexdigits[16] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' +}; + +/* + * utf8_encodelen - Calculate the UTF-8 encoding length + * + * This function takes a Unicode input string, ucsp, of ucslen bytes + * and calculates the size of the UTF-8 output in bytes (not including + * a NULL termination byte). The string must reside in kernel memory. + * + * If '/' chars are possible in the Unicode input then an alternate + * (replacement) char should be provided in altslash. + * + * FLAGS + * UTF_REVERSE_ENDIAN: Unicode byte order is opposite current runtime + * + * UTF_BIG_ENDIAN: Unicode byte order is always big endian + * + * UTF_LITTLE_ENDIAN: Unicode byte order is always little endian + * + * UTF_DECOMPOSED: generate fully decomposed output + * + * UTF_PRECOMPOSED is ignored since utf8_encodestr doesn't support it + * + * ERRORS + * None */ size_t -utf8_encodelen(ucsp, ucslen, altslash, flags) - const u_int16_t * ucsp; - size_t ucslen; - u_int16_t altslash; - int flags; +utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash, int flags) { u_int16_t ucs_ch; - int charcnt; + u_int16_t * chp = NULL; + u_int16_t sequence[8]; + int extra = 0; + size_t charcnt; int swapbytes = (flags & UTF_REVERSE_ENDIAN); + int decompose = (flags & UTF_DECOMPOSED); size_t len; - + charcnt = ucslen / 2; len = 0; while (charcnt-- > 0) { - ucs_ch = *ucsp++; - - if (swapbytes) - ucs_ch = NXSwapShort(ucs_ch); - if (altslash && ucs_ch == '/') - ucs_ch = altslash; - if (ucs_ch == '\0') - ucs_ch = 0xc080; - - len += UCS_TO_UTF_LEN(ucs_ch); + if (extra > 0) { + --extra; + ucs_ch = *chp++; + } else { + ucs_ch = *ucsp++; + if (swapbytes) { + ucs_ch = OSSwapInt16(ucs_ch); + } + if (ucs_ch == '/') { + ucs_ch = altslash ? altslash : '_'; + } else if (ucs_ch == '\0') { + ucs_ch = UCS_ALT_NULL; + } else if (decompose && unicode_decomposeable(ucs_ch)) { + extra = unicode_decompose(ucs_ch, sequence) - 1; + charcnt += extra; + ucs_ch = sequence[0]; + chp = &sequence[1]; + } + } + len += UNICODE_TO_UTF8_LEN(ucs_ch); } return (len); @@ -94,35 +238,47 @@ utf8_encodelen(ucsp, ucslen, altslash, flags) /* - * utf8_encodestr - Encodes a UCS-2 (Unicode) string to UTF-8 + * utf8_encodestr - Encodes a Unicode string to UTF-8 * * NOTES: - * The resulting UTF-8 string is not null terminated. + * The resulting UTF-8 string is NULL terminated. * * If '/' chars are allowed on disk then an alternate * (replacement) char must be provided in altslash. * * input flags: - * UTF_REVERSE_ENDIAN: UCS-2 byteorder is opposite current runtime + * UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime + * + * UTF_BIG_ENDIAN: Unicode byte order is always big endian + * + * UTF_LITTLE_ENDIAN: Unicode byte order is always little endian + * + * UTF_DECOMPOSED: generate fully decomposed output + * * UTF_NO_NULL_TERM: don't add NULL termination to UTF-8 output + * + * result: + * ENAMETOOLONG: Name didn't fit; only buflen bytes were encoded + * + * EINVAL: Illegal char found; char was replaced by an '_'. */ -int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) - const u_int16_t * ucsp; - size_t ucslen; - u_int8_t * utf8p; - size_t * utf8len; - size_t buflen; - u_int16_t altslash; - int flags; +int +utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p, + size_t * utf8len, size_t buflen, u_int16_t altslash, int flags) { u_int8_t * bufstart; u_int8_t * bufend; u_int16_t ucs_ch; - int charcnt; + u_int16_t * chp = NULL; + u_int16_t sequence[8]; + int extra = 0; + size_t charcnt; int swapbytes = (flags & UTF_REVERSE_ENDIAN); - int nullterm = ((flags & UTF_NO_NULL_TERM) == 0); + int nullterm = ((flags & UTF_NO_NULL_TERM) == 0); + int decompose = (flags & UTF_DECOMPOSED); + int sfmconv = (flags & UTF_SFM_CONVERSIONS); int result = 0; - + bufstart = utf8p; bufend = bufstart + buflen; if (nullterm) @@ -130,18 +286,37 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) charcnt = ucslen / 2; while (charcnt-- > 0) { - ucs_ch = *ucsp++; + if (extra > 0) { + --extra; + ucs_ch = *chp++; + } else { + ucs_ch = swapbytes ? OSSwapInt16(*ucsp++) : *ucsp++; + + if (decompose && unicode_decomposeable(ucs_ch)) { + extra = unicode_decompose(ucs_ch, sequence) - 1; + charcnt += extra; + ucs_ch = sequence[0]; + chp = &sequence[1]; + } + } - if (swapbytes) - ucs_ch = NXSwapShort(ucs_ch); - if (altslash && ucs_ch == '/') - ucs_ch = altslash; + /* Slash and NULL are not permitted */ + if (ucs_ch == '/') { + if (altslash) + ucs_ch = altslash; + else { + ucs_ch = '_'; + result = EINVAL; + } + } else if (ucs_ch == '\0') { + ucs_ch = UCS_ALT_NULL; + } - if ((ucs_ch < 0x0080) && (ucs_ch != '\0')) { + if (ucs_ch < 0x0080) { if (utf8p >= bufend) { result = ENAMETOOLONG; break; - } + } *utf8p++ = ucs_ch; } else if (ucs_ch < 0x800) { @@ -149,18 +324,56 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) result = ENAMETOOLONG; break; } - /* NOTE: NULL maps to 0xC080 */ - *utf8p++ = (ucs_ch >> 6) | 0xc0; - *utf8p++ = (ucs_ch & 0x3f) | 0x80; + *utf8p++ = 0xc0 | (ucs_ch >> 6); + *utf8p++ = 0x80 | (0x3f & ucs_ch); } else { + /* These chars never valid Unicode. */ + if (ucs_ch == 0xFFFE || ucs_ch == 0xFFFF) { + result = EINVAL; + break; + } + + /* Combine valid surrogate pairs */ + if (ucs_ch >= SP_HIGH_FIRST && ucs_ch <= SP_HIGH_LAST + && charcnt > 0) { + u_int16_t ch2; + u_int32_t pair; + + ch2 = swapbytes ? OSSwapInt16(*ucsp) : *ucsp; + if (ch2 >= SP_LOW_FIRST && ch2 <= SP_LOW_LAST) { + pair = ((ucs_ch - SP_HIGH_FIRST) << SP_HALF_SHIFT) + + (ch2 - SP_LOW_FIRST) + SP_HALF_BASE; + if ((utf8p + 3) >= bufend) { + result = ENAMETOOLONG; + break; + } + --charcnt; + ++ucsp; + *utf8p++ = 0xf0 | (pair >> 18); + *utf8p++ = 0x80 | (0x3f & (pair >> 12)); + *utf8p++ = 0x80 | (0x3f & (pair >> 6)); + *utf8p++ = 0x80 | (0x3f & pair); + continue; + } + } else if (sfmconv) { + ucs_ch = sfm_to_ucs(ucs_ch); + if (ucs_ch < 0x0080) { + if (utf8p >= bufend) { + result = ENAMETOOLONG; + break; + } + *utf8p++ = ucs_ch; + continue; + } + } if ((utf8p + 2) >= bufend) { result = ENAMETOOLONG; break; } - *utf8p++ = (ucs_ch >> 12) | 0xe0; - *utf8p++ = ((ucs_ch >> 6) & 0x3f) | 0x80; - *utf8p++ = ((ucs_ch) & 0x3f) | 0x80; + *utf8p++ = 0xe0 | (ucs_ch >> 12); + *utf8p++ = 0x80 | (0x3f & (ucs_ch >> 6)); + *utf8p++ = 0x80 | (0x3f & ucs_ch); } } @@ -171,9 +384,26 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) return (result); } +// Pushes a character taking account of combining character sequences +static void push(uint16_t ucs_ch, int *combcharcnt, uint16_t **ucsp) +{ + /* + * Make multiple combining character sequences canonical + */ + if (unicode_combinable(ucs_ch)) { + ++*combcharcnt; /* start tracking a run */ + } else if (*combcharcnt) { + if (*combcharcnt > 1) { + prioritysort(*ucsp - *combcharcnt, *combcharcnt); + } + *combcharcnt = 0; /* start over */ + } + + *(*ucsp)++ = ucs_ch; +} /* - * utf8_decodestr - Decodes a UTF-8 string back to UCS-2 (Unicode) + * utf8_decodestr - Decodes a UTF-8 string back to Unicode * * NOTES: * The input UTF-8 string does not need to be null terminated @@ -183,234 +413,767 @@ int utf8_encodestr(ucsp, ucslen, utf8p, utf8len, buflen, altslash, flags) * (replacement) char must be provided in altslash. * * input flags: - * UTF_REV_ENDIAN: UCS-2 byteorder is oposite current runtime - * UTF_DECOMPOSED: UCS-2 output string must be fully decompsed + * UTF_REV_ENDIAN: Unicode byte order is opposite current runtime + * + * UTF_BIG_ENDIAN: Unicode byte order is always big endian + * + * UTF_LITTLE_ENDIAN: Unicode byte order is always little endian + * + * UTF_DECOMPOSED: generate fully decomposed output (NFD) + * + * UTF_PRECOMPOSED: generate precomposed output (NFC) + * + * UTF_ESCAPE_ILLEGAL: percent escape any illegal UTF-8 input + * + * result: + * ENAMETOOLONG: Name didn't fit; only ucslen chars were decoded. + * + * EINVAL: Illegal UTF-8 sequence found. */ int -utf8_decodestr(utf8p, utf8len, ucsp, ucslen, buflen, altslash, flags) - const u_int8_t* utf8p; - size_t utf8len; - u_int16_t* ucsp; - size_t *ucslen; - size_t buflen; - u_int16_t altslash; - int flags; +utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp, + size_t *ucslen, size_t buflen, u_int16_t altslash, int flags) { u_int16_t* bufstart; u_int16_t* bufend; - u_int16_t ucs_ch; - u_int8_t byte; + unsigned int ucs_ch; + unsigned int byte; + int combcharcnt = 0; int result = 0; - int decompose, swapbytes; + int decompose, precompose, escaping; + int sfmconv; + int extrabytes; - decompose = (flags & UTF_DECOMPOSED); - swapbytes = (flags & UTF_REVERSE_ENDIAN); + decompose = (flags & UTF_DECOMPOSED); + precompose = (flags & UTF_PRECOMPOSED); + escaping = (flags & UTF_ESCAPE_ILLEGAL); + sfmconv = (flags & UTF_SFM_CONVERSIONS); bufstart = ucsp; bufend = (u_int16_t *)((u_int8_t *)ucsp + buflen); while (utf8len-- > 0 && (byte = *utf8p++) != '\0') { - if (ucsp >= bufend) { - result = ENAMETOOLONG; - goto stop; - } + if (ucsp >= bufend) + goto toolong; /* check for ascii */ if (byte < 0x80) { - ucs_ch = byte; + ucs_ch = sfmconv ? ucs_to_sfm(byte, utf8len == 0) : byte; } else { - switch (byte & 0xf0) { - /* 2 byte sequence*/ - case 0xc0: - case 0xd0: - /* extract bits 6 - 10 from first byte */ - ucs_ch = (byte & 0x1F) << 6; - if ((ucs_ch < 0x0080) && (*utf8p != 0x80)) { - result = EINVAL; /* seq not minimal */ - goto stop; - } - break; - /* 3 byte sequence*/ - case 0xe0: - /* extract bits 12 - 15 from first byte */ - ucs_ch = (byte & 0x0F) << 6; - - /* extract bits 6 - 11 from second byte */ - if (((byte = *utf8p++) & 0xc0) != 0x80) { - result = EINVAL; - goto stop; - } - utf8len--; - - ucs_ch += (byte & 0x3F); - ucs_ch <<= 6; + u_int32_t ch; - if (ucs_ch < 0x0800) { - result = EINVAL; /* seq not minimal */ - goto stop; + extrabytes = utf_extrabytes[byte >> 3]; + if ((extrabytes < 0) || ((int)utf8len < extrabytes)) { + goto escape; + } + utf8len -= extrabytes; + + switch (extrabytes) { + case 1: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto escape2; + ch += byte; + ch -= 0x00003080UL; + if (ch < 0x0080) + goto escape2; + ucs_ch = ch; + break; + case 2: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto escape2; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto escape3; + ch += byte; + ch -= 0x000E2080UL; + if (ch < 0x0800) + goto escape3; + if (ch >= 0xD800) { + if (ch <= 0xDFFF) + goto escape3; + if (ch == 0xFFFE || ch == 0xFFFF) + goto escape3; } + ucs_ch = ch; break; + case 3: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto escape2; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto escape3; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 4th byte */ + if ((byte >> 6) != 2) + goto escape4; + ch += byte; + ch -= 0x03C82080UL + SP_HALF_BASE; + ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST; + if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST) + goto escape4; + push(ucs_ch, &combcharcnt, &ucsp); + if (ucsp >= bufend) + goto toolong; + ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST; + if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST) { + --ucsp; + goto escape4; + } + *ucsp++ = ucs_ch; + continue; default: result = EINVAL; - goto stop; + goto exit; } - - /* extract bits 0 - 5 from final byte */ - if (((byte = *utf8p++) & 0xc0) != 0x80) { - result = EINVAL; - goto stop; - } - utf8len--; - ucs_ch += (byte & 0x3F); - if (decompose) { - u_int16_t comb_ch; + if (unicode_decomposeable(ucs_ch)) { + u_int16_t sequence[8]; + int count, i; - ucs_ch = ucs_decompose(ucs_ch, &comb_ch); + count = unicode_decompose(ucs_ch, sequence); - if (comb_ch) { - if (swapbytes) - *ucsp++ = NXSwapShort(ucs_ch); - else - *ucsp++ = ucs_ch; + for (i = 0; i < count; ++i) { + if (ucsp >= bufend) + goto toolong; - if (ucsp >= bufend) { - result = ENAMETOOLONG; - goto stop; + push(sequence[i], &combcharcnt, &ucsp); } - ucs_ch = comb_ch; + continue; + } + } else if (precompose && (ucsp != bufstart)) { + u_int16_t composite, base; + + if (unicode_combinable(ucs_ch)) { + base = ucsp[-1]; + composite = unicode_combine(base, ucs_ch); + if (composite) { + --ucsp; + ucs_ch = composite; + } } } + if (ucs_ch == UCS_ALT_NULL) + ucs_ch = '\0'; } - if (ucs_ch == altslash) ucs_ch = '/'; - if (swapbytes) - ucs_ch = NXSwapShort(ucs_ch); + push(ucs_ch, &combcharcnt, &ucsp); + continue; + + /* + * Escape illegal UTF-8 into something legal. + */ +escape4: + utf8p -= 3; + goto escape; +escape3: + utf8p -= 2; + goto escape; +escape2: + utf8p -= 1; +escape: + if (!escaping) { + result = EINVAL; + goto exit; + } + if (extrabytes > 0) + utf8len += extrabytes; + byte = *(utf8p - 1); + + if ((ucsp + 2) >= bufend) + goto toolong; + + /* Make a previous combining sequence canonical. */ + if (combcharcnt > 1) { + prioritysort(ucsp - combcharcnt, combcharcnt); + } + combcharcnt = 0; + + ucs_ch = '%'; *ucsp++ = ucs_ch; + ucs_ch = hexdigits[byte >> 4]; + *ucsp++ = ucs_ch; + ucs_ch = hexdigits[byte & 0x0F]; + *ucsp++ = ucs_ch; + } + /* + * Make a previous combining sequence canonical + */ + if (combcharcnt > 1) { + prioritysort(ucsp - combcharcnt, combcharcnt); + } + + if (flags & UTF_REVERSE_ENDIAN) { + uint16_t *p = bufstart; + while (p < ucsp) { + *p = OSSwapInt16(*p); + ++p; + } } -stop: + +exit: *ucslen = (u_int8_t*)ucsp - (u_int8_t*)bufstart; return (result); + +toolong: + result = ENAMETOOLONG; + goto exit; } /* - * Lookup tables for Unicode chars 0x00C0 thru 0x00FF - * primary_char yields first decomposed char. If this - * char is an alpha char then get the combining char - * from the combining_char table and add 0x0300 to it. + * utf8_validatestr - Check for a valid UTF-8 string. */ +int +utf8_validatestr(const u_int8_t* utf8p, size_t utf8len) +{ + unsigned int byte; + u_int32_t ch; + unsigned int ucs_ch; + size_t extrabytes; -static unsigned char primary_char[64] = { - 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0xC6, 0x43, - - 0x45, 0x45, 0x45, 0x45, 0x49, 0x49, 0x49, 0x49, - - 0xD0, 0x4E, 0x4F, 0x4F, 0x4F, 0x4F, 0x4F, 0xD7, + while (utf8len-- > 0 && (byte = *utf8p++) != '\0') { + if (byte < 0x80) + continue; /* plain ascii */ + + extrabytes = utf_extrabytes[byte >> 3]; + + if (utf8len < extrabytes) + goto invalid; + utf8len -= extrabytes; + + switch (extrabytes) { + case 1: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x00003080UL; + if (ch < 0x0080) + goto invalid; + break; + case 2: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x000E2080UL; + if (ch < 0x0800) + goto invalid; + if (ch >= 0xD800) { + if (ch <= 0xDFFF) + goto invalid; + if (ch == 0xFFFE || ch == 0xFFFF) + goto invalid; + } + break; + case 3: + ch = byte; ch <<= 6; /* 1st byte */ + byte = *utf8p++; /* 2nd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 3rd byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; ch <<= 6; + byte = *utf8p++; /* 4th byte */ + if ((byte >> 6) != 2) + goto invalid; + ch += byte; + ch -= 0x03C82080UL + SP_HALF_BASE; + ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST; + if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST) + goto invalid; + ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST; + if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST) + goto invalid; + break; + default: + goto invalid; + } + + } + return (0); +invalid: + return (EINVAL); +} - 0xD8, 0x55, 0x55, 0x55, 0x55, 0x59, 0xDE, 0xDF, +/* + * utf8_normalizestr - Normalize a UTF-8 string (NFC or NFD) + * + * This function takes an UTF-8 input string, instr, of inlen bytes + * and produces normalized UTF-8 output into a buffer of buflen bytes + * pointed to by outstr. The size of the output in bytes (not including + * a NULL termination byte) is returned in outlen. In-place conversions + * are not supported (i.e. instr != outstr).] + + * FLAGS + * UTF_DECOMPOSED: output string will be fully decomposed (NFD) + * + * UTF_PRECOMPOSED: output string will be precomposed (NFC) + * + * UTF_NO_NULL_TERM: do not add null termination to output string + * + * UTF_ESCAPE_ILLEGAL: percent escape any illegal UTF-8 input + * + * ERRORS + * ENAMETOOLONG: output did not fit or input exceeded MAXPATHLEN bytes + * + * EINVAL: illegal UTF-8 sequence encountered or invalid flags + */ +int +utf8_normalizestr(const u_int8_t* instr, size_t inlen, u_int8_t* outstr, + size_t *outlen, size_t buflen, int flags) +{ + u_int16_t unicodebuf[32]; + u_int16_t* unistr = NULL; + size_t unicode_bytes; + size_t uft8_bytes; + size_t inbuflen; + u_int8_t *outbufstart, *outbufend; + const u_int8_t *inbufstart; + unsigned int byte; + int decompose, precompose; + int result = 0; - 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0xE6, 0x63, + if (flags & ~(UTF_DECOMPOSED | UTF_PRECOMPOSED | UTF_NO_NULL_TERM | UTF_ESCAPE_ILLEGAL)) { + return (EINVAL); + } + decompose = (flags & UTF_DECOMPOSED); + precompose = (flags & UTF_PRECOMPOSED); + if ((decompose && precompose) || (!decompose && !precompose)) { + return (EINVAL); + } + outbufstart = outstr; + outbufend = outbufstart + buflen; + inbufstart = instr; + inbuflen = inlen; - 0x65, 0x65, 0x65, 0x65, 0x69, 0x69, 0x69, 0x69, + while (inlen-- > 0 && (byte = *instr++) != '\0') { + if (outstr >= outbufend) { + result = ENAMETOOLONG; + goto exit; + } + if (byte >= 0x80) { + goto nonASCII; + } + /* ASCII is already normalized. */ + *outstr++ = byte; + } +exit: + *outlen = outstr - outbufstart; + if (((flags & UTF_NO_NULL_TERM) == 0)) { + if (outstr < outbufend) + *outstr++ = '\0'; + else + result = ENAMETOOLONG; + } + return (result); - 0xF0, 0x6E, 0x6F, 0x6F, 0x6F, 0x6F, 0x6F, 0xF7, - 0xF8, 0x75, 0x75, 0x75, 0x75, 0x79, 0xFE, 0x79, -}; + /* + * Non-ASCII uses the existing utf8_encodestr/utf8_decodestr + * functions to perform the normalization. Since this will + * presumably be used to normalize filenames in the back-end + * (on disk or over-the-wire), it should be fast enough. + */ +nonASCII: -static unsigned char combining_char[64] = { - 0x00, 0x01, 0x02, 0x03, 0x08, 0x0A, 0xFF, 0x27, + /* Make sure the input size is reasonable. */ + if (inbuflen > MAXPATHLEN) { + result = ENAMETOOLONG; + goto exit; + } + /* + * Compute worst case Unicode buffer size. + * + * For pre-composed output, every UTF-8 input byte will be at + * most 2 Unicode bytes. For decomposed output, 2 UTF-8 bytes + * (smallest composite char sequence) may yield 6 Unicode bytes + * (1 base char + 2 combining chars). + */ + unicode_bytes = precompose ? (inbuflen * 2) : (inbuflen * 3); + + if (unicode_bytes <= sizeof(unicodebuf)) + unistr = &unicodebuf[0]; + else + MALLOC(unistr, uint16_t *, unicode_bytes, M_TEMP, M_WAITOK); + + /* Normalize the string. */ + result = utf8_decodestr(inbufstart, inbuflen, unistr, &unicode_bytes, + unicode_bytes, 0, flags & ~UTF_NO_NULL_TERM); + if (result == 0) { + /* Put results back into UTF-8. */ + result = utf8_encodestr(unistr, unicode_bytes, outbufstart, + &uft8_bytes, buflen, 0, UTF_NO_NULL_TERM); + outstr = outbufstart + uft8_bytes; + } + if (unistr && unistr != &unicodebuf[0]) { + FREE(unistr, M_TEMP); + } + goto exit; +} - 0x00, 0x01, 0x02, 0x08, 0x00, 0x01, 0x02, 0x08, - 0xFF, 0x03, 0x00, 0x01, 0x02, 0x03, 0x08, 0xFF, + /* + * Unicode 3.2 decomposition code (derived from Core Foundation) + */ - 0xFF, 0x00, 0x01, 0x02, 0x08, 0x01, 0xFF, 0xFF, +typedef struct { + u_int32_t _key; + u_int32_t _value; +} unicode_mappings32; - 0x00, 0x01, 0x02, 0x03, 0x08, 0x0A, 0xFF, 0x27, +static inline u_int32_t +getmappedvalue32(const unicode_mappings32 *theTable, u_int32_t numElem, + u_int16_t character) +{ + const unicode_mappings32 *p, *q, *divider; + + if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key)) + return (0); + + p = theTable; + q = p + (numElem-1); + while (p <= q) { + divider = p + ((q - p) >> 1); /* divide by 2 */ + if (character < divider->_key) { q = divider - 1; } + else if (character > divider->_key) { p = divider + 1; } + else { return (divider->_value); } + } + return (0); +} - 0x00, 0x01, 0x02, 0x08, 0x00, 0x01, 0x02, 0x08, +#define RECURSIVE_DECOMPOSITION (1 << 15) +#define EXTRACT_COUNT(value) (((value) >> 12) & 0x0007) - 0xFF, 0x03, 0x00, 0x01, 0x02, 0x03, 0x08, 0xFF, +typedef struct { + u_int16_t _key; + u_int16_t _value; +} unicode_mappings16; - 0xFF, 0x00, 0x01, 0x02, 0x08, 0x01, 0xFF, 0x08 -}; +static inline u_int16_t +getmappedvalue16(const unicode_mappings16 *theTable, u_int32_t numElem, + u_int16_t character) +{ + const unicode_mappings16 *p, *q, *divider; + + if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key)) + return (0); + + p = theTable; + q = p + (numElem-1); + while (p <= q) { + divider = p + ((q - p) >> 1); /* divide by 2 */ + if (character < divider->_key) + q = divider - 1; + else if (character > divider->_key) + p = divider + 1; + else + return (divider->_value); + } + return (0); +} -/* CJK codepoints 0x3000 ~ 0x30FF */ -static const unsigned long __CJKDecompBitmap[] = { - 0x00000000, 0x00000000, 0x000AAAAA, 0xA540DB6C, /* 0x3000 */ - 0x00000802, 0x000AAAAA, 0xA540DB6C, 0x000009E2, /* 0x3080 */ -}; -#define IS_DECOMPOSABLE(table,unicodeVal) \ - (table[(unicodeVal) / 32] & (1 << (31 - ((unicodeVal) % 32)))) +static u_int32_t +unicode_recursive_decompose(u_int16_t character, u_int16_t *convertedChars) +{ + u_int16_t value; + u_int32_t length; + u_int16_t firstChar; + u_int16_t theChar; + const u_int16_t *bmpMappings; + u_int32_t usedLength; + + value = getmappedvalue16( + (const unicode_mappings16 *)__CFUniCharDecompositionTable, + __UniCharDecompositionTableLength, character); + length = EXTRACT_COUNT(value); + firstChar = value & 0x0FFF; + theChar = firstChar; + bmpMappings = (length == 1 ? &theChar : __CFUniCharMultipleDecompositionTable + firstChar); + usedLength = 0; + + if (value & RECURSIVE_DECOMPOSITION) { + usedLength = unicode_recursive_decompose((u_int16_t)*bmpMappings, convertedChars); + + --length; /* Decrement for the first char */ + if (!usedLength) + return 0; + ++bmpMappings; + convertedChars += usedLength; + } + + usedLength += length; + + while (length--) + *(convertedChars++) = *(bmpMappings++); + + return (usedLength); +} + +#define HANGUL_SBASE 0xAC00 +#define HANGUL_LBASE 0x1100 +#define HANGUL_VBASE 0x1161 +#define HANGUL_TBASE 0x11A7 + +#define HANGUL_SCOUNT 11172 +#define HANGUL_LCOUNT 19 +#define HANGUL_VCOUNT 21 +#define HANGUL_TCOUNT 28 +#define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT) /* - * ucs_decompose - decompose a composed UCS-2 char + * unicode_decompose - decompose a composed Unicode char * * Composed Unicode characters are forbidden on * HFS Plus volumes. ucs_decompose will convert a * composed character into its correct decomposed * sequence. * - * Currently only MacRoman and MacJapanese chars - * are handled. Other composed characters are - * passed unchanged. + * Similar to CFUniCharDecomposeCharacter + */ +static int +unicode_decompose(u_int16_t character, u_int16_t *convertedChars) +{ + if ((character >= HANGUL_SBASE) && + (character <= (HANGUL_SBASE + HANGUL_SCOUNT))) { + u_int32_t length; + + character -= HANGUL_SBASE; + length = (character % HANGUL_TCOUNT ? 3 : 2); + + *(convertedChars++) = + character / HANGUL_NCOUNT + HANGUL_LBASE; + *(convertedChars++) = + (character % HANGUL_NCOUNT) / HANGUL_TCOUNT + HANGUL_VBASE; + if (length > 2) + *convertedChars = (character % HANGUL_TCOUNT) + HANGUL_TBASE; + return (length); + } else { + return (unicode_recursive_decompose(character, convertedChars)); + } +} + +/* + * unicode_combine - generate a precomposed Unicode char + * + * Precomposed Unicode characters are required for some volume + * formats and network protocols. unicode_combine will combine + * a decomposed character sequence into a single precomposed + * (composite) character. + * + * Similar toCFUniCharPrecomposeCharacter but unicode_combine + * also handles Hangul Jamo characters. */ static u_int16_t -ucs_decompose(register u_int16_t ch, u_int16_t *cmb) +unicode_combine(u_int16_t base, u_int16_t combining) { - u_int16_t base; + u_int32_t value; + + /* Check HANGUL */ + if ((combining >= HANGUL_VBASE) && (combining < (HANGUL_TBASE + HANGUL_TCOUNT))) { + /* 2 char Hangul sequences */ + if ((combining < (HANGUL_VBASE + HANGUL_VCOUNT)) && + (base >= HANGUL_LBASE && base < (HANGUL_LBASE + HANGUL_LCOUNT))) { + return (HANGUL_SBASE + + ((base - HANGUL_LBASE)*(HANGUL_VCOUNT*HANGUL_TCOUNT)) + + ((combining - HANGUL_VBASE)*HANGUL_TCOUNT)); + } - *cmb = 0; + /* 3 char Hangul sequences */ + if ((combining > HANGUL_TBASE) && + (base >= HANGUL_SBASE && base < (HANGUL_SBASE + HANGUL_SCOUNT))) { + if ((base - HANGUL_SBASE) % HANGUL_TCOUNT) + return (0); + else + return (base + (combining - HANGUL_TBASE)); + } + } - if ((ch <= 0x00FF) && (ch >= 0x00C0)) { - ch -= 0x00C0; - - base = (u_int16_t) primary_char[ch]; + value = getmappedvalue32( + (const unicode_mappings32 *)__CFUniCharPrecompSourceTable, + __CFUniCharPrecompositionTableLength, combining); - if (base <= 'z') { - *cmb = (u_int16_t)0x0300 + (u_int16_t)combining_char[ch]; - } - } else if ((ch > 0x3000) && (ch < 0x3100) && - IS_DECOMPOSABLE(__CJKDecompBitmap, ch - 0x3000)) { - - /* Handle HIRAGANA LETTERs */ - switch(ch) { - case 0x3071: base = 0x306F; *cmb = 0x309A; break; /* PA */ - case 0x3074: base = 0x3072; *cmb = 0x309A; break; /* PI */ - case 0x3077: base = 0x3075; *cmb = 0x309A; break; /* PU */ - case 0x307A: base = 0x3078; *cmb = 0x309A; break; /* PE */ - - case 0x307D: base = 0x307B; *cmb = 0x309A; break; /* PO */ - case 0x3094: base = 0x3046; *cmb = 0x3099; break; /* VU */ - case 0x30D1: base = 0x30CF; *cmb = 0x309A; break; /* PA */ - case 0x30D4: base = 0x30D2; *cmb = 0x309A; break; /* PI */ - - case 0x30D7: base = 0x30D5; *cmb = 0x309A; break; /* PU */ - case 0x30DA: base = 0x30D8; *cmb = 0x309A; break; /* PE */ - case 0x30DD: base = 0x30DB; *cmb = 0x309A; break; /* PO */ - case 0x30F4: base = 0x30A6; *cmb = 0x3099; break; /* VU */ - - case 0x30F7: base = 0x30EF; *cmb = 0x3099; break; /* VA */ - case 0x30F8: base = 0x30F0; *cmb = 0x3099; break; /* VI */ - case 0x30F9: base = 0x30F1; *cmb = 0x3099; break; /* VE */ - case 0x30FA: base = 0x30F2; *cmb = 0x3099; break; /* VO */ - - default: - /* the rest (41 of them) have a simple conversion */ - base = ch - 1; - *cmb = 0x3099; + if (value) { + value = getmappedvalue16( + (const unicode_mappings16 *) + ((const u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)), + (value >> 16), base); + } + return (value); +} + + +/* + * prioritysort - order combining chars into canonical order + * + * Similar to CFUniCharPrioritySort + */ +static void +prioritysort(u_int16_t* characters, int count) +{ + u_int32_t p1, p2; + u_int16_t *ch1, *ch2; + u_int16_t *end; + int changes = 0; + + end = characters + count; + do { + changes = 0; + ch1 = characters; + ch2 = characters + 1; + p2 = get_combining_class(*ch1); + while (ch2 < end) { + p1 = p2; + p2 = get_combining_class(*ch2); + if (p1 > p2 && p2 != 0) { + u_int32_t tmp; + + tmp = *ch1; + *ch1 = *ch2; + *ch2 = tmp; + changes = 1; + + /* + * Make sure that p2 contains the combining class for the + * character now stored at *ch2. This isn't required for + * correctness, but it will be more efficient if a character + * with a large combining class has to "bubble past" several + * characters with lower combining classes. + */ + p2 = p1; + } + ++ch1; + ++ch2; } - } else { - base = ch; + } while (changes); +} + + +/* + * Invalid NTFS filename characters are encodeded using the + * SFM (Services for Macintosh) private use Unicode characters. + * + * These should only be used for SMB, MSDOS or NTFS. + * + * Illegal NTFS Char SFM Unicode Char + * ---------------------------------------- + * 0x01-0x1f 0xf001-0xf01f + * '"' 0xf020 + * '*' 0xf021 + * '/' 0xf022 + * '<' 0xf023 + * '>' 0xf024 + * '?' 0xf025 + * '\' 0xf026 + * '|' 0xf027 + * ' ' 0xf028 (Only if last char of the name) + * '.' 0xf029 (Only if last char of the name) + * ---------------------------------------- + * + * Reference: http://support.microsoft.com/kb/q117258/ + */ + +#define MAX_SFM2MAC 0x29 +#define SFMCODE_PREFIX_MASK 0xf000 + +/* + * In the Mac OS 9 days the colon was illegal in a file name. For that reason + * SFM had no conversion for the colon. There is a conversion for the + * slash. In Mac OS X the slash is illegal in a file name. So for us the colon + * is a slash and a slash is a colon. So we can just replace the slash with the + * colon in our tables and everything will just work. + */ +static u_int8_t +sfm2mac[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 00 - 07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 08 - 0F */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 10 - 17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 18 - 1F */ + 0x22, 0x2a, 0x3a, 0x3c, 0x3e, 0x3f, 0x5c, 0x7c, /* 20 - 27 */ + 0x20, 0x2e /* 28 - 29 */ +}; +#define SFM2MAC_LEN ((sizeof(sfm2mac))/sizeof(sfm2mac[0])) + +static u_int8_t +mac2sfm[] = { + 0x20, 0x21, 0x20, 0x23, 0x24, 0x25, 0x26, 0x27, /* 20 - 27 */ + 0x28, 0x29, 0x21, 0x2b, 0x2c, 0x2d, 0x2e, 0x22, /* 28 - 2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 30 - 37 */ + 0x38, 0x39, 0x22, 0x3b, 0x23, 0x3d, 0x24, 0x25, /* 38 - 3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 40 - 47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 48 - 4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 50 - 57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x26, 0x5d, 0x5e, 0x5f, /* 58 - 5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 60 - 67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 68 - 6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 70 - 77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x27, 0x7d, 0x7e, 0x7f /* 78 - 7f */ +}; +#define MAC2SFM_LEN ((sizeof(mac2sfm))/sizeof(mac2sfm[0])) + + +/* + * Encode illegal NTFS filename characters into SFM Private Unicode characters + * + * Assumes non-zero ASCII input. + */ +static u_int16_t +ucs_to_sfm(u_int16_t ucs_ch, int lastchar) +{ + /* The last character of filename cannot be a space or period. */ + if (lastchar) { + if (ucs_ch == 0x20) + return (0xf028); + else if (ucs_ch == 0x2e) + return (0xf029); } - - return (base); + /* 0x01 - 0x1f is simple transformation. */ + if (ucs_ch <= 0x1f) { + return (ucs_ch | 0xf000); + } else /* 0x20 - 0x7f */ { + u_int16_t lsb; + + assert((ucs_ch - 0x0020) < MAC2SFM_LEN); + lsb = mac2sfm[ucs_ch - 0x0020]; + if (lsb != ucs_ch) + return(0xf000 | lsb); + } + return (ucs_ch); } +/* + * Decode any SFM Private Unicode characters + */ +static u_int16_t +sfm_to_ucs(u_int16_t ucs_ch) +{ + if (((ucs_ch & 0xffC0) == SFMCODE_PREFIX_MASK) && + ((ucs_ch & 0x003f) <= MAX_SFM2MAC)) { + assert((ucs_ch & 0x003f) < SFM2MAC_LEN); + ucs_ch = sfm2mac[ucs_ch & 0x003f]; + } + return (ucs_ch); +} + +