[apple/xnu.git] / bsd / vfs / vfs_utfconv.c

/*
 * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
 
 /*
 	Includes Unicode 3.2 decomposition code derived from Core Foundation
 */

#include <sys/param.h>
#include <sys/utfconv.h>
#include <sys/errno.h>
#include <architecture/byte_order.h>

/*
 * UTF-8 (Unicode Transformation Format)
 *
 * UTF-8 is the Unicode Transformation Format that serializes a Unicode
 * character as a sequence of one to four bytes. Only the shortest form
 * required to represent the significant Unicode bits is legal.
 * 
 * UTF-8 Multibyte Codes
 *
 * Bytes   Bits   Unicode Min  Unicode Max   UTF-8 Byte Sequence (binary)
 * -----------------------------------------------------------------------------
 *   1       7       0x0000        0x007F    0xxxxxxx
 *   2      11       0x0080        0x07FF    110xxxxx 10xxxxxx
 *   3      16       0x0800        0xFFFF    1110xxxx 10xxxxxx 10xxxxxx
 *   4      21      0x10000      0x10FFFF    11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
 * -----------------------------------------------------------------------------
 */


#define UNICODE_TO_UTF8_LEN(c)  \
	((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : (((c) & 0xf800) == 0xd800 ? 2 : 3)))

#define UCS_ALT_NULL	0x2400

/* Surrogate Pair Constants */
#define SP_HALF_SHIFT	10
#define SP_HALF_BASE	0x0010000UL
#define SP_HALF_MASK	0x3FFUL

#define SP_HIGH_FIRST	0xD800UL
#define SP_HIGH_LAST	0xDBFFUL
#define SP_LOW_FIRST	0xDC00UL
#define SP_LOW_LAST	0xDFFFUL


#include "vfs_utfconvdata.h"


/*
 * Test for a combining character.
 *
 * Similar to __CFUniCharIsNonBaseCharacter except that
 * unicode_combinable also includes Hangul Jamo characters.
 */
static inline int
unicode_combinable(u_int16_t character)
{
	const u_int8_t *bitmap = __CFUniCharCombiningBitmap;
	u_int8_t value;

	if (character < 0x0300)
		return (0);

	value = bitmap[(character >> 8) & 0xFF];

	if (value == 0xFF) {
		return (1);
	} else if (value) {
		bitmap = bitmap + ((value - 1) * 32) + 256;
		return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0);
	}
	return (0);
}

/*
 * Test for a precomposed character.
 *
 * Similar to __CFUniCharIsDecomposableCharacter.
 */
static inline int
unicode_decomposeable(u_int16_t character) {
	const u_int8_t *bitmap = __CFUniCharDecomposableBitmap;
	u_int8_t value;
	
	if (character < 0x00C0)
		return (0);

	value = bitmap[(character >> 8) & 0xFF];

	if (value == 0xFF) {
		return (1);
	} else if (value) {
		bitmap = bitmap + ((value - 1) * 32) + 256;
		return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0);
	}
    	return (0);
}

static int unicode_decompose(u_int16_t character, u_int16_t *convertedChars);

static u_int16_t unicode_combine(u_int16_t base, u_int16_t combining);


char utf_extrabytes[32] = {
	 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
	-1, -1, -1, -1, -1, -1, -1, -1,  1,  1,  1,  1,  2,  2,  3, -1
};


/*
 * utf8_encodelen - Calculates the UTF-8 encoding length for a Unicode filename
 *
 * NOTES:
 *    If '/' chars are allowed on disk then an alternate
 *    (replacement) char must be provided in altslash.
 *
 * input flags:
 *    UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime
 */
size_t
utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
               int flags)
{
	u_int16_t ucs_ch;
	int charcnt;
	int swapbytes = (flags & UTF_REVERSE_ENDIAN);
	size_t len;
	
	charcnt = ucslen / 2;
	len = 0;

	while (charcnt-- > 0) {
		ucs_ch = *ucsp++;

		if (swapbytes)
			ucs_ch = NXSwapShort(ucs_ch);
		if (ucs_ch == '/')
			ucs_ch = altslash ? altslash : '_';
		else if (ucs_ch == '\0')
			ucs_ch = UCS_ALT_NULL;
		
		len += UNICODE_TO_UTF8_LEN(ucs_ch);
	}

	return (len);
}


/*
 * utf8_encodestr - Encodes a Unicode string to UTF-8
 *
 * NOTES:
 *    The resulting UTF-8 string is NULL terminated.
 *
 *    If '/' chars are allowed on disk then an alternate
 *    (replacement) char must be provided in altslash.
 *
 * input flags:
 *    UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime
 *    UTF_NO_NULL_TERM:  don't add NULL termination to UTF-8 output
 *
 * result:
 *    ENAMETOOLONG: Name didn't fit; only buflen bytes were encoded
 *    EINVAL: Illegal char found; char was replaced by an '_'.
 */
int
utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
               size_t * utf8len, size_t buflen, u_int16_t altslash, int flags)
{
	u_int8_t * bufstart;
	u_int8_t * bufend;
	u_int16_t ucs_ch;
	u_int16_t * chp = NULL;
	u_int16_t sequence[8];
	int extra = 0;
	int charcnt;
	int swapbytes = (flags & UTF_REVERSE_ENDIAN);
	int nullterm  = ((flags & UTF_NO_NULL_TERM) == 0);
	int decompose = (flags & UTF_DECOMPOSED);
	int result = 0;
	
	bufstart = utf8p;
	bufend = bufstart + buflen;
	if (nullterm)
		--bufend;
	charcnt = ucslen / 2;

	while (charcnt-- > 0) {
		if (extra > 0) {
			--extra;
			ucs_ch = *chp++;
		} else {
			ucs_ch = swapbytes ? NXSwapShort(*ucsp++) : *ucsp++;

			if (decompose && unicode_decomposeable(ucs_ch)) {
				extra = unicode_decompose(ucs_ch, sequence) - 1;
				charcnt += extra;
				ucs_ch = sequence[0];
				chp = &sequence[1];
			}
		}

		/* Slash and NULL are not permitted */
		if (ucs_ch == '/') {
			if (altslash)
				ucs_ch = altslash;
			else {
				ucs_ch = '_';
				result = EINVAL;
			}
		} else if (ucs_ch == '\0') {
			ucs_ch = UCS_ALT_NULL;
		}

		if (ucs_ch < 0x0080) {
			if (utf8p >= bufend) {
				result = ENAMETOOLONG;
				break;
			}			
			*utf8p++ = ucs_ch;

		} else if (ucs_ch < 0x800) {
			if ((utf8p + 1) >= bufend) {
				result = ENAMETOOLONG;
				break;
			}
			*utf8p++ = 0xc0 | (ucs_ch >> 6);
			*utf8p++ = 0x80 | (0x3f & ucs_ch);

		} else {
			/* Combine valid surrogate pairs */
			if (ucs_ch >= SP_HIGH_FIRST && ucs_ch <= SP_HIGH_LAST
				&& charcnt > 0) {
				u_int16_t ch2;
				u_int32_t pair;

				ch2 = swapbytes ? NXSwapShort(*ucsp) : *ucsp;
				if (ch2 >= SP_LOW_FIRST && ch2 <= SP_LOW_LAST) {
					pair = ((ucs_ch - SP_HIGH_FIRST) << SP_HALF_SHIFT)
						+ (ch2 - SP_LOW_FIRST) + SP_HALF_BASE;
					if ((utf8p + 3) >= bufend) {
						result = ENAMETOOLONG;
						break;
					}
					--charcnt;
					++ucsp;				
					*utf8p++ = 0xf0 | (pair >> 18);
					*utf8p++ = 0x80 | (0x3f & (pair >> 12));
					*utf8p++ = 0x80 | (0x3f & (pair >> 6));
					*utf8p++ = 0x80 | (0x3f & pair);
					continue;
				}
			}
			if ((utf8p + 2) >= bufend) {
				result = ENAMETOOLONG;
				break;
			}
			*utf8p++ = 0xe0 | (ucs_ch >> 12);
			*utf8p++ = 0x80 | (0x3f & (ucs_ch >> 6));
			*utf8p++ = 0x80 | (0x3f & ucs_ch);
		}	
	}
	
	*utf8len = utf8p - bufstart;
	if (nullterm)
		*utf8p++ = '\0';

	return (result);
}


/*
 * utf8_decodestr - Decodes a UTF-8 string back to Unicode
 *
 * NOTES:
 *    The input UTF-8 string does not need to be null terminated
 *    if utf8len is set.
 *
 *    If '/' chars are allowed on disk then an alternate
 *    (replacement) char must be provided in altslash.
 *
 * input flags:
 *    UTF_REV_ENDIAN:   Unicode byteorder is oposite current runtime
 *    UTF_DECOMPOSED:   Unicode output string must be fully decompsed
 *
 * result:
 *    ENAMETOOLONG: Name didn't fit; only ucslen chars were decoded.
 *    EINVAL: Illegal UTF-8 sequence found.
 */
int
utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
               size_t *ucslen, size_t buflen, u_int16_t altslash, int flags)
{
	u_int16_t* bufstart;
	u_int16_t* bufend;
	u_int16_t ucs_ch;
	u_int8_t byte;
	int result = 0;
	int decompose, precompose, swapbytes;

	decompose =  (flags & UTF_DECOMPOSED);
	precompose = (flags & UTF_PRECOMPOSED);
	swapbytes =  (flags & UTF_REVERSE_ENDIAN);

	bufstart = ucsp;
	bufend = (u_int16_t *)((u_int8_t *)ucsp + buflen);

	while (utf8len-- > 0 && (byte = *utf8p++) != '\0') {
		if (ucsp >= bufend)
			goto toolong;

		/* check for ascii */
		if (byte < 0x80) {
			ucs_ch = byte;				/* 1st byte */
		} else {
			u_int32_t ch;
			int extrabytes = utf_extrabytes[byte >> 3];

			if (utf8len < extrabytes)
				goto invalid;
			utf8len -= extrabytes;

			switch (extrabytes) {
			case 1: ch = byte;			/* 1st byte */
					ch <<= 6;
			        ch += *utf8p++;		/* 2nd byte */
					ch -= 0x00003080UL;
					if (ch < 0x0080)
						goto invalid;
					ucs_ch = ch;
			        break;

			case 2:	ch = byte;			/* 1st byte */
					ch <<= 6;
					ch += *utf8p++;		/* 2nd byte */
					ch <<= 6;
					ch += *utf8p++;		/* 3rd byte */
					ch -= 0x000E2080UL;
					if (ch < 0x0800)
						goto invalid;
					ucs_ch = ch;
					break;

			case 3:	ch = byte;			/* 1st byte */
					ch <<= 6;
					ch += *utf8p++;		/* 2nd byte */
					ch <<= 6;
					ch += *utf8p++;		/* 3rd byte */
					ch <<= 6;
			        ch += *utf8p++;		/* 4th byte */
					ch -= 0x03C82080UL + SP_HALF_BASE;
					ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST;
					*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
					if (ucsp >= bufend)
						goto toolong;
					ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST;
					*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
			        continue;

			default:
					goto invalid;
			}
			if (decompose) {
				if (unicode_decomposeable(ucs_ch)) {
					u_int16_t sequence[8];
					int count, i;

					count = unicode_decompose(ucs_ch, sequence);

					for (i = 0; i < count; ++i) {
						ucs_ch = sequence[i];
						*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
						if (ucsp >= bufend)
							goto toolong;
					}
					continue;			
				}
			} else if (precompose && (ucsp != bufstart)) {
				u_int16_t composite, base;

				if (unicode_combinable(ucs_ch)) {
					base = swapbytes ? NXSwapShort(*(ucsp - 1)) : *(ucsp - 1);
					composite = unicode_combine(base, ucs_ch);
					if (composite) {
						--ucsp;
						ucs_ch = composite;
					}
				}
			}
			if (ucs_ch == UCS_ALT_NULL)
				ucs_ch = '\0';
		}
		if (ucs_ch == altslash)
			ucs_ch = '/';

		*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
	}

exit:
	*ucslen = (u_int8_t*)ucsp - (u_int8_t*)bufstart;

	return (result);

invalid:
	result = EINVAL;
	goto exit;

toolong:
	result = ENAMETOOLONG;
	goto exit;
}


 /*
  * Unicode 3.2 decomposition code (derived from Core Foundation)
  */

typedef struct {
	u_int32_t _key;
	u_int32_t _value;
} unicode_mappings32;

static inline u_int32_t
getmappedvalue32(const unicode_mappings32 *theTable, u_int32_t numElem,
		u_int16_t character)
{
	const unicode_mappings32 *p, *q, *divider;

	if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key))
		return (0);

	p = theTable;
	q = p + (numElem-1);
	while (p <= q) {
		divider = p + ((q - p) >> 1);	/* divide by 2 */
		if (character < divider->_key) { q = divider - 1; }
		else if (character > divider->_key) { p = divider + 1; }
		else { return (divider->_value); }
	}
	return (0);
}

#define RECURSIVE_DECOMPOSITION	(1 << 15)
#define EXTRACT_COUNT(value)	(((value) >> 12) & 0x0007)

typedef struct {
	u_int16_t _key;
	u_int16_t _value;
} unicode_mappings16;

static inline u_int16_t
getmappedvalue16(const unicode_mappings16 *theTable, u_int32_t numElem,
		u_int16_t character)
{
	const unicode_mappings16 *p, *q, *divider;

	if ((character < theTable[0]._key) || (character > theTable[numElem-1]._key))
		return (0);

	p = theTable;
	q = p + (numElem-1);
	while (p <= q) {
		divider = p + ((q - p) >> 1);	/* divide by 2 */
		if (character < divider->_key)
			q = divider - 1;
		else if (character > divider->_key)
			p = divider + 1;
		else
			return (divider->_value);
	}
	return (0);
}


static u_int32_t
unicode_recursive_decompose(u_int16_t character, u_int16_t *convertedChars)
{
	u_int16_t value;
	u_int32_t length;
	u_int16_t firstChar;
	u_int16_t theChar;
	const u_int16_t *bmpMappings;
	u_int32_t usedLength;

	value = getmappedvalue16(
		(const unicode_mappings16 *)__CFUniCharDecompositionTable,
		__UniCharDecompositionTableLength, character);
	length = EXTRACT_COUNT(value);
	firstChar = value & 0x0FFF;
	theChar = firstChar;
	bmpMappings = (length == 1 ? &theChar : __CFUniCharMultipleDecompositionTable + firstChar);
	usedLength = 0;

	if (value & RECURSIVE_DECOMPOSITION) {
	    usedLength = unicode_recursive_decompose((u_int16_t)*bmpMappings, convertedChars);
	
	    --length;	/* Decrement for the first char */
	    if (!usedLength)
	    	return 0;
	    ++bmpMappings;
	    convertedChars += usedLength;
	}
	
	usedLength += length;
	
	while (length--)
		*(convertedChars++) = *(bmpMappings++);
	
	return (usedLength);
}
    
#define HANGUL_SBASE 0xAC00
#define HANGUL_LBASE 0x1100
#define HANGUL_VBASE 0x1161
#define HANGUL_TBASE 0x11A7

#define HANGUL_SCOUNT 11172
#define HANGUL_LCOUNT 19
#define HANGUL_VCOUNT 21
#define HANGUL_TCOUNT 28
#define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT)

/*
 * unicode_decompose - decompose a composed Unicode char
 *
 * Composed Unicode characters are forbidden on
 * HFS Plus volumes. ucs_decompose will convert a
 * composed character into its correct decomposed
 * sequence.
 *
 * Similar to CFUniCharDecomposeCharacter
 */
static int
unicode_decompose(u_int16_t character, u_int16_t *convertedChars)
{
	if ((character >= HANGUL_SBASE) &&
	    (character <= (HANGUL_SBASE + HANGUL_SCOUNT))) {
		u_int32_t length;

		character -= HANGUL_SBASE;
		length = (character % HANGUL_TCOUNT ? 3 : 2);

		*(convertedChars++) =
			character / HANGUL_NCOUNT + HANGUL_LBASE;
		*(convertedChars++) =
			(character % HANGUL_NCOUNT) / HANGUL_TCOUNT + HANGUL_VBASE;
		if (length > 2)
			*convertedChars = (character % HANGUL_TCOUNT) + HANGUL_TBASE;
		return (length);
	} else {
		return (unicode_recursive_decompose(character, convertedChars));
	}
}

/*
 * unicode_combine - generate a precomposed Unicode char
 *
 * Precomposed Unicode characters are required for some volume
 * formats and network protocols.  unicode_combine will combine
 * a decomposed character sequence into a single precomposed
 * (composite) character.
 *
 * Similar toCFUniCharPrecomposeCharacter but unicode_combine
 * also handles Hangul Jamo characters.
 */
static u_int16_t
unicode_combine(u_int16_t base, u_int16_t combining)
{
	u_int32_t value;

	/* Check HANGUL */
	if ((combining >= HANGUL_VBASE) && (combining < (HANGUL_TBASE + HANGUL_TCOUNT))) {
		/* 2 char Hangul sequences */
		if ((combining < (HANGUL_VBASE + HANGUL_VCOUNT)) &&
		    (base >= HANGUL_LBASE && base < (HANGUL_LBASE + HANGUL_LCOUNT))) {
		    return (HANGUL_SBASE +
		            ((base - HANGUL_LBASE)*(HANGUL_VCOUNT*HANGUL_TCOUNT)) +
		            ((combining  - HANGUL_VBASE)*HANGUL_TCOUNT));
		}
	
		/* 3 char Hangul sequences */
		if ((combining > HANGUL_TBASE) &&
		    (base >= HANGUL_SBASE && base < (HANGUL_SBASE + HANGUL_SCOUNT))) {
			if ((base - HANGUL_SBASE) % HANGUL_TCOUNT)
				return (0);
			else
				return (base + (combining - HANGUL_TBASE));
		}
	}

	value = getmappedvalue32(
		(const unicode_mappings32 *)__CFUniCharPrecompSourceTable,
		__CFUniCharPrecompositionTableLength, combining);

	if (value) {
		value = getmappedvalue16(
			(const unicode_mappings16 *)
			((u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)),
			(value >> 16), base);
	}
	return (value);
}
Commit	Line	Data
1c79356b	1	/*
9bccf70c	2	* Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
1c79356b A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
43866e37	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b	7	*
43866e37 A	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
43866e37 A	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
1c79356b A	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
9bccf70c A	25
	26	/*
	27	Includes Unicode 3.2 decomposition code derived from Core Foundation
	28	*/
	29
1c79356b A	30	#include <sys/param.h>
	31	#include <sys/utfconv.h>
	32	#include <sys/errno.h>
	33	#include <architecture/byte_order.h>
	34
1c79356b	35	/*
765c9de3	36	* UTF-8 (Unicode Transformation Format)
1c79356b	37	*
765c9de3 A	38	* UTF-8 is the Unicode Transformation Format that serializes a Unicode
	39	* character as a sequence of one to four bytes. Only the shortest form
	40	* required to represent the significant Unicode bits is legal.
1c79356b A	41	*
	42	* UTF-8 Multibyte Codes
	43	*
765c9de3 A	44	* Bytes Bits Unicode Min Unicode Max UTF-8 Byte Sequence (binary)
	45	* -----------------------------------------------------------------------------
	46	* 1 7 0x0000 0x007F 0xxxxxxx
	47	* 2 11 0x0080 0x07FF 110xxxxx 10xxxxxx
	48	* 3 16 0x0800 0xFFFF 1110xxxx 10xxxxxx 10xxxxxx
	49	* 4 21 0x10000 0x10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
	50	* -----------------------------------------------------------------------------
1c79356b A	51	*/
	52
	53
765c9de3 A	54	#define UNICODE_TO_UTF8_LEN(c) \
765c9de3 A	55	((c) < 0x0080 ? 1 : ((c) < 0x0800 ? 2 : (((c) & 0xf800) == 0xd800 ? 2 : 3)))
0b4e3aa0 A	56
0b4e3aa0 A	57	#define UCS_ALT_NULL 0x2400
1c79356b	58
765c9de3 A	59	/* Surrogate Pair Constants */
	60	#define SP_HALF_SHIFT 10
	61	#define SP_HALF_BASE 0x0010000UL
	62	#define SP_HALF_MASK 0x3FFUL
	63
	64	#define SP_HIGH_FIRST 0xD800UL
	65	#define SP_HIGH_LAST 0xDBFFUL
	66	#define SP_LOW_FIRST 0xDC00UL
9bccf70c A	67	#define SP_LOW_LAST 0xDFFFUL
9bccf70c A	68
1c79356b	69
9bccf70c	70	#include "vfs_utfconvdata.h"
765c9de3	71
1c79356b	72
9bccf70c A	73	/*
	74	* Test for a combining character.
	75	*
	76	* Similar to __CFUniCharIsNonBaseCharacter except that
	77	* unicode_combinable also includes Hangul Jamo characters.
	78	*/
	79	static inline int
	80	unicode_combinable(u_int16_t character)
	81	{
	82	const u_int8_t *bitmap = __CFUniCharCombiningBitmap;
	83	u_int8_t value;
	84
	85	if (character < 0x0300)
	86	return (0);
	87
	88	value = bitmap[(character >> 8) & 0xFF];
	89
	90	if (value == 0xFF) {
	91	return (1);
	92	} else if (value) {
	93	bitmap = bitmap + ((value - 1) * 32) + 256;
	94	return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0);
	95	}
	96	return (0);
	97	}
	98
	99	/*
	100	* Test for a precomposed character.
	101	*
	102	* Similar to __CFUniCharIsDecomposableCharacter.
	103	*/
	104	static inline int
	105	unicode_decomposeable(u_int16_t character) {
	106	const u_int8_t *bitmap = __CFUniCharDecomposableBitmap;
	107	u_int8_t value;
	108
	109	if (character < 0x00C0)
	110	return (0);
	111
	112	value = bitmap[(character >> 8) & 0xFF];
	113
	114	if (value == 0xFF) {
	115	return (1);
	116	} else if (value) {
	117	bitmap = bitmap + ((value - 1) * 32) + 256;
	118	return (bitmap[(character & 0xFF) / 8] & (1 << (character % 8)) ? 1 : 0);
	119	}
	120	return (0);
	121	}
	122
	123	static int unicode_decompose(u_int16_t character, u_int16_t *convertedChars);
	124
	125	static u_int16_t unicode_combine(u_int16_t base, u_int16_t combining);
0b4e3aa0	126
1c79356b	127
765c9de3 A	128	char utf_extrabytes[32] = {
	129	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	130	-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 2, 2, 3, -1
	131	};
	132
	133
1c79356b	134	/*
765c9de3	135	* utf8_encodelen - Calculates the UTF-8 encoding length for a Unicode filename
1c79356b A	136	*
	137	* NOTES:
	138	* If '/' chars are allowed on disk then an alternate
	139	* (replacement) char must be provided in altslash.
	140	*
	141	* input flags:
765c9de3	142	* UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime
1c79356b A	143	*/
1c79356b A	144	size_t
765c9de3 A	145	utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
765c9de3 A	146	int flags)
1c79356b A	147	{
	148	u_int16_t ucs_ch;
	149	int charcnt;
	150	int swapbytes = (flags & UTF_REVERSE_ENDIAN);
	151	size_t len;
	152
	153	charcnt = ucslen / 2;
	154	len = 0;
	155
	156	while (charcnt-- > 0) {
	157	ucs_ch = *ucsp++;
	158
	159	if (swapbytes)
	160	ucs_ch = NXSwapShort(ucs_ch);
0b4e3aa0 A	161	if (ucs_ch == '/')
	162	ucs_ch = altslash ? altslash : '_';
	163	else if (ucs_ch == '\0')
	164	ucs_ch = UCS_ALT_NULL;
1c79356b	165
765c9de3	166	len += UNICODE_TO_UTF8_LEN(ucs_ch);
1c79356b A	167	}
	168
	169	return (len);
	170	}
	171
	172
	173	/*
765c9de3	174	* utf8_encodestr - Encodes a Unicode string to UTF-8
1c79356b A	175	*
1c79356b A	176	* NOTES:
0b4e3aa0	177	* The resulting UTF-8 string is NULL terminated.
1c79356b A	178	*
	179	* If '/' chars are allowed on disk then an alternate
	180	* (replacement) char must be provided in altslash.
	181	*
	182	* input flags:
765c9de3	183	* UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime
1c79356b	184	* UTF_NO_NULL_TERM: don't add NULL termination to UTF-8 output
0b4e3aa0 A	185	*
	186	* result:
	187	* ENAMETOOLONG: Name didn't fit; only buflen bytes were encoded
	188	* EINVAL: Illegal char found; char was replaced by an '_'.
1c79356b	189	*/
765c9de3 A	190	int
	191	utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
	192	size_t * utf8len, size_t buflen, u_int16_t altslash, int flags)
1c79356b A	193	{
	194	u_int8_t * bufstart;
	195	u_int8_t * bufend;
	196	u_int16_t ucs_ch;
9bccf70c A	197	u_int16_t * chp = NULL;
	198	u_int16_t sequence[8];
	199	int extra = 0;
1c79356b A	200	int charcnt;
1c79356b A	201	int swapbytes = (flags & UTF_REVERSE_ENDIAN);
0b4e3aa0 A	202	int nullterm = ((flags & UTF_NO_NULL_TERM) == 0);
0b4e3aa0 A	203	int decompose = (flags & UTF_DECOMPOSED);
1c79356b A	204	int result = 0;
	205
	206	bufstart = utf8p;
	207	bufend = bufstart + buflen;
	208	if (nullterm)
	209	--bufend;
	210	charcnt = ucslen / 2;
	211
	212	while (charcnt-- > 0) {
9bccf70c A	213	if (extra > 0) {
	214	--extra;
	215	ucs_ch = *chp++;
0b4e3aa0 A	216	} else {
0b4e3aa0 A	217	ucs_ch = swapbytes ? NXSwapShort(ucsp++) : ucsp++;
9bccf70c A	218
	219	if (decompose && unicode_decomposeable(ucs_ch)) {
	220	extra = unicode_decompose(ucs_ch, sequence) - 1;
	221	charcnt += extra;
	222	ucs_ch = sequence[0];
	223	chp = &sequence[1];
	224	}
0b4e3aa0	225	}
1c79356b	226
0b4e3aa0 A	227	/* Slash and NULL are not permitted */
	228	if (ucs_ch == '/') {
	229	if (altslash)
	230	ucs_ch = altslash;
	231	else {
	232	ucs_ch = '_';
	233	result = EINVAL;
	234	}
	235	} else if (ucs_ch == '\0') {
	236	ucs_ch = UCS_ALT_NULL;
	237	}
1c79356b	238
0b4e3aa0	239	if (ucs_ch < 0x0080) {
1c79356b A	240	if (utf8p >= bufend) {
	241	result = ENAMETOOLONG;
	242	break;
765c9de3	243	}
1c79356b A	244	*utf8p++ = ucs_ch;
	245
	246	} else if (ucs_ch < 0x800) {
	247	if ((utf8p + 1) >= bufend) {
	248	result = ENAMETOOLONG;
	249	break;
	250	}
765c9de3 A	251	*utf8p++ = 0xc0 \| (ucs_ch >> 6);
765c9de3 A	252	*utf8p++ = 0x80 \| (0x3f & ucs_ch);
1c79356b A	253
1c79356b A	254	} else {
765c9de3 A	255	/* Combine valid surrogate pairs */
	256	if (ucs_ch >= SP_HIGH_FIRST && ucs_ch <= SP_HIGH_LAST
	257	&& charcnt > 0) {
	258	u_int16_t ch2;
	259	u_int32_t pair;
	260
	261	ch2 = swapbytes ? NXSwapShort(ucsp) : ucsp;
	262	if (ch2 >= SP_LOW_FIRST && ch2 <= SP_LOW_LAST) {
	263	pair = ((ucs_ch - SP_HIGH_FIRST) << SP_HALF_SHIFT)
	264	+ (ch2 - SP_LOW_FIRST) + SP_HALF_BASE;
	265	if ((utf8p + 3) >= bufend) {
	266	result = ENAMETOOLONG;
	267	break;
	268	}
	269	--charcnt;
	270	++ucsp;
	271	*utf8p++ = 0xf0 \| (pair >> 18);
	272	*utf8p++ = 0x80 \| (0x3f & (pair >> 12));
	273	*utf8p++ = 0x80 \| (0x3f & (pair >> 6));
	274	*utf8p++ = 0x80 \| (0x3f & pair);
	275	continue;
	276	}
	277	}
1c79356b A	278	if ((utf8p + 2) >= bufend) {
	279	result = ENAMETOOLONG;
	280	break;
	281	}
765c9de3 A	282	*utf8p++ = 0xe0 \| (ucs_ch >> 12);
	283	*utf8p++ = 0x80 \| (0x3f & (ucs_ch >> 6));
	284	*utf8p++ = 0x80 \| (0x3f & ucs_ch);
1c79356b A	285	}
	286	}
	287
	288	*utf8len = utf8p - bufstart;
	289	if (nullterm)
	290	*utf8p++ = '\0';
	291
	292	return (result);
	293	}
	294
	295
	296	/*
765c9de3	297	* utf8_decodestr - Decodes a UTF-8 string back to Unicode
1c79356b A	298	*
	299	* NOTES:
	300	* The input UTF-8 string does not need to be null terminated
	301	* if utf8len is set.
	302	*
	303	* If '/' chars are allowed on disk then an alternate
	304	* (replacement) char must be provided in altslash.
	305	*
	306	* input flags:
765c9de3 A	307	* UTF_REV_ENDIAN: Unicode byteorder is oposite current runtime
765c9de3 A	308	* UTF_DECOMPOSED: Unicode output string must be fully decompsed
0b4e3aa0 A	309	*
	310	* result:
	311	* ENAMETOOLONG: Name didn't fit; only ucslen chars were decoded.
	312	* EINVAL: Illegal UTF-8 sequence found.
1c79356b A	313	*/
1c79356b A	314	int
765c9de3 A	315	utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
765c9de3 A	316	size_t *ucslen, size_t buflen, u_int16_t altslash, int flags)
1c79356b A	317	{
	318	u_int16_t* bufstart;
	319	u_int16_t* bufend;
	320	u_int16_t ucs_ch;
	321	u_int8_t byte;
	322	int result = 0;
0b4e3aa0	323	int decompose, precompose, swapbytes;
1c79356b	324
0b4e3aa0 A	325	decompose = (flags & UTF_DECOMPOSED);
	326	precompose = (flags & UTF_PRECOMPOSED);
	327	swapbytes = (flags & UTF_REVERSE_ENDIAN);
1c79356b A	328
	329	bufstart = ucsp;
	330	bufend = (u_int16_t )((u_int8_t )ucsp + buflen);
	331
	332	while (utf8len-- > 0 && (byte = *utf8p++) != '\0') {
765c9de3 A	333	if (ucsp >= bufend)
765c9de3 A	334	goto toolong;
1c79356b A	335
	336	/* check for ascii */
	337	if (byte < 0x80) {
765c9de3	338	ucs_ch = byte; /* 1st byte */
1c79356b	339	} else {
765c9de3 A	340	u_int32_t ch;
	341	int extrabytes = utf_extrabytes[byte >> 3];
	342
	343	if (utf8len < extrabytes)
	344	goto invalid;
	345	utf8len -= extrabytes;
	346
	347	switch (extrabytes) {
	348	case 1: ch = byte; /* 1st byte */
	349	ch <<= 6;
	350	ch += utf8p++; / 2nd byte */
	351	ch -= 0x00003080UL;
	352	if (ch < 0x0080)
	353	goto invalid;
	354	ucs_ch = ch;
	355	break;
	356
	357	case 2: ch = byte; /* 1st byte */
	358	ch <<= 6;
	359	ch += utf8p++; / 2nd byte */
	360	ch <<= 6;
	361	ch += utf8p++; / 3rd byte */
	362	ch -= 0x000E2080UL;
	363	if (ch < 0x0800)
	364	goto invalid;
	365	ucs_ch = ch;
	366	break;
	367
	368	case 3: ch = byte; /* 1st byte */
	369	ch <<= 6;
	370	ch += utf8p++; / 2nd byte */
	371	ch <<= 6;
	372	ch += utf8p++; / 3rd byte */
	373	ch <<= 6;
	374	ch += utf8p++; / 4th byte */
	375	ch -= 0x03C82080UL + SP_HALF_BASE;
	376	ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST;
	377	*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
	378	if (ucsp >= bufend)
	379	goto toolong;
	380	ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST;
	381	*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
	382	continue;
1c79356b	383
1c79356b	384	default:
765c9de3	385	goto invalid;
1c79356b	386	}
1c79356b	387	if (decompose) {
9bccf70c A	388	if (unicode_decomposeable(ucs_ch)) {
	389	u_int16_t sequence[8];
	390	int count, i;
1c79356b	391
9bccf70c	392	count = unicode_decompose(ucs_ch, sequence);
1c79356b	393
9bccf70c A	394	for (i = 0; i < count; ++i) {
9bccf70c A	395	ucs_ch = sequence[i];
0b4e3aa0	396	*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
765c9de3 A	397	if (ucsp >= bufend)
765c9de3 A	398	goto toolong;
0b4e3aa0	399	}
9bccf70c	400	continue;
0b4e3aa0 A	401	}
	402	} else if (precompose && (ucsp != bufstart)) {
	403	u_int16_t composite, base;
	404
9bccf70c A	405	if (unicode_combinable(ucs_ch)) {
	406	base = swapbytes ? NXSwapShort((ucsp - 1)) : (ucsp - 1);
	407	composite = unicode_combine(base, ucs_ch);
	408	if (composite) {
	409	--ucsp;
	410	ucs_ch = composite;
	411	}
1c79356b A	412	}
1c79356b A	413	}
0b4e3aa0 A	414	if (ucs_ch == UCS_ALT_NULL)
0b4e3aa0 A	415	ucs_ch = '\0';
1c79356b	416	}
1c79356b A	417	if (ucs_ch == altslash)
1c79356b A	418	ucs_ch = '/';
1c79356b	419
765c9de3	420	*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
1c79356b	421	}
765c9de3 A	422
765c9de3 A	423	exit:
1c79356b A	424	ucslen = (u_int8_t)ucsp - (u_int8_t*)bufstart;
	425
	426	return (result);
765c9de3 A	427
	428	invalid:
	429	result = EINVAL;
	430	goto exit;
	431
	432	toolong:
	433	result = ENAMETOOLONG;
	434	goto exit;
1c79356b A	435	}
	436
	437
9bccf70c A	438	/*
	439	* Unicode 3.2 decomposition code (derived from Core Foundation)
	440	*/
1c79356b	441
9bccf70c A	442	typedef struct {
	443	u_int32_t _key;
	444	u_int32_t _value;
	445	} unicode_mappings32;
0b4e3aa0	446
9bccf70c A	447	static inline u_int32_t
	448	getmappedvalue32(const unicode_mappings32 *theTable, u_int32_t numElem,
	449	u_int16_t character)
	450	{
	451	const unicode_mappings32 p, q, *divider;
1c79356b	452
9bccf70c A	453	if ((character < theTable[0]._key) \|\| (character > theTable[numElem-1]._key))
9bccf70c A	454	return (0);
1c79356b	455
9bccf70c A	456	p = theTable;
	457	q = p + (numElem-1);
	458	while (p <= q) {
	459	divider = p + ((q - p) >> 1); /* divide by 2 */
	460	if (character < divider->_key) { q = divider - 1; }
	461	else if (character > divider->_key) { p = divider + 1; }
	462	else { return (divider->_value); }
	463	}
	464	return (0);
	465	}
1c79356b	466
9bccf70c A	467	#define RECURSIVE_DECOMPOSITION (1 << 15)
9bccf70c A	468	#define EXTRACT_COUNT(value) (((value) >> 12) & 0x0007)
1c79356b	469
9bccf70c A	470	typedef struct {
	471	u_int16_t _key;
	472	u_int16_t _value;
	473	} unicode_mappings16;
1c79356b	474
9bccf70c A	475	static inline u_int16_t
	476	getmappedvalue16(const unicode_mappings16 *theTable, u_int32_t numElem,
	477	u_int16_t character)
	478	{
	479	const unicode_mappings16 p, q, *divider;
1c79356b	480
9bccf70c A	481	if ((character < theTable[0]._key) \|\| (character > theTable[numElem-1]._key))
9bccf70c A	482	return (0);
1c79356b	483
9bccf70c A	484	p = theTable;
	485	q = p + (numElem-1);
	486	while (p <= q) {
	487	divider = p + ((q - p) >> 1); /* divide by 2 */
	488	if (character < divider->_key)
	489	q = divider - 1;
	490	else if (character > divider->_key)
	491	p = divider + 1;
	492	else
	493	return (divider->_value);
	494	}
	495	return (0);
	496	}
	497
	498
	499	static u_int32_t
	500	unicode_recursive_decompose(u_int16_t character, u_int16_t *convertedChars)
	501	{
	502	u_int16_t value;
	503	u_int32_t length;
	504	u_int16_t firstChar;
	505	u_int16_t theChar;
	506	const u_int16_t *bmpMappings;
	507	u_int32_t usedLength;
	508
	509	value = getmappedvalue16(
	510	(const unicode_mappings16 *)__CFUniCharDecompositionTable,
	511	__UniCharDecompositionTableLength, character);
	512	length = EXTRACT_COUNT(value);
	513	firstChar = value & 0x0FFF;
	514	theChar = firstChar;
	515	bmpMappings = (length == 1 ? &theChar : __CFUniCharMultipleDecompositionTable + firstChar);
	516	usedLength = 0;
	517
	518	if (value & RECURSIVE_DECOMPOSITION) {
	519	usedLength = unicode_recursive_decompose((u_int16_t)*bmpMappings, convertedChars);
	520
	521	--length; /* Decrement for the first char */
	522	if (!usedLength)
	523	return 0;
	524	++bmpMappings;
	525	convertedChars += usedLength;
	526	}
0b4e3aa0	527
9bccf70c	528	usedLength += length;
0b4e3aa0	529
9bccf70c A	530	while (length--)
9bccf70c A	531	(convertedChars++) = (bmpMappings++);
0b4e3aa0	532
9bccf70c A	533	return (usedLength);
	534	}
	535
	536	#define HANGUL_SBASE 0xAC00
	537	#define HANGUL_LBASE 0x1100
	538	#define HANGUL_VBASE 0x1161
	539	#define HANGUL_TBASE 0x11A7
	540
	541	#define HANGUL_SCOUNT 11172
	542	#define HANGUL_LCOUNT 19
	543	#define HANGUL_VCOUNT 21
	544	#define HANGUL_TCOUNT 28
	545	#define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT)
1c79356b A	546
1c79356b A	547	/*
9bccf70c	548	* unicode_decompose - decompose a composed Unicode char
1c79356b A	549	*
	550	* Composed Unicode characters are forbidden on
	551	* HFS Plus volumes. ucs_decompose will convert a
	552	* composed character into its correct decomposed
	553	* sequence.
	554	*
9bccf70c	555	* Similar to CFUniCharDecomposeCharacter
1c79356b	556	*/
9bccf70c A	557	static int
9bccf70c A	558	unicode_decompose(u_int16_t character, u_int16_t *convertedChars)
1c79356b	559	{
9bccf70c A	560	if ((character >= HANGUL_SBASE) &&
	561	(character <= (HANGUL_SBASE + HANGUL_SCOUNT))) {
	562	u_int32_t length;
	563
	564	character -= HANGUL_SBASE;
	565	length = (character % HANGUL_TCOUNT ? 3 : 2);
	566
	567	*(convertedChars++) =
	568	character / HANGUL_NCOUNT + HANGUL_LBASE;
	569	*(convertedChars++) =
	570	(character % HANGUL_NCOUNT) / HANGUL_TCOUNT + HANGUL_VBASE;
	571	if (length > 2)
	572	*convertedChars = (character % HANGUL_TCOUNT) + HANGUL_TBASE;
	573	return (length);
1c79356b	574	} else {
9bccf70c	575	return (unicode_recursive_decompose(character, convertedChars));
1c79356b	576	}
1c79356b A	577	}
1c79356b A	578
0b4e3aa0	579	/*
9bccf70c	580	* unicode_combine - generate a precomposed Unicode char
0b4e3aa0 A	581	*
0b4e3aa0 A	582	* Precomposed Unicode characters are required for some volume
9bccf70c A	583	* formats and network protocols. unicode_combine will combine
9bccf70c A	584	* a decomposed character sequence into a single precomposed
0b4e3aa0 A	585	* (composite) character.
0b4e3aa0 A	586	*
9bccf70c A	587	* Similar toCFUniCharPrecomposeCharacter but unicode_combine
9bccf70c A	588	* also handles Hangul Jamo characters.
0b4e3aa0 A	589	*/
0b4e3aa0 A	590	static u_int16_t
9bccf70c	591	unicode_combine(u_int16_t base, u_int16_t combining)
0b4e3aa0	592	{
9bccf70c A	593	u_int32_t value;
	594
	595	/* Check HANGUL */
	596	if ((combining >= HANGUL_VBASE) && (combining < (HANGUL_TBASE + HANGUL_TCOUNT))) {
	597	/* 2 char Hangul sequences */
	598	if ((combining < (HANGUL_VBASE + HANGUL_VCOUNT)) &&
	599	(base >= HANGUL_LBASE && base < (HANGUL_LBASE + HANGUL_LCOUNT))) {
	600	return (HANGUL_SBASE +
	601	((base - HANGUL_LBASE)(HANGUL_VCOUNTHANGUL_TCOUNT)) +
	602	((combining - HANGUL_VBASE)*HANGUL_TCOUNT));
0b4e3aa0	603	}
9bccf70c A	604
	605	/* 3 char Hangul sequences */
	606	if ((combining > HANGUL_TBASE) &&
	607	(base >= HANGUL_SBASE && base < (HANGUL_SBASE + HANGUL_SCOUNT))) {
	608	if ((base - HANGUL_SBASE) % HANGUL_TCOUNT)
	609	return (0);
	610	else
	611	return (base + (combining - HANGUL_TBASE));
0b4e3aa0	612	}
0b4e3aa0 A	613	}
0b4e3aa0 A	614
9bccf70c A	615	value = getmappedvalue32(
	616	(const unicode_mappings32 *)__CFUniCharPrecompSourceTable,
	617	__CFUniCharPrecompositionTableLength, combining);
0b4e3aa0	618
9bccf70c A	619	if (value) {
	620	value = getmappedvalue16(
	621	(const unicode_mappings16 *)
	622	((u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)),
	623	(value >> 16), base);
0b4e3aa0	624	}
9bccf70c	625	return (value);
0b4e3aa0 A	626	}
0b4e3aa0 A	627