X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d7e50217d7adf6e52786a38bcaa4cd698cb9a79e..13f56ec4e58bf8687e2a68032c093c0213dd519b:/bsd/kern/kern_subr.c?ds=sidebyside diff --git a/bsd/kern/kern_subr.c b/bsd/kern/kern_subr.c index 7e94acb9c..cc05a7db7 100644 --- a/bsd/kern/kern_subr.c +++ b/bsd/kern/kern_subr.c @@ -1,16 +1,19 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ - * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -20,7 +23,7 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ /* @@ -65,10 +68,12 @@ #include #include -#include +#include #include #include #include +#include +#include #include @@ -76,109 +81,187 @@ #define DBG_UIO_COPYOUT 16 #define DBG_UIO_COPYIN 17 +#if DEBUG +#include + +static uint32_t uio_t_count = 0; +#endif /* DEBUG */ + +#define IS_VALID_UIO_SEGFLG(segflg) \ + ( (segflg) == UIO_USERSPACE || \ + (segflg) == UIO_SYSSPACE || \ + (segflg) == UIO_USERSPACE32 || \ + (segflg) == UIO_USERSPACE64 || \ + (segflg) == UIO_SYSSPACE32 || \ + (segflg) == UIO_USERISPACE || \ + (segflg) == UIO_PHYS_USERSPACE || \ + (segflg) == UIO_PHYS_SYSSPACE || \ + (segflg) == UIO_USERISPACE32 || \ + (segflg) == UIO_PHYS_USERSPACE32 || \ + (segflg) == UIO_USERISPACE64 || \ + (segflg) == UIO_PHYS_USERSPACE64 ) + +/* + * Returns: 0 Success + * uiomove64:EFAULT + * + * Notes: The first argument should be a caddr_t, but const poisoning + * for typedef'ed types doesn't work in gcc. + */ int -uiomove(cp, n, uio) - register caddr_t cp; - register int n; - register struct uio *uio; +uiomove(const char * cp, int n, uio_t uio) { - return uiomove64((addr64_t)((unsigned int)cp), n, uio); + return uiomove64((const addr64_t)(uintptr_t)cp, n, uio); } +/* + * Returns: 0 Success + * EFAULT + * copyout:EFAULT + * copyin:EFAULT + * copywithin:EFAULT + * copypv:EFAULT + */ int -uiomove64(addr64_t cp, int n, struct uio *uio) +uiomove64(const addr64_t c_cp, int n, struct uio *uio) { - register struct iovec *iov; - u_int cnt; + addr64_t cp = c_cp; +#if LP64KERN + uint64_t acnt; +#else + u_int acnt; +#endif int error = 0; #if DIAGNOSTIC if (uio->uio_rw != UIO_READ && uio->uio_rw != UIO_WRITE) panic("uiomove: mode"); - if (uio->uio_segflg == UIO_USERSPACE && uio->uio_procp != current_proc()) - panic("uiomove proc"); #endif - while (n > 0 && uio->uio_resid) { - iov = uio->uio_iov; - cnt = iov->iov_len; - if (cnt == 0) { - uio->uio_iov++; - uio->uio_iovcnt--; +#if LP64_DEBUG + if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) { + panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + while (n > 0 && uio_resid(uio)) { + uio_update(uio, 0); + acnt = uio_curriovlen(uio); + if (acnt == 0) { continue; } - if (cnt > n) - cnt = n; + if (n > 0 && acnt > (uint64_t)n) + acnt = n; + switch (uio->uio_segflg) { + case UIO_USERSPACE64: + case UIO_USERISPACE64: + case UIO_USERSPACE32: + case UIO_USERISPACE32: case UIO_USERSPACE: case UIO_USERISPACE: + // LP64 - 3rd argument in debug code is 64 bit, expected to be 32 bit if (uio->uio_rw == UIO_READ) { KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START, - (caddr_t)cp, iov->iov_base, cnt, 0,0); + (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 0,0); - error = copyout((caddr_t)cp, iov->iov_base, cnt); + error = copyout( CAST_DOWN(caddr_t, cp), uio->uio_iovs.uiovp->iov_base, acnt ); KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END, - (caddr_t)cp, iov->iov_base, cnt, 0,0); + (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 0,0); } else { KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START, - iov->iov_base, (caddr_t)cp, cnt, 0,0); + (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 0,0); - error = copyin(iov->iov_base, (caddr_t)cp, cnt); + error = copyin(uio->uio_iovs.uiovp->iov_base, CAST_DOWN(caddr_t, cp), acnt); KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END, - iov->iov_base, (caddr_t)cp, cnt, 0,0); + (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 0,0); } if (error) return (error); break; + case UIO_SYSSPACE32: case UIO_SYSSPACE: if (uio->uio_rw == UIO_READ) - error = copywithin((caddr_t)cp, iov->iov_base, - cnt); + error = copywithin(CAST_DOWN(caddr_t, cp), CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base), + acnt); else - error = copywithin(iov->iov_base, (caddr_t)cp, - cnt); + error = copywithin(CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base), CAST_DOWN(caddr_t, cp), + acnt); break; + case UIO_PHYS_USERSPACE64: + case UIO_PHYS_USERSPACE32: case UIO_PHYS_USERSPACE: if (uio->uio_rw == UIO_READ) { KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START, - cp, iov->iov_base, cnt, 1,0); + (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 1,0); + + error = copypv((addr64_t)cp, uio->uio_iovs.uiovp->iov_base, acnt, cppvPsrc | cppvNoRefSrc); + if (error) /* Copy physical to virtual */ + error = EFAULT; + + KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END, + (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 1,0); + } + else + { + KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START, + (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 1,0); + + error = copypv(uio->uio_iovs.uiovp->iov_base, (addr64_t)cp, acnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk); + if (error) /* Copy virtual to physical */ + error = EFAULT; + + KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END, + (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 1,0); + } + if (error) + return (error); + break; + + case UIO_PHYS_SYSSPACE: + if (uio->uio_rw == UIO_READ) + { + KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START, + (int)cp, (uintptr_t)uio->uio_iovs.kiovp->iov_base, acnt, 2,0); - if (error = copypv((addr64_t)cp, (addr64_t)((unsigned int)iov->iov_base), cnt, cppvPsrc | cppvNoRefSrc)) /* Copy physical to virtual */ + error = copypv((addr64_t)cp, uio->uio_iovs.kiovp->iov_base, acnt, cppvKmap | cppvPsrc | cppvNoRefSrc); + if (error) /* Copy physical to virtual */ error = EFAULT; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END, - cp, iov->iov_base, cnt, 1,0); + (int)cp, (uintptr_t)uio->uio_iovs.kiovp->iov_base, acnt, 2,0); } else { KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START, - iov->iov_base, cp, cnt, 1,0); + (uintptr_t)uio->uio_iovs.kiovp->iov_base, (int)cp, acnt, 2,0); - if (error = copypv((addr64_t)((unsigned int)iov->iov_base), (addr64_t)cp, cnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk)) /* Copy virtual to physical */ + error = copypv(uio->uio_iovs.kiovp->iov_base, (addr64_t)cp, acnt, cppvKmap | cppvPsnk | cppvNoRefSrc | cppvNoModSnk); + if (error) /* Copy virtual to physical */ error = EFAULT; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END, - iov->iov_base, cp, cnt, 1,0); + (uintptr_t)uio->uio_iovs.kiovp->iov_base, (int)cp, acnt, 2,0); } if (error) return (error); break; + + default: + break; } - iov->iov_base += cnt; - iov->iov_len -= cnt; - uio->uio_resid -= cnt; - uio->uio_offset += cnt; - cp += cnt; - n -= cnt; + uio_update(uio, acnt); + cp += acnt; + n -= acnt; } return (error); } @@ -187,120 +270,906 @@ uiomove64(addr64_t cp, int n, struct uio *uio) * Give next character to user as result of read. */ int -ureadc(c, uio) - register int c; - register struct uio *uio; +ureadc(int c, struct uio *uio) { - register struct iovec *iov; - - if (uio->uio_resid <= 0) + if (uio_resid(uio) <= 0) panic("ureadc: non-positive resid"); -again: + uio_update(uio, 0); if (uio->uio_iovcnt == 0) panic("ureadc: non-positive iovcnt"); - iov = uio->uio_iov; - if (iov->iov_len <= 0) { - uio->uio_iovcnt--; - uio->uio_iov++; - goto again; - } + if (uio_curriovlen(uio) <= 0) + panic("ureadc: non-positive iovlen"); + switch (uio->uio_segflg) { + case UIO_USERSPACE32: case UIO_USERSPACE: - if (subyte(iov->iov_base, c) < 0) + case UIO_USERISPACE32: + case UIO_USERISPACE: + case UIO_USERSPACE64: + case UIO_USERISPACE64: + if (subyte((user_addr_t)uio->uio_iovs.uiovp->iov_base, c) < 0) return (EFAULT); break; + case UIO_SYSSPACE32: case UIO_SYSSPACE: - *iov->iov_base = c; + *(CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base)) = c; break; - case UIO_USERISPACE: - if (suibyte(iov->iov_base, c) < 0) - return (EFAULT); + default: break; } - iov->iov_base++; - iov->iov_len--; - uio->uio_resid--; - uio->uio_offset++; + uio_update(uio, 1); return (0); } -#if defined(vax) || defined(ppc) -/* unused except by ct.c, other oddities XXX */ /* - * Get next character written in by user from uio. + * General routine to allocate a hash table. */ -uwritec(uio) - struct uio *uio; +void * +hashinit(int elements, int type, u_long *hashmask) { - register struct iovec *iov; - register int c; - - if (uio->uio_resid <= 0) - return (-1); -again: - if (uio->uio_iovcnt <= 0) - panic("uwritec: non-positive iovcnt"); - iov = uio->uio_iov; - if (iov->iov_len == 0) { - uio->uio_iov++; - if (--uio->uio_iovcnt == 0) - return (-1); - goto again; + long hashsize; + LIST_HEAD(generic, generic) *hashtbl; + int i; + + if (elements <= 0) + panic("hashinit: bad cnt"); + for (hashsize = 1; hashsize <= elements; hashsize <<= 1) + continue; + hashsize >>= 1; + MALLOC(hashtbl, struct generic *, + hashsize * sizeof(*hashtbl), type, M_WAITOK|M_ZERO); + if (hashtbl != NULL) { + for (i = 0; i < hashsize; i++) + LIST_INIT(&hashtbl[i]); + *hashmask = hashsize - 1; } - switch (uio->uio_segflg) { + return (hashtbl); +} + +/* + * uio_resid - return the residual IO value for the given uio_t + */ +user_ssize_t uio_resid( uio_t a_uio ) +{ +#if DEBUG + if (a_uio == NULL) { + printf("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +/* if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */ +/* panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); */ +/* } */ +#endif /* DEBUG */ + /* return 0 if there are no active iovecs */ + if (a_uio == NULL) { + return( 0 ); + } + + return( a_uio->uio_resid_64 ); +} + +/* + * uio_setresid - set the residual IO value for the given uio_t + */ +void uio_setresid( uio_t a_uio, user_ssize_t a_value ) +{ +#if DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +/* if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */ +/* panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); */ +/* } */ +#endif /* DEBUG */ + + if (a_uio == NULL) { + return; + } + + a_uio->uio_resid_64 = a_value; + return; +} + +/* + * uio_curriovbase - return the base address of the current iovec associated + * with the given uio_t. May return 0. + */ +user_addr_t uio_curriovbase( uio_t a_uio ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL || a_uio->uio_iovcnt < 1) { + return(0); + } + + if (UIO_IS_USER_SPACE(a_uio)) { + return(a_uio->uio_iovs.uiovp->iov_base); + } + return((user_addr_t)a_uio->uio_iovs.kiovp->iov_base); + +} + +/* + * uio_curriovlen - return the length value of the current iovec associated + * with the given uio_t. + */ +user_size_t uio_curriovlen( uio_t a_uio ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL || a_uio->uio_iovcnt < 1) { + return(0); + } + + if (UIO_IS_USER_SPACE(a_uio)) { + return(a_uio->uio_iovs.uiovp->iov_len); + } + return((user_size_t)a_uio->uio_iovs.kiovp->iov_len); +} + +/* + * uio_setcurriovlen - set the length value of the current iovec associated + * with the given uio_t. + */ +__private_extern__ void uio_setcurriovlen( uio_t a_uio, user_size_t a_value ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL) { + return; + } + + if (UIO_IS_USER_SPACE(a_uio)) { + a_uio->uio_iovs.uiovp->iov_len = a_value; + } + else { +#if LP64_DEBUG + if (a_value > 0xFFFFFFFFull) { + panic("%s :%d - invalid a_value\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + a_uio->uio_iovs.kiovp->iov_len = (size_t)a_value; + } + return; +} + +/* + * uio_iovcnt - return count of active iovecs for the given uio_t + */ +int uio_iovcnt( uio_t a_uio ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL) { + return(0); + } + + return( a_uio->uio_iovcnt ); +} + +/* + * uio_offset - return the current offset value for the given uio_t + */ +off_t uio_offset( uio_t a_uio ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL) { + return(0); + } + return( a_uio->uio_offset ); +} + +/* + * uio_setoffset - set the current offset value for the given uio_t + */ +void uio_setoffset( uio_t a_uio, off_t a_offset ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL) { + return; + } + a_uio->uio_offset = a_offset; + return; +} + +/* + * uio_rw - return the read / write flag for the given uio_t + */ +int uio_rw( uio_t a_uio ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL) { + return(-1); + } + return( a_uio->uio_rw ); +} + +/* + * uio_setrw - set the read / write flag for the given uio_t + */ +void uio_setrw( uio_t a_uio, int a_value ) +{ + if (a_uio == NULL) { +#if LP64_DEBUG + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); +#endif /* LP64_DEBUG */ + return; + } + +#if LP64_DEBUG + if (!(a_value == UIO_READ || a_value == UIO_WRITE)) { + panic("%s :%d - invalid a_value\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_value == UIO_READ || a_value == UIO_WRITE) { + a_uio->uio_rw = a_value; + } + return; +} + +/* + * uio_isuserspace - return non zero value if the address space + * flag is for a user address space (could be 32 or 64 bit). + */ +int uio_isuserspace( uio_t a_uio ) +{ + if (a_uio == NULL) { +#if LP64_DEBUG + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); +#endif /* LP64_DEBUG */ + return(0); + } + + if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) { + return( 1 ); + } + return( 0 ); +} + + +/* + * uio_create - create an uio_t. + * Space is allocated to hold up to a_iovcount number of iovecs. The uio_t + * is not fully initialized until all iovecs are added using uio_addiov calls. + * a_iovcount is the maximum number of iovecs you may add. + */ +uio_t uio_create( int a_iovcount, /* number of iovecs */ + off_t a_offset, /* current offset */ + int a_spacetype, /* type of address space */ + int a_iodirection ) /* read or write flag */ +{ + void * my_buf_p; + size_t my_size; + uio_t my_uio; + + my_size = UIO_SIZEOF(a_iovcount); + my_buf_p = kalloc(my_size); + my_uio = uio_createwithbuffer( a_iovcount, + a_offset, + a_spacetype, + a_iodirection, + my_buf_p, + my_size ); + if (my_uio != 0) { + /* leave a note that we allocated this uio_t */ + my_uio->uio_flags |= UIO_FLAGS_WE_ALLOCED; +#if DEBUG + (void)hw_atomic_add(&uio_t_count, 1); +#endif + } + + return( my_uio ); +} + + +/* + * uio_createwithbuffer - create an uio_t. + * Create a uio_t using the given buffer. The uio_t + * is not fully initialized until all iovecs are added using uio_addiov calls. + * a_iovcount is the maximum number of iovecs you may add. + * This call may fail if the given buffer is not large enough. + */ +__private_extern__ uio_t + uio_createwithbuffer( int a_iovcount, /* number of iovecs */ + off_t a_offset, /* current offset */ + int a_spacetype, /* type of address space */ + int a_iodirection, /* read or write flag */ + void *a_buf_p, /* pointer to a uio_t buffer */ + size_t a_buffer_size ) /* size of uio_t buffer */ +{ + uio_t my_uio = (uio_t) a_buf_p; + size_t my_size; + + my_size = UIO_SIZEOF(a_iovcount); + if (a_buffer_size < my_size) { +#if DEBUG + panic("%s :%d - a_buffer_size is too small\n", __FILE__, __LINE__); +#endif /* DEBUG */ + return( NULL ); + } + my_size = a_buffer_size; + +#if DEBUG + if (my_uio == 0) { + panic("%s :%d - could not allocate uio_t\n", __FILE__, __LINE__); + } + if (!IS_VALID_UIO_SEGFLG(a_spacetype)) { + panic("%s :%d - invalid address space type\n", __FILE__, __LINE__); + } + if (!(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE)) { + panic("%s :%d - invalid IO direction flag\n", __FILE__, __LINE__); + } + if (a_iovcount > UIO_MAXIOV) { + panic("%s :%d - invalid a_iovcount\n", __FILE__, __LINE__); + } +#endif /* DEBUG */ + + bzero(my_uio, my_size); + my_uio->uio_size = my_size; + + /* + * we use uio_segflg to indicate if the uio_t is the new format or + * old (pre LP64 support) legacy format + * This switch statement should canonicalize incoming space type + * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or + * UIO_SYSSPACE/UIO_PHYS_SYSSPACE + */ + switch (a_spacetype) { case UIO_USERSPACE: - c = fubyte(iov->iov_base); + my_uio->uio_segflg = UIO_USERSPACE32; break; - - case UIO_SYSSPACE: - c = *iov->iov_base & 0377; + case UIO_SYSSPACE32: + my_uio->uio_segflg = UIO_SYSSPACE; break; - - case UIO_USERISPACE: - c = fuibyte(iov->iov_base); + case UIO_PHYS_USERSPACE: + my_uio->uio_segflg = UIO_PHYS_USERSPACE32; break; + default: + my_uio->uio_segflg = a_spacetype; + break; + } + + if (a_iovcount > 0) { + my_uio->uio_iovs.uiovp = (struct user_iovec *) + (((uint8_t *)my_uio) + sizeof(struct uio)); + } + else { + my_uio->uio_iovs.uiovp = NULL; + } + + my_uio->uio_max_iovs = a_iovcount; + my_uio->uio_offset = a_offset; + my_uio->uio_rw = a_iodirection; + my_uio->uio_flags = UIO_FLAGS_INITED; + + return( my_uio ); +} + +/* + * uio_spacetype - return the address space type for the given uio_t + */ +__private_extern__ int uio_spacetype( uio_t a_uio ) +{ + if (a_uio == NULL) { +#if LP64_DEBUG + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); +#endif /* LP64_DEBUG */ + return(-1); + } + return( a_uio->uio_segflg ); +} + +/* + * uio_iovsaddr - get the address of the iovec array for the given uio_t. + * This returns the location of the iovecs within the uio. + * NOTE - for compatibility mode we just return the current value in uio_iovs + * which will increase as the IO is completed and is NOT embedded within the + * uio, it is a seperate array of one or more iovecs. + */ +__private_extern__ struct user_iovec * uio_iovsaddr( uio_t a_uio ) +{ + struct user_iovec * my_addr; + + if (a_uio == NULL) { + return(NULL); + } + + if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) { + /* we need this for compatibility mode. */ + my_addr = (struct user_iovec *) a_uio->uio_iovs.uiovp; + } + else { +#if DEBUG + panic("uio_iovsaddr called for UIO_SYSSPACE request"); +#endif + my_addr = 0; + } + return(my_addr); +} + +/* + * uio_reset - reset an uio_t. + * Reset the given uio_t to initial values. The uio_t is not fully initialized + * until all iovecs are added using uio_addiov calls. + * The a_iovcount value passed in the uio_create is the maximum number of + * iovecs you may add. + */ +void uio_reset( uio_t a_uio, + off_t a_offset, /* current offset */ + int a_spacetype, /* type of address space */ + int a_iodirection ) /* read or write flag */ +{ + vm_size_t my_size; + int my_max_iovs; + u_int32_t my_old_flags; + +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - could not allocate uio_t\n", __FILE__, __LINE__); + } + if (!IS_VALID_UIO_SEGFLG(a_spacetype)) { + panic("%s :%d - invalid address space type\n", __FILE__, __LINE__); + } + if (!(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE)) { + panic("%s :%d - invalid IO direction flag\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL) { + return; + } + + my_size = a_uio->uio_size; + my_old_flags = a_uio->uio_flags; + my_max_iovs = a_uio->uio_max_iovs; + bzero(a_uio, my_size); + a_uio->uio_size = my_size; + + /* + * we use uio_segflg to indicate if the uio_t is the new format or + * old (pre LP64 support) legacy format + * This switch statement should canonicalize incoming space type + * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or + * UIO_SYSSPACE/UIO_PHYS_SYSSPACE + */ + switch (a_spacetype) { + case UIO_USERSPACE: + a_uio->uio_segflg = UIO_USERSPACE32; + break; + case UIO_SYSSPACE32: + a_uio->uio_segflg = UIO_SYSSPACE; + break; + case UIO_PHYS_USERSPACE: + a_uio->uio_segflg = UIO_PHYS_USERSPACE32; + break; default: - c = 0; /* avoid uninitialized variable warning */ - panic("uwritec: bogus uio_segflg"); + a_uio->uio_segflg = a_spacetype; break; } - if (c < 0) - return (-1); - iov->iov_base++; - iov->iov_len--; - uio->uio_resid--; - uio->uio_offset++; - return (c); + + if (my_max_iovs > 0) { + a_uio->uio_iovs.uiovp = (struct user_iovec *) + (((uint8_t *)a_uio) + sizeof(struct uio)); + } + else { + a_uio->uio_iovs.uiovp = NULL; + } + + a_uio->uio_max_iovs = my_max_iovs; + a_uio->uio_offset = a_offset; + a_uio->uio_rw = a_iodirection; + a_uio->uio_flags = my_old_flags; + + return; } -#endif /* vax || ppc */ /* - * General routine to allocate a hash table. + * uio_free - free a uio_t allocated via uio_init. this also frees all + * associated iovecs. */ -void * -hashinit(elements, type, hashmask) - int elements, type; - u_long *hashmask; +void uio_free( uio_t a_uio ) { - long hashsize; - LIST_HEAD(generic, generic) *hashtbl; +#if DEBUG + if (a_uio == NULL) { + panic("%s :%d - passing NULL uio_t\n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) { +#if DEBUG + if (hw_atomic_sub(&uio_t_count, 1) == UINT_MAX) + panic("%s :%d - uio_t_count underflow\n", __FILE__, __LINE__); +#endif + kfree(a_uio, a_uio->uio_size); + } + + +} + +/* + * uio_addiov - add an iovec to the given uio_t. You may call this up to + * the a_iovcount number that was passed to uio_create. This call will + * increment the residual IO count as iovecs are added to the uio_t. + * returns 0 if add was successful else non zero. + */ +int uio_addiov( uio_t a_uio, user_addr_t a_baseaddr, user_size_t a_length ) +{ + int i; + + if (a_uio == NULL) { +#if DEBUG + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); +#endif /* LP64_DEBUG */ + return(-1); + } + + if (UIO_IS_USER_SPACE(a_uio)) { + for ( i = 0; i < a_uio->uio_max_iovs; i++ ) { + if (a_uio->uio_iovs.uiovp[i].iov_len == 0 && a_uio->uio_iovs.uiovp[i].iov_base == 0) { + a_uio->uio_iovs.uiovp[i].iov_len = a_length; + a_uio->uio_iovs.uiovp[i].iov_base = a_baseaddr; + a_uio->uio_iovcnt++; + a_uio->uio_resid_64 += a_length; + return( 0 ); + } + } + } + else { + for ( i = 0; i < a_uio->uio_max_iovs; i++ ) { + if (a_uio->uio_iovs.kiovp[i].iov_len == 0 && a_uio->uio_iovs.kiovp[i].iov_base == 0) { + a_uio->uio_iovs.kiovp[i].iov_len = (u_int64_t)a_length; + a_uio->uio_iovs.kiovp[i].iov_base = (u_int64_t)a_baseaddr; + a_uio->uio_iovcnt++; + a_uio->uio_resid_64 += a_length; + return( 0 ); + } + } + } + + return( -1 ); +} + +/* + * uio_getiov - get iovec data associated with the given uio_t. Use + * a_index to iterate over each iovec (0 to (uio_iovcnt(uio_t) - 1)). + * a_baseaddr_p and a_length_p may be NULL. + * returns -1 when a_index is >= uio_t.uio_iovcnt or invalid uio_t. + * returns 0 when data is returned. + */ +int uio_getiov( uio_t a_uio, + int a_index, + user_addr_t * a_baseaddr_p, + user_size_t * a_length_p ) +{ + if (a_uio == NULL) { +#if DEBUG + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); +#endif /* DEBUG */ + return(-1); + } + if ( a_index < 0 || a_index >= a_uio->uio_iovcnt) { + return(-1); + } + + if (UIO_IS_USER_SPACE(a_uio)) { + if (a_baseaddr_p != NULL) { + *a_baseaddr_p = a_uio->uio_iovs.uiovp[a_index].iov_base; + } + if (a_length_p != NULL) { + *a_length_p = a_uio->uio_iovs.uiovp[a_index].iov_len; + } + } + else { + if (a_baseaddr_p != NULL) { + *a_baseaddr_p = a_uio->uio_iovs.kiovp[a_index].iov_base; + } + if (a_length_p != NULL) { + *a_length_p = a_uio->uio_iovs.kiovp[a_index].iov_len; + } + } + + return( 0 ); +} + +/* + * uio_calculateresid - runs through all iovecs associated with this + * uio_t and calculates (and sets) the residual IO count. + */ +__private_extern__ void uio_calculateresid( uio_t a_uio ) +{ + int i; + + if (a_uio == NULL) { +#if LP64_DEBUG + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); +#endif /* LP64_DEBUG */ + return; + } + + a_uio->uio_iovcnt = a_uio->uio_max_iovs; + if (UIO_IS_USER_SPACE(a_uio)) { + a_uio->uio_resid_64 = 0; + for ( i = 0; i < a_uio->uio_max_iovs; i++ ) { + if (a_uio->uio_iovs.uiovp[i].iov_len != 0 && a_uio->uio_iovs.uiovp[i].iov_base != 0) { + a_uio->uio_resid_64 += a_uio->uio_iovs.uiovp[i].iov_len; + } + } + + /* position to first non zero length iovec (4235922) */ + while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.uiovp->iov_len == 0) { + a_uio->uio_iovcnt--; + if (a_uio->uio_iovcnt > 0) { + a_uio->uio_iovs.uiovp++; + } + } + } + else { + a_uio->uio_resid_64 = 0; + for ( i = 0; i < a_uio->uio_max_iovs; i++ ) { + if (a_uio->uio_iovs.kiovp[i].iov_len != 0 && a_uio->uio_iovs.kiovp[i].iov_base != 0) { + a_uio->uio_resid_64 += a_uio->uio_iovs.kiovp[i].iov_len; + } + } + + /* position to first non zero length iovec (4235922) */ + while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.kiovp->iov_len == 0) { + a_uio->uio_iovcnt--; + if (a_uio->uio_iovcnt > 0) { + a_uio->uio_iovs.kiovp++; + } + } + } + + return; +} + +/* + * uio_update - update the given uio_t for a_count of completed IO. + * This call decrements the current iovec length and residual IO value + * and increments the current iovec base address and offset value. + * If the current iovec length is 0 then advance to the next + * iovec (if any). + * If the a_count passed in is 0, than only do the advancement + * over any 0 length iovec's. + */ +void uio_update( uio_t a_uio, user_size_t a_count ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } + if (UIO_IS_32_BIT_SPACE(a_uio) && a_count > 0xFFFFFFFFull) { + panic("%s :%d - invalid count value \n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL || a_uio->uio_iovcnt < 1) { + return; + } + + if (UIO_IS_USER_SPACE(a_uio)) { + /* + * if a_count == 0, then we are asking to skip over + * any empty iovs + */ + if (a_count) { + if (a_count > a_uio->uio_iovs.uiovp->iov_len) { + a_uio->uio_iovs.uiovp->iov_base += a_uio->uio_iovs.uiovp->iov_len; + a_uio->uio_iovs.uiovp->iov_len = 0; + } + else { + a_uio->uio_iovs.uiovp->iov_base += a_count; + a_uio->uio_iovs.uiovp->iov_len -= a_count; + } + if (a_uio->uio_resid_64 < 0) { + a_uio->uio_resid_64 = 0; + } + if (a_count > (user_size_t)a_uio->uio_resid_64) { + a_uio->uio_offset += a_uio->uio_resid_64; + a_uio->uio_resid_64 = 0; + } + else { + a_uio->uio_offset += a_count; + a_uio->uio_resid_64 -= a_count; + } + } + /* + * advance to next iovec if current one is totally consumed + */ + while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.uiovp->iov_len == 0) { + a_uio->uio_iovcnt--; + if (a_uio->uio_iovcnt > 0) { + a_uio->uio_iovs.uiovp++; + } + } + } + else { + /* + * if a_count == 0, then we are asking to skip over + * any empty iovs + */ + if (a_count) { + if (a_count > a_uio->uio_iovs.kiovp->iov_len) { + a_uio->uio_iovs.kiovp->iov_base += a_uio->uio_iovs.kiovp->iov_len; + a_uio->uio_iovs.kiovp->iov_len = 0; + } + else { + a_uio->uio_iovs.kiovp->iov_base += a_count; + a_uio->uio_iovs.kiovp->iov_len -= a_count; + } + if (a_uio->uio_resid_64 < 0) { + a_uio->uio_resid_64 = 0; + } + if (a_count > (user_size_t)a_uio->uio_resid_64) { + a_uio->uio_offset += a_uio->uio_resid_64; + a_uio->uio_resid_64 = 0; + } + else { + a_uio->uio_offset += a_count; + a_uio->uio_resid_64 -= a_count; + } + } + /* + * advance to next iovec if current one is totally consumed + */ + while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.kiovp->iov_len == 0) { + a_uio->uio_iovcnt--; + if (a_uio->uio_iovcnt > 0) { + a_uio->uio_iovs.kiovp++; + } + } + } + return; +} + +/* + * uio_pushback - undo uncommitted I/O by subtracting from the + * current base address and offset, and incrementing the residiual + * IO. If the UIO was previously exhausted, this call will panic. + * New code should not use this functionality. + */ +__private_extern__ void uio_pushback( uio_t a_uio, user_size_t a_count ) +{ +#if LP64_DEBUG + if (a_uio == NULL) { + panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); + } + if (UIO_IS_32_BIT_SPACE(a_uio) && a_count > 0xFFFFFFFFull) { + panic("%s :%d - invalid count value \n", __FILE__, __LINE__); + } +#endif /* LP64_DEBUG */ + + if (a_uio == NULL || a_count == 0) { + return; + } + + if (a_uio->uio_iovcnt < 1) { + panic("Invalid uio for pushback"); + } + + if (UIO_IS_USER_SPACE(a_uio)) { + a_uio->uio_iovs.uiovp->iov_base -= a_count; + a_uio->uio_iovs.uiovp->iov_len += a_count; + } + else { + a_uio->uio_iovs.kiovp->iov_base -= a_count; + a_uio->uio_iovs.kiovp->iov_len += a_count; + } + + a_uio->uio_offset -= a_count; + a_uio->uio_resid_64 += a_count; + + return; +} + + +/* + * uio_duplicate - allocate a new uio and make a copy of the given uio_t. + * may return NULL. + */ +uio_t uio_duplicate( uio_t a_uio ) +{ + uio_t my_uio; + int i; + + if (a_uio == NULL) { + return(NULL); + } + + my_uio = (uio_t) kalloc(a_uio->uio_size); + if (my_uio == 0) { + panic("%s :%d - allocation failed\n", __FILE__, __LINE__); + } + + bcopy((void *)a_uio, (void *)my_uio, a_uio->uio_size); + /* need to set our iovec pointer to point to first active iovec */ + if (my_uio->uio_max_iovs > 0) { + my_uio->uio_iovs.uiovp = (struct user_iovec *) + (((uint8_t *)my_uio) + sizeof(struct uio)); + + /* advance to first nonzero iovec */ + if (my_uio->uio_iovcnt > 0) { + for ( i = 0; i < my_uio->uio_max_iovs; i++ ) { + if (UIO_IS_USER_SPACE(a_uio)) { + if (my_uio->uio_iovs.uiovp->iov_len != 0) { + break; + } + my_uio->uio_iovs.uiovp++; + } + else { + if (my_uio->uio_iovs.kiovp->iov_len != 0) { + break; + } + my_uio->uio_iovs.kiovp++; + } + } + } + } + + my_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED; + + return(my_uio); +} + +int copyin_user_iovec_array(user_addr_t uaddr, int spacetype, int count, struct user_iovec *dst) +{ + size_t size_of_iovec = ( spacetype == UIO_USERSPACE64 ? sizeof(struct user64_iovec) : sizeof(struct user32_iovec)); + int error; int i; - if (elements <= 0) - panic("hashinit: bad cnt"); - for (hashsize = 1; hashsize <= elements; hashsize <<= 1) - continue; - hashsize >>= 1; - MALLOC(hashtbl, struct generic *, - (u_long)hashsize * sizeof(*hashtbl), type, M_WAITOK); - bzero(hashtbl, (u_long)hashsize * sizeof(*hashtbl)); - for (i = 0; i < hashsize; i++) - LIST_INIT(&hashtbl[i]); - *hashmask = hashsize - 1; - return (hashtbl); + // copyin to the front of "dst", without regard for putting records in the right places + error = copyin(uaddr, dst, count * size_of_iovec); + if (error) + return (error); + + // now, unpack the entries in reverse order, so we don't overwrite anything + for (i = count - 1; i >= 0; i--) { + if (spacetype == UIO_USERSPACE64) { + struct user64_iovec iovec = ((struct user64_iovec *)dst)[i]; + dst[i].iov_base = iovec.iov_base; + dst[i].iov_len = iovec.iov_len; + } else { + struct user32_iovec iovec = ((struct user32_iovec *)dst)[i]; + dst[i].iov_base = iovec.iov_base; + dst[i].iov_len = iovec.iov_len; + } + } + + return (0); }