]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/kern_subr.c
xnu-6153.81.5.tar.gz
[apple/xnu.git] / bsd / kern / kern_subr.c
1 /*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1982, 1986, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 * (c) UNIX System Laboratories, Inc.
33 * All or some portions of this file are derived from material licensed
34 * to the University of California by American Telephone and Telegraph
35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
36 * the permission of UNIX System Laboratories, Inc.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the University of
49 * California, Berkeley and its contributors.
50 * 4. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
67 */
68
69 #include <machine/atomic.h>
70
71 #include <sys/param.h>
72 #include <sys/systm.h>
73 #include <sys/proc_internal.h>
74 #include <sys/malloc.h>
75 #include <sys/queue.h>
76 #include <vm/pmap.h>
77 #include <sys/uio_internal.h>
78 #include <kern/kalloc.h>
79
80 #include <kdebug.h>
81
82 #include <sys/kdebug.h>
83 #define DBG_UIO_COPYOUT 16
84 #define DBG_UIO_COPYIN 17
85
86 #if DEBUG
87 #include <kern/simple_lock.h>
88
89 static uint32_t uio_t_count = 0;
90 #endif /* DEBUG */
91
92 #define IS_VALID_UIO_SEGFLG(segflg) \
93 ( (segflg) == UIO_USERSPACE || \
94 (segflg) == UIO_SYSSPACE || \
95 (segflg) == UIO_USERSPACE32 || \
96 (segflg) == UIO_USERSPACE64 || \
97 (segflg) == UIO_SYSSPACE32 || \
98 (segflg) == UIO_USERISPACE || \
99 (segflg) == UIO_PHYS_USERSPACE || \
100 (segflg) == UIO_PHYS_SYSSPACE || \
101 (segflg) == UIO_USERISPACE32 || \
102 (segflg) == UIO_PHYS_USERSPACE32 || \
103 (segflg) == UIO_USERISPACE64 || \
104 (segflg) == UIO_PHYS_USERSPACE64 )
105
106 /*
107 * Returns: 0 Success
108 * uiomove64:EFAULT
109 *
110 * Notes: The first argument should be a caddr_t, but const poisoning
111 * for typedef'ed types doesn't work in gcc.
112 */
113 int
114 uiomove(const char * cp, int n, uio_t uio)
115 {
116 return uiomove64((const addr64_t)(uintptr_t)cp, n, uio);
117 }
118
119 /*
120 * Returns: 0 Success
121 * EFAULT
122 * copyout:EFAULT
123 * copyin:EFAULT
124 * copywithin:EFAULT
125 * copypv:EFAULT
126 */
127 int
128 uiomove64(const addr64_t c_cp, int n, struct uio *uio)
129 {
130 addr64_t cp = c_cp;
131 uint64_t acnt;
132 int error = 0;
133
134 #if DIAGNOSTIC
135 if (uio->uio_rw != UIO_READ && uio->uio_rw != UIO_WRITE) {
136 panic("uiomove: mode");
137 }
138 #endif
139
140 #if LP64_DEBUG
141 if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
142 panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);
143 }
144 #endif /* LP64_DEBUG */
145
146 while (n > 0 && uio_resid(uio)) {
147 uio_update(uio, 0);
148 acnt = uio_curriovlen(uio);
149 if (acnt == 0) {
150 continue;
151 }
152 if (n > 0 && acnt > (uint64_t)n) {
153 acnt = n;
154 }
155
156 switch ((int) uio->uio_segflg) {
157 case UIO_USERSPACE64:
158 case UIO_USERISPACE64:
159 case UIO_USERSPACE32:
160 case UIO_USERISPACE32:
161 case UIO_USERSPACE:
162 case UIO_USERISPACE:
163 // LP64 - 3rd argument in debug code is 64 bit, expected to be 32 bit
164 if (uio->uio_rw == UIO_READ) {
165 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
166 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 0, 0);
167
168 error = copyout( CAST_DOWN(caddr_t, cp), uio->uio_iovs.uiovp->iov_base, acnt );
169
170 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
171 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 0, 0);
172 } else {
173 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
174 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 0, 0);
175
176 error = copyin(uio->uio_iovs.uiovp->iov_base, CAST_DOWN(caddr_t, cp), acnt);
177
178 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
179 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 0, 0);
180 }
181 if (error) {
182 return error;
183 }
184 break;
185
186 case UIO_SYSSPACE32:
187 case UIO_SYSSPACE:
188 if (uio->uio_rw == UIO_READ) {
189 error = copywithin(CAST_DOWN(caddr_t, cp), CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base),
190 acnt);
191 } else {
192 error = copywithin(CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base), CAST_DOWN(caddr_t, cp),
193 acnt);
194 }
195 break;
196
197 case UIO_PHYS_USERSPACE64:
198 case UIO_PHYS_USERSPACE32:
199 case UIO_PHYS_USERSPACE:
200 if (uio->uio_rw == UIO_READ) {
201 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
202 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 1, 0);
203
204 error = copypv((addr64_t)cp, uio->uio_iovs.uiovp->iov_base, acnt, cppvPsrc | cppvNoRefSrc);
205 if (error) { /* Copy physical to virtual */
206 error = EFAULT;
207 }
208
209 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
210 (int)cp, (uintptr_t)uio->uio_iovs.uiovp->iov_base, acnt, 1, 0);
211 } else {
212 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
213 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 1, 0);
214
215 error = copypv(uio->uio_iovs.uiovp->iov_base, (addr64_t)cp, acnt, cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
216 if (error) { /* Copy virtual to physical */
217 error = EFAULT;
218 }
219
220 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
221 (uintptr_t)uio->uio_iovs.uiovp->iov_base, (int)cp, acnt, 1, 0);
222 }
223 if (error) {
224 return error;
225 }
226 break;
227
228 case UIO_PHYS_SYSSPACE:
229 if (uio->uio_rw == UIO_READ) {
230 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_START,
231 (int)cp, (uintptr_t)uio->uio_iovs.kiovp->iov_base, acnt, 2, 0);
232
233 error = copypv((addr64_t)cp, uio->uio_iovs.kiovp->iov_base, acnt, cppvKmap | cppvPsrc | cppvNoRefSrc);
234 if (error) { /* Copy physical to virtual */
235 error = EFAULT;
236 }
237
238 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYOUT)) | DBG_FUNC_END,
239 (int)cp, (uintptr_t)uio->uio_iovs.kiovp->iov_base, acnt, 2, 0);
240 } else {
241 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_START,
242 (uintptr_t)uio->uio_iovs.kiovp->iov_base, (int)cp, acnt, 2, 0);
243
244 error = copypv(uio->uio_iovs.kiovp->iov_base, (addr64_t)cp, acnt, cppvKmap | cppvPsnk | cppvNoRefSrc | cppvNoModSnk);
245 if (error) { /* Copy virtual to physical */
246 error = EFAULT;
247 }
248
249 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, DBG_UIO_COPYIN)) | DBG_FUNC_END,
250 (uintptr_t)uio->uio_iovs.kiovp->iov_base, (int)cp, acnt, 2, 0);
251 }
252 if (error) {
253 return error;
254 }
255 break;
256
257 default:
258 break;
259 }
260 uio_update(uio, acnt);
261 cp += acnt;
262 n -= acnt;
263 }
264 return error;
265 }
266
267 /*
268 * Give next character to user as result of read.
269 */
270 int
271 ureadc(int c, struct uio *uio)
272 {
273 if (uio_resid(uio) <= 0) {
274 panic("ureadc: non-positive resid");
275 }
276 uio_update(uio, 0);
277 if (uio->uio_iovcnt == 0) {
278 panic("ureadc: non-positive iovcnt");
279 }
280 if (uio_curriovlen(uio) <= 0) {
281 panic("ureadc: non-positive iovlen");
282 }
283
284 switch ((int) uio->uio_segflg) {
285 case UIO_USERSPACE32:
286 case UIO_USERSPACE:
287 case UIO_USERISPACE32:
288 case UIO_USERISPACE:
289 case UIO_USERSPACE64:
290 case UIO_USERISPACE64:
291 if (subyte((user_addr_t)uio->uio_iovs.uiovp->iov_base, c) < 0) {
292 return EFAULT;
293 }
294 break;
295
296 case UIO_SYSSPACE32:
297 case UIO_SYSSPACE:
298 *(CAST_DOWN(caddr_t, uio->uio_iovs.kiovp->iov_base)) = c;
299 break;
300
301 default:
302 break;
303 }
304 uio_update(uio, 1);
305 return 0;
306 }
307
308 /*
309 * General routine to allocate a hash table.
310 */
311 void *
312 hashinit(int elements, int type, u_long *hashmask)
313 {
314 long hashsize;
315 LIST_HEAD(generic, generic) * hashtbl;
316 int i;
317
318 if (elements <= 0) {
319 panic("hashinit: bad cnt");
320 }
321 for (hashsize = 1; hashsize <= elements; hashsize <<= 1) {
322 continue;
323 }
324 hashsize >>= 1;
325 MALLOC(hashtbl, struct generic *,
326 hashsize * sizeof(*hashtbl), type, M_WAITOK | M_ZERO);
327 if (hashtbl != NULL) {
328 for (i = 0; i < hashsize; i++) {
329 LIST_INIT(&hashtbl[i]);
330 }
331 *hashmask = hashsize - 1;
332 }
333 return hashtbl;
334 }
335
336 /*
337 * uio_resid - return the residual IO value for the given uio_t
338 */
339 user_ssize_t
340 uio_resid( uio_t a_uio )
341 {
342 #if DEBUG
343 if (a_uio == NULL) {
344 printf("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
345 }
346 /* if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */
347 /* panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); */
348 /* } */
349 #endif /* DEBUG */
350
351 /* return 0 if there are no active iovecs */
352 if (a_uio == NULL) {
353 return 0;
354 }
355
356 return a_uio->uio_resid_64;
357 }
358
359 /*
360 * uio_setresid - set the residual IO value for the given uio_t
361 */
362 void
363 uio_setresid( uio_t a_uio, user_ssize_t a_value )
364 {
365 #if DEBUG
366 if (a_uio == NULL) {
367 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
368 }
369 /* if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */
370 /* panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); */
371 /* } */
372 #endif /* DEBUG */
373
374 if (a_uio == NULL) {
375 return;
376 }
377
378 a_uio->uio_resid_64 = a_value;
379 return;
380 }
381
382 /*
383 * uio_curriovbase - return the base address of the current iovec associated
384 * with the given uio_t. May return 0.
385 */
386 user_addr_t
387 uio_curriovbase( uio_t a_uio )
388 {
389 #if LP64_DEBUG
390 if (a_uio == NULL) {
391 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
392 }
393 #endif /* LP64_DEBUG */
394
395 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
396 return 0;
397 }
398
399 if (UIO_IS_USER_SPACE(a_uio)) {
400 return a_uio->uio_iovs.uiovp->iov_base;
401 }
402 return (user_addr_t)a_uio->uio_iovs.kiovp->iov_base;
403 }
404
405 /*
406 * uio_curriovlen - return the length value of the current iovec associated
407 * with the given uio_t.
408 */
409 user_size_t
410 uio_curriovlen( uio_t a_uio )
411 {
412 #if LP64_DEBUG
413 if (a_uio == NULL) {
414 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
415 }
416 #endif /* LP64_DEBUG */
417
418 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
419 return 0;
420 }
421
422 if (UIO_IS_USER_SPACE(a_uio)) {
423 return a_uio->uio_iovs.uiovp->iov_len;
424 }
425 return (user_size_t)a_uio->uio_iovs.kiovp->iov_len;
426 }
427
428 /*
429 * uio_setcurriovlen - set the length value of the current iovec associated
430 * with the given uio_t.
431 */
432 __private_extern__ void
433 uio_setcurriovlen( uio_t a_uio, user_size_t a_value )
434 {
435 #if LP64_DEBUG
436 if (a_uio == NULL) {
437 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
438 }
439 #endif /* LP64_DEBUG */
440
441 if (a_uio == NULL) {
442 return;
443 }
444
445 if (UIO_IS_USER_SPACE(a_uio)) {
446 a_uio->uio_iovs.uiovp->iov_len = a_value;
447 } else {
448 #if LP64_DEBUG
449 if (a_value > 0xFFFFFFFFull) {
450 panic("%s :%d - invalid a_value\n", __FILE__, __LINE__);
451 }
452 #endif /* LP64_DEBUG */
453 a_uio->uio_iovs.kiovp->iov_len = (size_t)a_value;
454 }
455 return;
456 }
457
458 /*
459 * uio_iovcnt - return count of active iovecs for the given uio_t
460 */
461 int
462 uio_iovcnt( uio_t a_uio )
463 {
464 #if LP64_DEBUG
465 if (a_uio == NULL) {
466 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
467 }
468 #endif /* LP64_DEBUG */
469
470 if (a_uio == NULL) {
471 return 0;
472 }
473
474 return a_uio->uio_iovcnt;
475 }
476
477 /*
478 * uio_offset - return the current offset value for the given uio_t
479 */
480 off_t
481 uio_offset( uio_t a_uio )
482 {
483 #if LP64_DEBUG
484 if (a_uio == NULL) {
485 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
486 }
487 #endif /* LP64_DEBUG */
488
489 if (a_uio == NULL) {
490 return 0;
491 }
492 return a_uio->uio_offset;
493 }
494
495 /*
496 * uio_setoffset - set the current offset value for the given uio_t
497 */
498 void
499 uio_setoffset( uio_t a_uio, off_t a_offset )
500 {
501 #if LP64_DEBUG
502 if (a_uio == NULL) {
503 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
504 }
505 #endif /* LP64_DEBUG */
506
507 if (a_uio == NULL) {
508 return;
509 }
510 a_uio->uio_offset = a_offset;
511 return;
512 }
513
514 /*
515 * uio_rw - return the read / write flag for the given uio_t
516 */
517 int
518 uio_rw( uio_t a_uio )
519 {
520 #if LP64_DEBUG
521 if (a_uio == NULL) {
522 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
523 }
524 #endif /* LP64_DEBUG */
525
526 if (a_uio == NULL) {
527 return -1;
528 }
529 return a_uio->uio_rw;
530 }
531
532 /*
533 * uio_setrw - set the read / write flag for the given uio_t
534 */
535 void
536 uio_setrw( uio_t a_uio, int a_value )
537 {
538 if (a_uio == NULL) {
539 #if LP64_DEBUG
540 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
541 #endif /* LP64_DEBUG */
542 return;
543 }
544
545 #if LP64_DEBUG
546 if (!(a_value == UIO_READ || a_value == UIO_WRITE)) {
547 panic("%s :%d - invalid a_value\n", __FILE__, __LINE__);
548 }
549 #endif /* LP64_DEBUG */
550
551 if (a_value == UIO_READ || a_value == UIO_WRITE) {
552 a_uio->uio_rw = a_value;
553 }
554 return;
555 }
556
557 /*
558 * uio_isuserspace - return non zero value if the address space
559 * flag is for a user address space (could be 32 or 64 bit).
560 */
561 int
562 uio_isuserspace( uio_t a_uio )
563 {
564 if (a_uio == NULL) {
565 #if LP64_DEBUG
566 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
567 #endif /* LP64_DEBUG */
568 return 0;
569 }
570
571 if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) {
572 return 1;
573 }
574 return 0;
575 }
576
577
578 /*
579 * uio_create - create an uio_t.
580 * Space is allocated to hold up to a_iovcount number of iovecs. The uio_t
581 * is not fully initialized until all iovecs are added using uio_addiov calls.
582 * a_iovcount is the maximum number of iovecs you may add.
583 */
584 uio_t
585 uio_create( int a_iovcount, /* number of iovecs */
586 off_t a_offset, /* current offset */
587 int a_spacetype, /* type of address space */
588 int a_iodirection ) /* read or write flag */
589 {
590 void * my_buf_p;
591 size_t my_size;
592 uio_t my_uio;
593
594 my_size = UIO_SIZEOF(a_iovcount);
595 my_buf_p = kalloc(my_size);
596 my_uio = uio_createwithbuffer( a_iovcount,
597 a_offset,
598 a_spacetype,
599 a_iodirection,
600 my_buf_p,
601 my_size );
602 if (my_uio != 0) {
603 /* leave a note that we allocated this uio_t */
604 my_uio->uio_flags |= UIO_FLAGS_WE_ALLOCED;
605 #if DEBUG
606 os_atomic_inc(&uio_t_count, relaxed);
607 #endif
608 }
609
610 return my_uio;
611 }
612
613
614 /*
615 * uio_createwithbuffer - create an uio_t.
616 * Create a uio_t using the given buffer. The uio_t
617 * is not fully initialized until all iovecs are added using uio_addiov calls.
618 * a_iovcount is the maximum number of iovecs you may add.
619 * This call may fail if the given buffer is not large enough.
620 */
621 __private_extern__ uio_t
622 uio_createwithbuffer( int a_iovcount, /* number of iovecs */
623 off_t a_offset, /* current offset */
624 int a_spacetype, /* type of address space */
625 int a_iodirection, /* read or write flag */
626 void *a_buf_p, /* pointer to a uio_t buffer */
627 size_t a_buffer_size ) /* size of uio_t buffer */
628 {
629 uio_t my_uio = (uio_t) a_buf_p;
630 size_t my_size;
631
632 my_size = UIO_SIZEOF(a_iovcount);
633 if (a_buffer_size < my_size) {
634 #if DEBUG
635 panic("%s :%d - a_buffer_size is too small\n", __FILE__, __LINE__);
636 #endif /* DEBUG */
637 return NULL;
638 }
639 my_size = a_buffer_size;
640
641 #if DEBUG
642 if (my_uio == 0) {
643 panic("%s :%d - could not allocate uio_t\n", __FILE__, __LINE__);
644 }
645 if (!IS_VALID_UIO_SEGFLG(a_spacetype)) {
646 panic("%s :%d - invalid address space type\n", __FILE__, __LINE__);
647 }
648 if (!(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE)) {
649 panic("%s :%d - invalid IO direction flag\n", __FILE__, __LINE__);
650 }
651 if (a_iovcount > UIO_MAXIOV) {
652 panic("%s :%d - invalid a_iovcount\n", __FILE__, __LINE__);
653 }
654 #endif /* DEBUG */
655
656 bzero(my_uio, my_size);
657 my_uio->uio_size = my_size;
658
659 /*
660 * we use uio_segflg to indicate if the uio_t is the new format or
661 * old (pre LP64 support) legacy format
662 * This switch statement should canonicalize incoming space type
663 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
664 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
665 */
666 switch (a_spacetype) {
667 case UIO_USERSPACE:
668 my_uio->uio_segflg = UIO_USERSPACE32;
669 break;
670 case UIO_SYSSPACE32:
671 my_uio->uio_segflg = UIO_SYSSPACE;
672 break;
673 case UIO_PHYS_USERSPACE:
674 my_uio->uio_segflg = UIO_PHYS_USERSPACE32;
675 break;
676 default:
677 my_uio->uio_segflg = a_spacetype;
678 break;
679 }
680
681 if (a_iovcount > 0) {
682 my_uio->uio_iovs.uiovp = (struct user_iovec *)
683 (((uint8_t *)my_uio) + sizeof(struct uio));
684 } else {
685 my_uio->uio_iovs.uiovp = NULL;
686 }
687
688 my_uio->uio_max_iovs = a_iovcount;
689 my_uio->uio_offset = a_offset;
690 my_uio->uio_rw = a_iodirection;
691 my_uio->uio_flags = UIO_FLAGS_INITED;
692
693 return my_uio;
694 }
695
696 /*
697 * uio_spacetype - return the address space type for the given uio_t
698 */
699 __private_extern__ int
700 uio_spacetype( uio_t a_uio )
701 {
702 if (a_uio == NULL) {
703 #if LP64_DEBUG
704 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
705 #endif /* LP64_DEBUG */
706 return -1;
707 }
708
709 return a_uio->uio_segflg;
710 }
711
712 /*
713 * uio_iovsaddr - get the address of the iovec array for the given uio_t.
714 * This returns the location of the iovecs within the uio.
715 * NOTE - for compatibility mode we just return the current value in uio_iovs
716 * which will increase as the IO is completed and is NOT embedded within the
717 * uio, it is a seperate array of one or more iovecs.
718 */
719 __private_extern__ struct user_iovec *
720 uio_iovsaddr( uio_t a_uio )
721 {
722 struct user_iovec * my_addr;
723
724 if (a_uio == NULL) {
725 return NULL;
726 }
727
728 if (UIO_SEG_IS_USER_SPACE(a_uio->uio_segflg)) {
729 /* we need this for compatibility mode. */
730 my_addr = (struct user_iovec *) a_uio->uio_iovs.uiovp;
731 } else {
732 #if DEBUG
733 panic("uio_iovsaddr called for UIO_SYSSPACE request");
734 #endif
735 my_addr = 0;
736 }
737 return my_addr;
738 }
739
740 /*
741 * uio_reset - reset an uio_t.
742 * Reset the given uio_t to initial values. The uio_t is not fully initialized
743 * until all iovecs are added using uio_addiov calls.
744 * The a_iovcount value passed in the uio_create is the maximum number of
745 * iovecs you may add.
746 */
747 void
748 uio_reset( uio_t a_uio,
749 off_t a_offset, /* current offset */
750 int a_spacetype, /* type of address space */
751 int a_iodirection ) /* read or write flag */
752 {
753 vm_size_t my_size;
754 int my_max_iovs;
755 u_int32_t my_old_flags;
756
757 #if LP64_DEBUG
758 if (a_uio == NULL) {
759 panic("%s :%d - could not allocate uio_t\n", __FILE__, __LINE__);
760 }
761 if (!IS_VALID_UIO_SEGFLG(a_spacetype)) {
762 panic("%s :%d - invalid address space type\n", __FILE__, __LINE__);
763 }
764 if (!(a_iodirection == UIO_READ || a_iodirection == UIO_WRITE)) {
765 panic("%s :%d - invalid IO direction flag\n", __FILE__, __LINE__);
766 }
767 #endif /* LP64_DEBUG */
768
769 if (a_uio == NULL) {
770 return;
771 }
772
773 my_size = a_uio->uio_size;
774 my_old_flags = a_uio->uio_flags;
775 my_max_iovs = a_uio->uio_max_iovs;
776 bzero(a_uio, my_size);
777 a_uio->uio_size = my_size;
778
779 /*
780 * we use uio_segflg to indicate if the uio_t is the new format or
781 * old (pre LP64 support) legacy format
782 * This switch statement should canonicalize incoming space type
783 * to one of UIO_USERSPACE32/64, UIO_PHYS_USERSPACE32/64, or
784 * UIO_SYSSPACE/UIO_PHYS_SYSSPACE
785 */
786 switch (a_spacetype) {
787 case UIO_USERSPACE:
788 a_uio->uio_segflg = UIO_USERSPACE32;
789 break;
790 case UIO_SYSSPACE32:
791 a_uio->uio_segflg = UIO_SYSSPACE;
792 break;
793 case UIO_PHYS_USERSPACE:
794 a_uio->uio_segflg = UIO_PHYS_USERSPACE32;
795 break;
796 default:
797 a_uio->uio_segflg = a_spacetype;
798 break;
799 }
800
801 if (my_max_iovs > 0) {
802 a_uio->uio_iovs.uiovp = (struct user_iovec *)
803 (((uint8_t *)a_uio) + sizeof(struct uio));
804 } else {
805 a_uio->uio_iovs.uiovp = NULL;
806 }
807
808 a_uio->uio_max_iovs = my_max_iovs;
809 a_uio->uio_offset = a_offset;
810 a_uio->uio_rw = a_iodirection;
811 a_uio->uio_flags = my_old_flags;
812
813 return;
814 }
815
816 /*
817 * uio_free - free a uio_t allocated via uio_init. this also frees all
818 * associated iovecs.
819 */
820 void
821 uio_free( uio_t a_uio )
822 {
823 #if DEBUG
824 if (a_uio == NULL) {
825 panic("%s :%d - passing NULL uio_t\n", __FILE__, __LINE__);
826 }
827 #endif /* LP64_DEBUG */
828
829 if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) {
830 #if DEBUG
831 if (os_atomic_dec_orig(&uio_t_count, relaxed) == 0) {
832 panic("%s :%d - uio_t_count underflow\n", __FILE__, __LINE__);
833 }
834 #endif
835 kfree(a_uio, a_uio->uio_size);
836 }
837 }
838
839 /*
840 * uio_addiov - add an iovec to the given uio_t. You may call this up to
841 * the a_iovcount number that was passed to uio_create. This call will
842 * increment the residual IO count as iovecs are added to the uio_t.
843 * returns 0 if add was successful else non zero.
844 */
845 int
846 uio_addiov( uio_t a_uio, user_addr_t a_baseaddr, user_size_t a_length )
847 {
848 int i;
849 user_size_t resid;
850
851 if (a_uio == NULL) {
852 #if DEBUG
853 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
854 #endif
855 return -1;
856 }
857
858 if (os_add_overflow(a_length, a_uio->uio_resid_64, &resid)) {
859 #if DEBUG
860 panic("%s :%d - invalid length %lu\n", __FILE__, __LINE__, (unsigned long)a_length);
861 #endif
862 return -1;
863 }
864
865 if (UIO_IS_USER_SPACE(a_uio)) {
866 for (i = 0; i < a_uio->uio_max_iovs; i++) {
867 if (a_uio->uio_iovs.uiovp[i].iov_len == 0 && a_uio->uio_iovs.uiovp[i].iov_base == 0) {
868 a_uio->uio_iovs.uiovp[i].iov_len = a_length;
869 a_uio->uio_iovs.uiovp[i].iov_base = a_baseaddr;
870 a_uio->uio_iovcnt++;
871 a_uio->uio_resid_64 = resid;
872 return 0;
873 }
874 }
875 } else {
876 for (i = 0; i < a_uio->uio_max_iovs; i++) {
877 if (a_uio->uio_iovs.kiovp[i].iov_len == 0 && a_uio->uio_iovs.kiovp[i].iov_base == 0) {
878 a_uio->uio_iovs.kiovp[i].iov_len = (u_int64_t)a_length;
879 a_uio->uio_iovs.kiovp[i].iov_base = (u_int64_t)a_baseaddr;
880 a_uio->uio_iovcnt++;
881 a_uio->uio_resid_64 = resid;
882 return 0;
883 }
884 }
885 }
886
887 return -1;
888 }
889
890 /*
891 * uio_getiov - get iovec data associated with the given uio_t. Use
892 * a_index to iterate over each iovec (0 to (uio_iovcnt(uio_t) - 1)).
893 * a_baseaddr_p and a_length_p may be NULL.
894 * returns -1 when a_index is >= uio_t.uio_iovcnt or invalid uio_t.
895 * returns 0 when data is returned.
896 */
897 int
898 uio_getiov( uio_t a_uio,
899 int a_index,
900 user_addr_t * a_baseaddr_p,
901 user_size_t * a_length_p )
902 {
903 if (a_uio == NULL) {
904 #if DEBUG
905 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
906 #endif /* DEBUG */
907 return -1;
908 }
909 if (a_index < 0 || a_index >= a_uio->uio_iovcnt) {
910 return -1;
911 }
912
913 if (UIO_IS_USER_SPACE(a_uio)) {
914 if (a_baseaddr_p != NULL) {
915 *a_baseaddr_p = a_uio->uio_iovs.uiovp[a_index].iov_base;
916 }
917 if (a_length_p != NULL) {
918 *a_length_p = a_uio->uio_iovs.uiovp[a_index].iov_len;
919 }
920 } else {
921 if (a_baseaddr_p != NULL) {
922 *a_baseaddr_p = a_uio->uio_iovs.kiovp[a_index].iov_base;
923 }
924 if (a_length_p != NULL) {
925 *a_length_p = a_uio->uio_iovs.kiovp[a_index].iov_len;
926 }
927 }
928
929 return 0;
930 }
931
932 /*
933 * uio_calculateresid - runs through all iovecs associated with this
934 * uio_t and calculates (and sets) the residual IO count.
935 */
936 __private_extern__ int
937 uio_calculateresid( uio_t a_uio )
938 {
939 int i;
940 u_int64_t resid = 0;
941
942 if (a_uio == NULL) {
943 #if LP64_DEBUG
944 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
945 #endif /* LP64_DEBUG */
946 return EINVAL;
947 }
948
949 a_uio->uio_iovcnt = a_uio->uio_max_iovs;
950 if (UIO_IS_USER_SPACE(a_uio)) {
951 a_uio->uio_resid_64 = 0;
952 for (i = 0; i < a_uio->uio_max_iovs; i++) {
953 if (a_uio->uio_iovs.uiovp[i].iov_len != 0 && a_uio->uio_iovs.uiovp[i].iov_base != 0) {
954 if (a_uio->uio_iovs.uiovp[i].iov_len > LONG_MAX) {
955 return EINVAL;
956 }
957 resid += a_uio->uio_iovs.uiovp[i].iov_len;
958 if (resid > LONG_MAX) {
959 return EINVAL;
960 }
961 }
962 }
963 a_uio->uio_resid_64 = resid;
964
965 /* position to first non zero length iovec (4235922) */
966 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.uiovp->iov_len == 0) {
967 a_uio->uio_iovcnt--;
968 if (a_uio->uio_iovcnt > 0) {
969 a_uio->uio_iovs.uiovp++;
970 }
971 }
972 } else {
973 a_uio->uio_resid_64 = 0;
974 for (i = 0; i < a_uio->uio_max_iovs; i++) {
975 if (a_uio->uio_iovs.kiovp[i].iov_len != 0 && a_uio->uio_iovs.kiovp[i].iov_base != 0) {
976 if (a_uio->uio_iovs.kiovp[i].iov_len > LONG_MAX) {
977 return EINVAL;
978 }
979 resid += a_uio->uio_iovs.kiovp[i].iov_len;
980 if (resid > LONG_MAX) {
981 return EINVAL;
982 }
983 }
984 }
985 a_uio->uio_resid_64 = resid;
986
987 /* position to first non zero length iovec (4235922) */
988 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.kiovp->iov_len == 0) {
989 a_uio->uio_iovcnt--;
990 if (a_uio->uio_iovcnt > 0) {
991 a_uio->uio_iovs.kiovp++;
992 }
993 }
994 }
995
996 return 0;
997 }
998
999 /*
1000 * uio_update - update the given uio_t for a_count of completed IO.
1001 * This call decrements the current iovec length and residual IO value
1002 * and increments the current iovec base address and offset value.
1003 * If the current iovec length is 0 then advance to the next
1004 * iovec (if any).
1005 * If the a_count passed in is 0, than only do the advancement
1006 * over any 0 length iovec's.
1007 */
1008 void
1009 uio_update( uio_t a_uio, user_size_t a_count )
1010 {
1011 #if LP64_DEBUG
1012 if (a_uio == NULL) {
1013 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
1014 }
1015 if (UIO_IS_32_BIT_SPACE(a_uio) && a_count > 0xFFFFFFFFull) {
1016 panic("%s :%d - invalid count value \n", __FILE__, __LINE__);
1017 }
1018 #endif /* LP64_DEBUG */
1019
1020 if (a_uio == NULL || a_uio->uio_iovcnt < 1) {
1021 return;
1022 }
1023
1024 if (UIO_IS_USER_SPACE(a_uio)) {
1025 /*
1026 * if a_count == 0, then we are asking to skip over
1027 * any empty iovs
1028 */
1029 if (a_count) {
1030 if (a_count > a_uio->uio_iovs.uiovp->iov_len) {
1031 a_uio->uio_iovs.uiovp->iov_base += a_uio->uio_iovs.uiovp->iov_len;
1032 a_uio->uio_iovs.uiovp->iov_len = 0;
1033 } else {
1034 a_uio->uio_iovs.uiovp->iov_base += a_count;
1035 a_uio->uio_iovs.uiovp->iov_len -= a_count;
1036 }
1037 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1038 a_uio->uio_offset += a_uio->uio_resid_64;
1039 a_uio->uio_resid_64 = 0;
1040 } else {
1041 a_uio->uio_offset += a_count;
1042 a_uio->uio_resid_64 -= a_count;
1043 }
1044 }
1045 /*
1046 * advance to next iovec if current one is totally consumed
1047 */
1048 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.uiovp->iov_len == 0) {
1049 a_uio->uio_iovcnt--;
1050 if (a_uio->uio_iovcnt > 0) {
1051 a_uio->uio_iovs.uiovp++;
1052 }
1053 }
1054 } else {
1055 /*
1056 * if a_count == 0, then we are asking to skip over
1057 * any empty iovs
1058 */
1059 if (a_count) {
1060 if (a_count > a_uio->uio_iovs.kiovp->iov_len) {
1061 a_uio->uio_iovs.kiovp->iov_base += a_uio->uio_iovs.kiovp->iov_len;
1062 a_uio->uio_iovs.kiovp->iov_len = 0;
1063 } else {
1064 a_uio->uio_iovs.kiovp->iov_base += a_count;
1065 a_uio->uio_iovs.kiovp->iov_len -= a_count;
1066 }
1067 if (a_count > (user_size_t)a_uio->uio_resid_64) {
1068 a_uio->uio_offset += a_uio->uio_resid_64;
1069 a_uio->uio_resid_64 = 0;
1070 } else {
1071 a_uio->uio_offset += a_count;
1072 a_uio->uio_resid_64 -= a_count;
1073 }
1074 }
1075 /*
1076 * advance to next iovec if current one is totally consumed
1077 */
1078 while (a_uio->uio_iovcnt > 0 && a_uio->uio_iovs.kiovp->iov_len == 0) {
1079 a_uio->uio_iovcnt--;
1080 if (a_uio->uio_iovcnt > 0) {
1081 a_uio->uio_iovs.kiovp++;
1082 }
1083 }
1084 }
1085 return;
1086 }
1087
1088 /*
1089 * uio_pushback - undo uncommitted I/O by subtracting from the
1090 * current base address and offset, and incrementing the residiual
1091 * IO. If the UIO was previously exhausted, this call will panic.
1092 * New code should not use this functionality.
1093 */
1094 __private_extern__ void
1095 uio_pushback( uio_t a_uio, user_size_t a_count )
1096 {
1097 #if LP64_DEBUG
1098 if (a_uio == NULL) {
1099 panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__);
1100 }
1101 if (UIO_IS_32_BIT_SPACE(a_uio) && a_count > 0xFFFFFFFFull) {
1102 panic("%s :%d - invalid count value \n", __FILE__, __LINE__);
1103 }
1104 #endif /* LP64_DEBUG */
1105
1106 if (a_uio == NULL || a_count == 0) {
1107 return;
1108 }
1109
1110 if (a_uio->uio_iovcnt < 1) {
1111 panic("Invalid uio for pushback");
1112 }
1113
1114 if (UIO_IS_USER_SPACE(a_uio)) {
1115 a_uio->uio_iovs.uiovp->iov_base -= a_count;
1116 a_uio->uio_iovs.uiovp->iov_len += a_count;
1117 } else {
1118 a_uio->uio_iovs.kiovp->iov_base -= a_count;
1119 a_uio->uio_iovs.kiovp->iov_len += a_count;
1120 }
1121
1122 a_uio->uio_offset -= a_count;
1123 a_uio->uio_resid_64 += a_count;
1124
1125 return;
1126 }
1127
1128
1129 /*
1130 * uio_duplicate - allocate a new uio and make a copy of the given uio_t.
1131 * may return NULL.
1132 */
1133 uio_t
1134 uio_duplicate( uio_t a_uio )
1135 {
1136 uio_t my_uio;
1137 int i;
1138
1139 if (a_uio == NULL) {
1140 return NULL;
1141 }
1142
1143 my_uio = (uio_t) kalloc(a_uio->uio_size);
1144 if (my_uio == 0) {
1145 panic("%s :%d - allocation failed\n", __FILE__, __LINE__);
1146 }
1147
1148 bcopy((void *)a_uio, (void *)my_uio, a_uio->uio_size);
1149 /* need to set our iovec pointer to point to first active iovec */
1150 if (my_uio->uio_max_iovs > 0) {
1151 my_uio->uio_iovs.uiovp = (struct user_iovec *)
1152 (((uint8_t *)my_uio) + sizeof(struct uio));
1153
1154 /* advance to first nonzero iovec */
1155 if (my_uio->uio_iovcnt > 0) {
1156 for (i = 0; i < my_uio->uio_max_iovs; i++) {
1157 if (UIO_IS_USER_SPACE(a_uio)) {
1158 if (my_uio->uio_iovs.uiovp->iov_len != 0) {
1159 break;
1160 }
1161 my_uio->uio_iovs.uiovp++;
1162 } else {
1163 if (my_uio->uio_iovs.kiovp->iov_len != 0) {
1164 break;
1165 }
1166 my_uio->uio_iovs.kiovp++;
1167 }
1168 }
1169 }
1170 }
1171
1172 my_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED;
1173 #if DEBUG
1174 os_atomic_inc(&uio_t_count, relaxed);
1175 #endif
1176
1177
1178 return my_uio;
1179 }
1180
1181 int
1182 copyin_user_iovec_array(user_addr_t uaddr, int spacetype, int count, struct user_iovec *dst)
1183 {
1184 size_t size_of_iovec = (spacetype == UIO_USERSPACE64 ? sizeof(struct user64_iovec) : sizeof(struct user32_iovec));
1185 int error;
1186 int i;
1187
1188 // copyin to the front of "dst", without regard for putting records in the right places
1189 error = copyin(uaddr, dst, count * size_of_iovec);
1190 if (error) {
1191 return error;
1192 }
1193
1194 // now, unpack the entries in reverse order, so we don't overwrite anything
1195 for (i = count - 1; i >= 0; i--) {
1196 if (spacetype == UIO_USERSPACE64) {
1197 struct user64_iovec iovec = ((struct user64_iovec *)dst)[i];
1198 dst[i].iov_base = iovec.iov_base;
1199 dst[i].iov_len = iovec.iov_len;
1200 } else {
1201 struct user32_iovec iovec = ((struct user32_iovec *)dst)[i];
1202 dst[i].iov_base = iovec.iov_base;
1203 dst[i].iov_len = iovec.iov_len;
1204 }
1205 }
1206
1207 return 0;
1208 }