2 * Copyright (c) 2011-2014 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <nfs/nfs_conf.h>
33 #include <sys/param.h>
34 #include <sys/mount_internal.h>
35 #include <sys/malloc.h>
36 #include <sys/queue.h>
38 #include <libkern/libkern.h>
39 #include <libkern/OSAtomic.h>
40 #include <kern/debug.h>
41 #include <kern/thread.h>
43 #include <nfs/rpcv2.h>
44 #include <nfs/nfsproto.h>
48 #define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
50 #define DPRINT(fmt, ...)
54 TAILQ_ENTRY(nfsrv_uc_arg
) nua_svcq
;
56 struct nfsrv_sock
*nua_slp
;
57 int nua_waitflag
; /* Should always be MBUF_DONTWAIT */
61 #define NFS_UC_QUEUED 0x0001
63 #define NFS_UC_HASH_SZ 7
64 #define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)
66 TAILQ_HEAD(nfsrv_uc_q
, nfsrv_uc_arg
);
68 static struct nfsrv_uc_queue
{
70 struct nfsrv_uc_q ucq_queue
[1];
73 } nfsrv_uc_queue_tbl
[NFS_UC_HASH_SZ
];
74 #define NFS_UC_QUEUE_SLEEPING 0x0001
76 static LCK_GRP_DECLARE(nfsrv_uc_group
, "nfs_upcall_locks");
77 static LCK_MTX_DECLARE(nfsrv_uc_shutdown_lock
, &nfsrv_uc_group
);
78 static volatile int nfsrv_uc_shutdown
= 0;
79 static int32_t nfsrv_uc_thread_count
;
81 extern kern_return_t
thread_terminate(thread_t
);
84 int nfsrv_uc_use_proxy
= 1;
85 uint32_t nfsrv_uc_queue_limit
;
86 uint32_t nfsrv_uc_queue_max_seen
;
87 volatile uint32_t nfsrv_uc_queue_count
;
91 * Thread that dequeues up-calls and runs the nfsrv_rcv routine
94 nfsrv_uc_thread(void *arg
, wait_result_t wr __unused
)
96 int qi
= (int)(uintptr_t)arg
;
98 struct nfsrv_uc_arg
*ep
= NULL
;
99 struct nfsrv_uc_queue
*myqueue
= &nfsrv_uc_queue_tbl
[qi
];
101 DPRINT("nfsrv_uc_thread %d started\n", qi
);
102 while (!nfsrv_uc_shutdown
) {
103 lck_mtx_lock(&myqueue
->ucq_lock
);
105 while (!nfsrv_uc_shutdown
&& TAILQ_EMPTY(myqueue
->ucq_queue
)) {
106 myqueue
->ucq_flags
|= NFS_UC_QUEUE_SLEEPING
;
107 error
= msleep(myqueue
, &myqueue
->ucq_lock
, PSOCK
, "nfsd_upcall_handler", NULL
);
108 myqueue
->ucq_flags
&= ~NFS_UC_QUEUE_SLEEPING
;
110 printf("nfsrv_uc_thread received error %d\n", error
);
113 if (nfsrv_uc_shutdown
) {
114 lck_mtx_unlock(&myqueue
->ucq_lock
);
119 ep
= TAILQ_FIRST(myqueue
->ucq_queue
);
120 DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi
, ep
, myqueue
);
122 TAILQ_REMOVE(myqueue
->ucq_queue
, ep
, nua_svcq
);
124 ep
->nua_flags
&= ~NFS_UC_QUEUED
;
126 lck_mtx_unlock(&myqueue
->ucq_lock
);
128 #ifdef NFS_UC_Q_DEBUG
129 OSDecrementAtomic(&nfsrv_uc_queue_count
);
132 DPRINT("calling nfsrv_rcv for %p\n", (void *)ep
->nua_slp
);
133 nfsrv_rcv(ep
->nua_so
, (void *)ep
->nua_slp
, ep
->nua_waitflag
);
136 lck_mtx_lock(&nfsrv_uc_shutdown_lock
);
137 nfsrv_uc_thread_count
--;
138 wakeup(&nfsrv_uc_thread_count
);
139 lck_mtx_unlock(&nfsrv_uc_shutdown_lock
);
141 thread_terminate(current_thread());
145 * Dequeue a closed nfsrv_sock if needed from the up-call queue.
146 * Call from nfsrv_zapsock
149 nfsrv_uc_dequeue(struct nfsrv_sock
*slp
)
151 struct nfsrv_uc_arg
*ap
= slp
->ns_ua
;
152 struct nfsrv_uc_queue
*myqueue
= &nfsrv_uc_queue_tbl
[ap
->nua_qi
];
155 * We assume that the socket up-calls have been stop and the socket
156 * is shutting down so no need for acquiring the lock to check that
157 * the flag is cleared.
159 if (ap
== NULL
|| (ap
->nua_flags
& NFS_UC_QUEUED
) == 0) {
162 /* If we're queued we might race with nfsrv_uc_thread */
163 lck_mtx_lock(&myqueue
->ucq_lock
);
164 if (ap
->nua_flags
& NFS_UC_QUEUED
) {
165 printf("nfsrv_uc_dequeue remove %p\n", ap
);
166 TAILQ_REMOVE(myqueue
->ucq_queue
, ap
, nua_svcq
);
167 ap
->nua_flags
&= ~NFS_UC_QUEUED
;
168 #ifdef NFS_UC_Q_DEBUG
169 OSDecrementAtomic(&nfsrv_uc_queue_count
);
172 FREE(slp
->ns_ua
, M_TEMP
);
174 lck_mtx_unlock(&myqueue
->ucq_lock
);
178 * Allocate and initialize globals for nfsrv_sock up-call support.
183 for (int i
= 0; i
< NFS_UC_HASH_SZ
; i
++) {
184 TAILQ_INIT(nfsrv_uc_queue_tbl
[i
].ucq_queue
);
185 lck_mtx_init(&nfsrv_uc_queue_tbl
[i
].ucq_lock
, &nfsrv_uc_group
, LCK_ATTR_NULL
);
186 nfsrv_uc_queue_tbl
[i
].ucq_thd
= THREAD_NULL
;
187 nfsrv_uc_queue_tbl
[i
].ucq_flags
= 0;
192 * Start up-call threads to service nfsrv_sock(s)
193 * Called from the first call of nfsrv_uc_addsock
201 #ifdef NFS_UC_Q_DEBUG
202 if (!nfsrv_uc_use_proxy
) {
206 DPRINT("nfsrv_uc_start\n");
208 /* Wait until previous shutdown finishes */
209 lck_mtx_lock(&nfsrv_uc_shutdown_lock
);
210 while (nfsrv_uc_shutdown
|| nfsrv_uc_thread_count
> 0) {
211 msleep(&nfsrv_uc_thread_count
, &nfsrv_uc_shutdown_lock
, PSOCK
, "nfsd_upcall_shutdown_wait", NULL
);
214 /* Start up-call threads */
215 for (i
= 0; i
< NFS_UC_HASH_SZ
; i
++) {
216 error
= kernel_thread_start(nfsrv_uc_thread
, (void *)(uintptr_t)i
, &nfsrv_uc_queue_tbl
[nfsrv_uc_thread_count
].ucq_thd
);
218 nfsrv_uc_thread_count
++;
220 printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error
);
223 if (nfsrv_uc_thread_count
== 0) {
224 printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
229 #ifdef NFS_UC_Q_DEBUG
230 nfsrv_uc_queue_count
= 0ULL;
231 nfsrv_uc_queue_max_seen
= 0ULL;
233 lck_mtx_unlock(&nfsrv_uc_shutdown_lock
);
237 * Stop the up-call threads.
238 * Called from nfsrv_uc_cleanup.
244 int32_t thread_count
= nfsrv_uc_thread_count
;
246 DPRINT("Entering nfsrv_uc_stop\n");
248 /* Signal up-call threads to stop */
249 nfsrv_uc_shutdown
= 1;
250 for (i
= 0; i
< thread_count
; i
++) {
251 lck_mtx_lock(&nfsrv_uc_queue_tbl
[i
].ucq_lock
);
252 wakeup(&nfsrv_uc_queue_tbl
[i
]);
253 lck_mtx_unlock(&nfsrv_uc_queue_tbl
[i
].ucq_lock
);
256 /* Wait until they are done shutting down */
257 lck_mtx_lock(&nfsrv_uc_shutdown_lock
);
258 while (nfsrv_uc_thread_count
> 0) {
259 msleep(&nfsrv_uc_thread_count
, &nfsrv_uc_shutdown_lock
, PSOCK
, "nfsd_upcall_shutdown_stop", NULL
);
262 /* Deallocate old threads */
263 for (i
= 0; i
< nfsrv_uc_thread_count
; i
++) {
264 if (nfsrv_uc_queue_tbl
[i
].ucq_thd
!= THREAD_NULL
) {
265 thread_deallocate(nfsrv_uc_queue_tbl
[i
].ucq_thd
);
267 nfsrv_uc_queue_tbl
[i
].ucq_thd
= THREAD_NULL
;
270 /* Enable restarting */
271 nfsrv_uc_shutdown
= 0;
272 lck_mtx_unlock(&nfsrv_uc_shutdown_lock
);
276 * Shutdown up-calls for nfsrv_socks.
277 * Make sure nothing is queued on the up-call queues
278 * Shutdown the up-call threads
279 * Called from nfssvc_cleanup.
282 nfsrv_uc_cleanup(void)
286 DPRINT("Entering nfsrv_uc_cleanup\n");
289 * Every thing should be dequeued at this point or will be as sockets are closed
290 * but to be safe, we'll make sure.
292 for (i
= 0; i
< NFS_UC_HASH_SZ
; i
++) {
293 struct nfsrv_uc_queue
*queue
= &nfsrv_uc_queue_tbl
[i
];
295 lck_mtx_lock(&queue
->ucq_lock
);
296 while (!TAILQ_EMPTY(queue
->ucq_queue
)) {
297 struct nfsrv_uc_arg
*ep
= TAILQ_FIRST(queue
->ucq_queue
);
298 TAILQ_REMOVE(queue
->ucq_queue
, ep
, nua_svcq
);
299 ep
->nua_flags
&= ~NFS_UC_QUEUED
;
301 lck_mtx_unlock(&queue
->ucq_lock
);
308 * This is the nfs up-call routine for server sockets.
309 * We used to set nfsrv_rcv as the up-call routine, but
310 * recently that seems like we are doing to much work for
311 * the interface thread, so we just queue the arguments
312 * that we would have gotten for nfsrv_rcv and let a
313 * worker thread dequeue them and pass them on to nfsrv_rcv.
316 nfsrv_uc_proxy(socket_t so
, void *arg
, int waitflag
)
318 struct nfsrv_uc_arg
*uap
= (struct nfsrv_uc_arg
*)arg
;
319 int qi
= uap
->nua_qi
;
320 struct nfsrv_uc_queue
*myqueue
= &nfsrv_uc_queue_tbl
[qi
];
322 lck_mtx_lock(&myqueue
->ucq_lock
);
323 DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap
, uap
->nua_slp
);
324 DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi
, myqueue
);
325 if (uap
== NULL
|| uap
->nua_flags
& NFS_UC_QUEUED
) {
326 lck_mtx_unlock(&myqueue
->ucq_lock
);
327 return; /* Already queued or freed */
331 uap
->nua_waitflag
= waitflag
;
333 TAILQ_INSERT_TAIL(myqueue
->ucq_queue
, uap
, nua_svcq
);
335 uap
->nua_flags
|= NFS_UC_QUEUED
;
336 if (myqueue
->ucq_flags
& NFS_UC_QUEUE_SLEEPING
) {
340 #ifdef NFS_UC_Q_DEBUG
342 uint32_t count
= OSIncrementAtomic(&nfsrv_uc_queue_count
);
344 /* This is a bit racey but just for debug */
345 if (count
> nfsrv_uc_queue_max_seen
) {
346 nfsrv_uc_queue_max_seen
= count
;
349 if (nfsrv_uc_queue_limit
&& count
> nfsrv_uc_queue_limit
) {
350 panic("nfsd up-call queue limit exceeded\n");
354 lck_mtx_unlock(&myqueue
->ucq_lock
);
359 * Set the up-call routine on the socket associated with the passed in
361 * Assumes nfsd_mutex is held.
364 nfsrv_uc_addsock(struct nfsrv_sock
*slp
, int start
)
367 struct nfsrv_uc_arg
*arg
;
369 if (start
&& nfsrv_uc_thread_count
== 0) {
374 * We don't take a lock since once we're up nfsrv_uc_thread_count does
375 * not change until shutdown and then we should not be adding sockets to
378 if (nfsrv_uc_thread_count
) {
379 MALLOC(arg
, struct nfsrv_uc_arg
*, sizeof(struct nfsrv_uc_arg
), M_TEMP
, M_WAITOK
| M_ZERO
);
386 arg
->nua_qi
= NFS_UC_HASH(slp
);
388 sock_setupcall(slp
->ns_so
, nfsrv_uc_proxy
, arg
);
392 DPRINT("setting nfsrv_rcv up-call\n");
393 sock_setupcall(slp
->ns_so
, nfsrv_rcv
, slp
);
396 /* just playin' it safe */
397 sock_setsockopt(slp
->ns_so
, SOL_SOCKET
, SO_UPCALLCLOSEWAIT
, &on
, sizeof(on
));
402 #endif /* CONFIG_NFS_SERVER */