bsd/nfs/nfs_upcall.c

   1 /*
   2  * Copyright (c) 2011 Apple Inc.  All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <stdint.h>
  29 #include <sys/param.h>
  30 #include <sys/mount_internal.h>
  31 #include <sys/malloc.h>
  32 #include <sys/queue.h>
  33
  34 #include <libkern/libkern.h>
  35 #include <libkern/OSAtomic.h>
  36 #include <kern/debug.h>
  37 #include <kern/thread.h>
  38
  39 #include <nfs/rpcv2.h>
  40 #include <nfs/nfsproto.h>
  41 #include <nfs/nfs.h>
  42
  43 #ifdef NFS_UC_DEBUG
  44 #define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
  45 #else
  46 #define DPRINT(fmt, ...)
  47 #endif
  48
  49 struct nfsrv_uc_arg {
  50         TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq;
  51         socket_t nua_so;
  52         struct nfsrv_sock *nua_slp;
  53         int nua_waitflag;  /* Should always be MBUF_DONTWAIT */
  54         uint32_t nua_flags;
  55         uint32_t nua_qi;
  56 };
  57 #define NFS_UC_QUEUED   0x0001
  58
  59 #define NFS_UC_HASH_SZ 7
  60 #define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)
  61
  62 TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg);
  63
  64 static struct nfsrv_uc_queue {
  65         lck_mtx_t               *ucq_lock;
  66         struct nfsrv_uc_q       ucq_queue[1];
  67         thread_t                ucq_thd;
  68         uint32_t                ucq_flags;
  69 } nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ];
  70 #define NFS_UC_QUEUE_SLEEPING   0x0001
  71
  72 static lck_grp_t *nfsrv_uc_group;
  73 static lck_mtx_t *nfsrv_uc_shutdown_lock;
  74 static volatile int nfsrv_uc_shutdown = 0;
  75 static int32_t nfsrv_uc_thread_count;
  76
  77 extern kern_return_t thread_terminate(thread_t);
  78
  79 #ifdef NFS_UC_Q_DEBUG
  80 int nfsrv_uc_use_proxy = 1;
  81 uint32_t nfsrv_uc_queue_limit;
  82 uint32_t nfsrv_uc_queue_max_seen;
  83 volatile uint32_t nfsrv_uc_queue_count;
  84 #endif
  85
  86 /*
  87  * Thread that dequeues up-calls and runs the nfsrv_rcv routine
  88  */
  89 static void
  90 nfsrv_uc_thread(void *arg, wait_result_t wr __unused)
  91 {
  92         int qi = (int)(uintptr_t)arg;
  93         int error;
  94         struct nfsrv_uc_arg *ep = NULL;
  95         struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
  96
  97         DPRINT("nfsrv_uc_thread %d started\n", qi);
  98         while (!nfsrv_uc_shutdown) {
  99                 lck_mtx_lock(myqueue->ucq_lock);
 100
 101                 while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) {
 102                         myqueue->ucq_flags |= NFS_UC_QUEUE_SLEEPING;
 103                         error = msleep(myqueue, myqueue->ucq_lock, PSOCK, "nfsd_upcall_handler", NULL);
 104                         myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING;
 105                         if (error) {
 106                                 printf("nfsrv_uc_thread received error %d\n", error);
 107                         }
 108                 }
 109                 if (nfsrv_uc_shutdown) {
 110                         lck_mtx_unlock(myqueue->ucq_lock);
 111                         break;
 112                 }
 113
 114
 115                 ep = TAILQ_FIRST(myqueue->ucq_queue);
 116                 DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi, ep, myqueue);
 117
 118                 TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq);
 119
 120                 ep->nua_flags &= ~NFS_UC_QUEUED;
 121
 122                 lck_mtx_unlock(myqueue->ucq_lock);
 123
 124 #ifdef NFS_UC_Q_DEBUG
 125                 OSDecrementAtomic(&nfsrv_uc_queue_count);
 126 #endif
 127
 128                 DPRINT("calling nfsrv_rcv for %p\n", (void *)ep->nua_slp);
 129                 nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag);
 130         }
 131
 132         lck_mtx_lock(nfsrv_uc_shutdown_lock);
 133         nfsrv_uc_thread_count--;
 134         wakeup(&nfsrv_uc_thread_count);
 135         lck_mtx_unlock(nfsrv_uc_shutdown_lock);
 136
 137         thread_terminate(current_thread());
 138 }
 139
 140 /*
 141  * Dequeue a closed nfsrv_sock if needed from the up-call queue.
 142  * Call from nfsrv_zapsock
 143  */
 144 void
 145 nfsrv_uc_dequeue(struct nfsrv_sock *slp)
 146 {
 147         struct nfsrv_uc_arg *ap = slp->ns_ua;
 148         struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi];
 149
 150         /*
 151          * We assume that the socket up-calls have been stop and the socket
 152          * is shutting down so no need for acquiring the lock to check that
 153          * the flag is cleared.
 154          */
 155         if (ap == NULL || (ap->nua_flags & NFS_UC_QUEUED) == 0)
 156                 return;
 157         /* If we're queued we might race with nfsrv_uc_thread */
 158         lck_mtx_lock(myqueue->ucq_lock);
 159         if (ap->nua_flags & NFS_UC_QUEUED) {
 160                 printf("nfsrv_uc_dequeue remove %p\n", ap);
 161                 TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq);
 162                 ap->nua_flags &= ~NFS_UC_QUEUED;
 163 #ifdef NFS_UC_Q_DEBUG
 164                 OSDecrementAtomic(&nfsrv_uc_queue_count);
 165 #endif
 166         }
 167         lck_mtx_unlock(myqueue->ucq_lock);
 168 }
 169
 170 /*
 171  * Allocate and initialize globals for nfsrv_sock up-call support.
 172  */
 173 void
 174 nfsrv_uc_init(void)
 175 {
 176         int i;
 177
 178         nfsrv_uc_group = lck_grp_alloc_init("nfs_upcall_locks", LCK_GRP_ATTR_NULL);
 179         for (i = 0; i < NFS_UC_HASH_SZ; i++) {
 180                 TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue);
 181                 nfsrv_uc_queue_tbl[i].ucq_lock = lck_mtx_alloc_init(nfsrv_uc_group, LCK_ATTR_NULL);
 182                 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
 183                 nfsrv_uc_queue_tbl[i].ucq_flags = 0;
 184         }
 185         nfsrv_uc_shutdown_lock = lck_mtx_alloc_init(nfsrv_uc_group, LCK_ATTR_NULL);
 186 }
 187
 188 /*
 189  * Start up-call threads to service nfsrv_sock(s)
 190  * Called from the first call of nfsrv_uc_addsock
 191  */
 192 static void
 193 nfsrv_uc_start(void)
 194 {
 195         int32_t i;
 196         int error;
 197
 198 #ifdef NFS_UC_Q_DEBUG
 199         if (!nfsrv_uc_use_proxy)
 200                 return;
 201 #endif
 202         DPRINT("nfsrv_uc_start\n");
 203
 204         /* Wait until previous shutdown finishes */
 205         lck_mtx_lock(nfsrv_uc_shutdown_lock);
 206         while (nfsrv_uc_shutdown || nfsrv_uc_thread_count > 0)
 207                 msleep(&nfsrv_uc_thread_count, nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_wait", NULL);
 208
 209         /* Start up-call threads */
 210         for (i = 0; i < NFS_UC_HASH_SZ; i++) {
 211                 error = kernel_thread_start(nfsrv_uc_thread, (void *)(uintptr_t)i, &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd);
 212                 if (!error) {
 213                         nfsrv_uc_thread_count++;
 214                 } else {
 215                         printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error);
 216                 }
 217         }
 218         if (nfsrv_uc_thread_count == 0) {
 219                 printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
 220                 goto out;
 221         }
 222
 223 out:
 224 #ifdef NFS_UC_Q_DEBUG
 225         nfsrv_uc_queue_count = 0ULL;
 226         nfsrv_uc_queue_max_seen = 0ULL;
 227 #endif
 228         lck_mtx_unlock(nfsrv_uc_shutdown_lock);
 229 }
 230
 231 /*
 232  * Stop the up-call threads.
 233  * Called from nfsrv_uc_cleanup.
 234  */
 235 static void
 236 nfsrv_uc_stop(void)
 237 {
 238         int32_t i;
 239         int32_t thread_count = nfsrv_uc_thread_count;
 240
 241         DPRINT("Entering nfsrv_uc_stop\n");
 242
 243         /* Signal up-call threads to stop */
 244         nfsrv_uc_shutdown = 1;
 245         for (i = 0; i < thread_count; i++) {
 246                 lck_mtx_lock(nfsrv_uc_queue_tbl[i].ucq_lock);
 247                 wakeup(&nfsrv_uc_queue_tbl[i]);
 248                 lck_mtx_unlock(nfsrv_uc_queue_tbl[i].ucq_lock);
 249         }
 250
 251         /* Wait until they are done shutting down */
 252         lck_mtx_lock(nfsrv_uc_shutdown_lock);
 253         while (nfsrv_uc_thread_count > 0)
 254                 msleep(&nfsrv_uc_thread_count, nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_stop", NULL);
 255
 256         /* Deallocate old threads */
 257         for (i = 0; i < nfsrv_uc_thread_count; i++) {
 258                 if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL)
 259                         thread_deallocate(nfsrv_uc_queue_tbl[i].ucq_thd);
 260                 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
 261         }
 262
 263         /* Enable restarting */
 264         nfsrv_uc_shutdown = 0;
 265         lck_mtx_unlock(nfsrv_uc_shutdown_lock);
 266 }
 267
 268 /*
 269  * Shutdown up-calls for nfsrv_socks.
 270  *      Make sure nothing is queued on the up-call queues
 271  *      Shutdown the up-call threads
 272  * Called from nfssvc_cleanup.
 273  */
 274 void
 275 nfsrv_uc_cleanup(void)
 276 {
 277         int i;
 278
 279         DPRINT("Entering nfsrv_uc_cleanup\n");
 280
 281         /*
 282          * Every thing should be dequeued at this point or will be as sockets are closed
 283          * but to be safe, we'll make sure.
 284          */
 285         for (i = 0; i < NFS_UC_HASH_SZ; i++) {
 286                 struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i];
 287
 288                 lck_mtx_lock(queue->ucq_lock);
 289                 while (!TAILQ_EMPTY(queue->ucq_queue)) {
 290                         struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue);
 291                         TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq);
 292                         ep->nua_flags &= ~NFS_UC_QUEUED;
 293                 }
 294                 lck_mtx_unlock(queue->ucq_lock);
 295         }
 296
 297         nfsrv_uc_stop();
 298 }
 299
 300 /*
 301  * This is the nfs up-call routine for server sockets.
 302  * We used to set nfsrv_rcv as the up-call routine, but
 303  * recently that seems like we are doing to much work for
 304  * the interface thread, so we just queue the arguments
 305  * that we would have gotten for nfsrv_rcv and let a
 306  * worker thread dequeue them and pass them on to nfsrv_rcv.
 307  */
 308 static void
 309 nfsrv_uc_proxy(socket_t so, void *arg, int waitflag)
 310 {
 311         struct nfsrv_uc_arg *uap = (struct nfsrv_uc_arg *)arg;
 312         int qi = uap->nua_qi;
 313         struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
 314
 315         lck_mtx_lock(myqueue->ucq_lock);
 316         DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap, uap->nua_slp);
 317         DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi, myqueue);
 318         if (uap->nua_flags & NFS_UC_QUEUED) {
 319                 lck_mtx_unlock(myqueue->ucq_lock);
 320                 return;  /* Already queued */
 321         }
 322
 323         uap->nua_so = so;
 324         uap->nua_waitflag = waitflag;
 325
 326         TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq);
 327
 328         uap->nua_flags |= NFS_UC_QUEUED;
 329         if (myqueue->ucq_flags | NFS_UC_QUEUE_SLEEPING)
 330                 wakeup(myqueue);
 331
 332 #ifdef NFS_UC_Q_DEBUG
 333         {
 334                 uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count);
 335
 336                 /* This is a bit racey but just for debug */
 337                 if (count > nfsrv_uc_queue_max_seen)
 338                         nfsrv_uc_queue_max_seen = count;
 339
 340                 if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) {
 341                         panic("nfsd up-call queue limit exceeded\n");
 342                 }
 343         }
 344 #endif
 345         lck_mtx_unlock(myqueue->ucq_lock);
 346 }
 347
 348
 349 /*
 350  * Set the up-call routine on the socket associated with the passed in
 351  * nfsrv_sock.
 352  * Assumes nfsd_mutex is held.
 353  */
 354 void
 355 nfsrv_uc_addsock(struct nfsrv_sock *slp, int start)
 356 {
 357         int on = 1;
 358         struct nfsrv_uc_arg *arg;
 359
 360         if (start && nfsrv_uc_thread_count == 0)
 361                 nfsrv_uc_start();
 362
 363         /*
 364          * We don't take a lock since once we're up nfsrv_uc_thread_count does
 365          * not change until shutdown and then we should not be adding sockets to
 366          * generate up-calls.
 367          */
 368         if (nfsrv_uc_thread_count) {
 369                 MALLOC(arg, struct nfsrv_uc_arg *, sizeof (struct nfsrv_uc_arg), M_NFSSVC, M_WAITOK | M_ZERO);
 370                 if (arg == NULL)
 371                         goto direct;
 372
 373                 slp->ns_ua = arg;
 374                 arg->nua_slp = slp;
 375                 arg->nua_qi = NFS_UC_HASH(slp);
 376
 377                 sock_setupcall(slp->ns_so, nfsrv_uc_proxy, arg);
 378         } else {
 379 direct:
 380                 slp->ns_ua = NULL;
 381                 DPRINT("setting nfsrv_rcv up-call\n");
 382                 sock_setupcall(slp->ns_so, nfsrv_rcv, slp);
 383         }
 384
 385         /* just playin' it safe */
 386         sock_setsockopt(slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, &on, sizeof(on));
 387
 388         return;
 389 }
 390