[apple/xnu.git] / bsd / nfs / nfs_upcall.c

/*
 * Copyright (c) 2011-2014 Apple Inc.  All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <nfs/nfs_conf.h>
#if CONFIG_NFS_SERVER

#include <stdint.h>
#include <sys/param.h>
#include <sys/mount_internal.h>
#include <sys/malloc.h>
#include <sys/queue.h>

#include <libkern/libkern.h>
#include <libkern/OSAtomic.h>
#include <kern/debug.h>
#include <kern/thread.h>

#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
#include <nfs/nfs.h>

#ifdef NFS_UC_DEBUG
#define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
#else
#define DPRINT(fmt, ...)
#endif

struct nfsrv_uc_arg {
	TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq;
	socket_t nua_so;
	struct nfsrv_sock *nua_slp;
	int nua_waitflag;  /* Should always be MBUF_DONTWAIT */
	uint32_t nua_flags;
	uint32_t nua_qi;
};
#define NFS_UC_QUEUED   0x0001

#define NFS_UC_HASH_SZ 7
#define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)

TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg);

static struct nfsrv_uc_queue {
	lck_mtx_t               ucq_lock;
	struct nfsrv_uc_q       ucq_queue[1];
	thread_t                ucq_thd;
	uint32_t                ucq_flags;
} nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ];
#define NFS_UC_QUEUE_SLEEPING   0x0001

static LCK_GRP_DECLARE(nfsrv_uc_group, "nfs_upcall_locks");
static LCK_MTX_DECLARE(nfsrv_uc_shutdown_lock, &nfsrv_uc_group);
static volatile int nfsrv_uc_shutdown = 0;
static int32_t nfsrv_uc_thread_count;

extern kern_return_t thread_terminate(thread_t);

#ifdef NFS_UC_Q_DEBUG
int nfsrv_uc_use_proxy = 1;
uint32_t nfsrv_uc_queue_limit;
uint32_t nfsrv_uc_queue_max_seen;
volatile uint32_t nfsrv_uc_queue_count;
#endif

/*
 * Thread that dequeues up-calls and runs the nfsrv_rcv routine
 */
static void
nfsrv_uc_thread(void *arg, wait_result_t wr __unused)
{
	int qi = (int)(uintptr_t)arg;
	int error;
	struct nfsrv_uc_arg *ep = NULL;
	struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];

	DPRINT("nfsrv_uc_thread %d started\n", qi);
	while (!nfsrv_uc_shutdown) {
		lck_mtx_lock(&myqueue->ucq_lock);

		while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) {
			myqueue->ucq_flags |= NFS_UC_QUEUE_SLEEPING;
			error = msleep(myqueue, &myqueue->ucq_lock, PSOCK, "nfsd_upcall_handler", NULL);
			myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING;
			if (error) {
				printf("nfsrv_uc_thread received error %d\n", error);
			}
		}
		if (nfsrv_uc_shutdown) {
			lck_mtx_unlock(&myqueue->ucq_lock);
			break;
		}


		ep = TAILQ_FIRST(myqueue->ucq_queue);
		DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi, ep, myqueue);

		TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq);

		ep->nua_flags &= ~NFS_UC_QUEUED;

		lck_mtx_unlock(&myqueue->ucq_lock);

#ifdef NFS_UC_Q_DEBUG
		OSDecrementAtomic(&nfsrv_uc_queue_count);
#endif

		DPRINT("calling nfsrv_rcv for %p\n", (void *)ep->nua_slp);
		nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag);
	}

	lck_mtx_lock(&nfsrv_uc_shutdown_lock);
	nfsrv_uc_thread_count--;
	wakeup(&nfsrv_uc_thread_count);
	lck_mtx_unlock(&nfsrv_uc_shutdown_lock);

	thread_terminate(current_thread());
}

/*
 * Dequeue a closed nfsrv_sock if needed from the up-call queue.
 * Call from nfsrv_zapsock
 */
void
nfsrv_uc_dequeue(struct nfsrv_sock *slp)
{
	struct nfsrv_uc_arg *ap = slp->ns_ua;
	struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi];

	/*
	 * We assume that the socket up-calls have been stop and the socket
	 * is shutting down so no need for acquiring the lock to check that
	 * the flag is cleared.
	 */
	if (ap == NULL || (ap->nua_flags & NFS_UC_QUEUED) == 0) {
		return;
	}
	/* If we're queued we might race with nfsrv_uc_thread */
	lck_mtx_lock(&myqueue->ucq_lock);
	if (ap->nua_flags & NFS_UC_QUEUED) {
		printf("nfsrv_uc_dequeue remove %p\n", ap);
		TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq);
		ap->nua_flags &= ~NFS_UC_QUEUED;
#ifdef NFS_UC_Q_DEBUG
		OSDecrementAtomic(&nfsrv_uc_queue_count);
#endif
	}
	FREE(slp->ns_ua, M_TEMP);
	slp->ns_ua = NULL;
	lck_mtx_unlock(&myqueue->ucq_lock);
}

/*
 * Allocate and initialize globals for nfsrv_sock up-call support.
 */
void
nfsrv_uc_init(void)
{
	for (int i = 0; i < NFS_UC_HASH_SZ; i++) {
		TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue);
		lck_mtx_init(&nfsrv_uc_queue_tbl[i].ucq_lock, &nfsrv_uc_group, LCK_ATTR_NULL);
		nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
		nfsrv_uc_queue_tbl[i].ucq_flags = 0;
	}
}

/*
 * Start up-call threads to service nfsrv_sock(s)
 * Called from the first call of nfsrv_uc_addsock
 */
static void
nfsrv_uc_start(void)
{
	int32_t i;
	int error;

#ifdef NFS_UC_Q_DEBUG
	if (!nfsrv_uc_use_proxy) {
		return;
	}
#endif
	DPRINT("nfsrv_uc_start\n");

	/* Wait until previous shutdown finishes */
	lck_mtx_lock(&nfsrv_uc_shutdown_lock);
	while (nfsrv_uc_shutdown || nfsrv_uc_thread_count > 0) {
		msleep(&nfsrv_uc_thread_count, &nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_wait", NULL);
	}

	/* Start up-call threads */
	for (i = 0; i < NFS_UC_HASH_SZ; i++) {
		error = kernel_thread_start(nfsrv_uc_thread, (void *)(uintptr_t)i, &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd);
		if (!error) {
			nfsrv_uc_thread_count++;
		} else {
			printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error);
		}
	}
	if (nfsrv_uc_thread_count == 0) {
		printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
		goto out;
	}

out:
#ifdef NFS_UC_Q_DEBUG
	nfsrv_uc_queue_count = 0ULL;
	nfsrv_uc_queue_max_seen = 0ULL;
#endif
	lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
}

/*
 * Stop the up-call threads.
 * Called from nfsrv_uc_cleanup.
 */
static void
nfsrv_uc_stop(void)
{
	int32_t i;
	int32_t thread_count = nfsrv_uc_thread_count;

	DPRINT("Entering nfsrv_uc_stop\n");

	/* Signal up-call threads to stop */
	nfsrv_uc_shutdown = 1;
	for (i = 0; i < thread_count; i++) {
		lck_mtx_lock(&nfsrv_uc_queue_tbl[i].ucq_lock);
		wakeup(&nfsrv_uc_queue_tbl[i]);
		lck_mtx_unlock(&nfsrv_uc_queue_tbl[i].ucq_lock);
	}

	/* Wait until they are done shutting down */
	lck_mtx_lock(&nfsrv_uc_shutdown_lock);
	while (nfsrv_uc_thread_count > 0) {
		msleep(&nfsrv_uc_thread_count, &nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_stop", NULL);
	}

	/* Deallocate old threads */
	for (i = 0; i < nfsrv_uc_thread_count; i++) {
		if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL) {
			thread_deallocate(nfsrv_uc_queue_tbl[i].ucq_thd);
		}
		nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
	}

	/* Enable restarting */
	nfsrv_uc_shutdown = 0;
	lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
}

/*
 * Shutdown up-calls for nfsrv_socks.
 *	Make sure nothing is queued on the up-call queues
 *	Shutdown the up-call threads
 * Called from nfssvc_cleanup.
 */
void
nfsrv_uc_cleanup(void)
{
	int i;

	DPRINT("Entering nfsrv_uc_cleanup\n");

	/*
	 * Every thing should be dequeued at this point or will be as sockets are closed
	 * but to be safe, we'll make sure.
	 */
	for (i = 0; i < NFS_UC_HASH_SZ; i++) {
		struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i];

		lck_mtx_lock(&queue->ucq_lock);
		while (!TAILQ_EMPTY(queue->ucq_queue)) {
			struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue);
			TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq);
			ep->nua_flags &= ~NFS_UC_QUEUED;
		}
		lck_mtx_unlock(&queue->ucq_lock);
	}

	nfsrv_uc_stop();
}

/*
 * This is the nfs up-call routine for server sockets.
 * We used to set nfsrv_rcv as the up-call routine, but
 * recently that seems like we are doing to much work for
 * the interface thread, so we just queue the arguments
 * that we would have gotten for nfsrv_rcv and let a
 * worker thread dequeue them and pass them on to nfsrv_rcv.
 */
static void
nfsrv_uc_proxy(socket_t so, void *arg, int waitflag)
{
	struct nfsrv_uc_arg *uap = (struct nfsrv_uc_arg *)arg;
	int qi = uap->nua_qi;
	struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];

	lck_mtx_lock(&myqueue->ucq_lock);
	DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap, uap->nua_slp);
	DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi, myqueue);
	if (uap == NULL || uap->nua_flags & NFS_UC_QUEUED) {
		lck_mtx_unlock(&myqueue->ucq_lock);
		return;  /* Already queued or freed */
	}

	uap->nua_so = so;
	uap->nua_waitflag = waitflag;

	TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq);

	uap->nua_flags |= NFS_UC_QUEUED;
	if (myqueue->ucq_flags & NFS_UC_QUEUE_SLEEPING) {
		wakeup(myqueue);
	}

#ifdef NFS_UC_Q_DEBUG
	{
		uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count);

		/* This is a bit racey but just for debug */
		if (count > nfsrv_uc_queue_max_seen) {
			nfsrv_uc_queue_max_seen = count;
		}

		if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) {
			panic("nfsd up-call queue limit exceeded\n");
		}
	}
#endif
	lck_mtx_unlock(&myqueue->ucq_lock);
}


/*
 * Set the up-call routine on the socket associated with the passed in
 * nfsrv_sock.
 * Assumes nfsd_mutex is held.
 */
void
nfsrv_uc_addsock(struct nfsrv_sock *slp, int start)
{
	int on = 1;
	struct nfsrv_uc_arg *arg;

	if (start && nfsrv_uc_thread_count == 0) {
		nfsrv_uc_start();
	}

	/*
	 * We don't take a lock since once we're up nfsrv_uc_thread_count does
	 * not change until shutdown and then we should not be adding sockets to
	 * generate up-calls.
	 */
	if (nfsrv_uc_thread_count) {
		MALLOC(arg, struct nfsrv_uc_arg *, sizeof(struct nfsrv_uc_arg), M_TEMP, M_WAITOK | M_ZERO);
		if (arg == NULL) {
			goto direct;
		}

		slp->ns_ua = arg;
		arg->nua_slp = slp;
		arg->nua_qi = NFS_UC_HASH(slp);

		sock_setupcall(slp->ns_so, nfsrv_uc_proxy, arg);
	} else {
direct:
		slp->ns_ua = NULL;
		DPRINT("setting nfsrv_rcv up-call\n");
		sock_setupcall(slp->ns_so, nfsrv_rcv, slp);
	}

	/* just playin' it safe */
	sock_setsockopt(slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, &on, sizeof(on));

	return;
}

#endif /* CONFIG_NFS_SERVER */
Commit	Line	Data
316670eb	1	/*
3e170ce0	2	* Copyright (c) 2011-2014 Apple Inc. All rights reserved.
316670eb A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
ea3f0419 A	28
	29	#include <nfs/nfs_conf.h>
	30	#if CONFIG_NFS_SERVER
	31
316670eb A	32	#include <stdint.h>
	33	#include <sys/param.h>
	34	#include <sys/mount_internal.h>
	35	#include <sys/malloc.h>
	36	#include <sys/queue.h>
	37
	38	#include <libkern/libkern.h>
	39	#include <libkern/OSAtomic.h>
	40	#include <kern/debug.h>
	41	#include <kern/thread.h>
	42
	43	#include <nfs/rpcv2.h>
	44	#include <nfs/nfsproto.h>
	45	#include <nfs/nfs.h>
	46
	47	#ifdef NFS_UC_DEBUG
	48	#define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
	49	#else
	50	#define DPRINT(fmt, ...)
	51	#endif
	52
	53	struct nfsrv_uc_arg {
	54	TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq;
	55	socket_t nua_so;
	56	struct nfsrv_sock *nua_slp;
	57	int nua_waitflag; /* Should always be MBUF_DONTWAIT */
	58	uint32_t nua_flags;
	59	uint32_t nua_qi;
	60	};
0a7de745	61	#define NFS_UC_QUEUED 0x0001
316670eb A	62
	63	#define NFS_UC_HASH_SZ 7
	64	#define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)
	65
	66	TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg);
	67
	68	static struct nfsrv_uc_queue {
c3c9b80d	69	lck_mtx_t ucq_lock;
0a7de745 A	70	struct nfsrv_uc_q ucq_queue[1];
	71	thread_t ucq_thd;
	72	uint32_t ucq_flags;
316670eb	73	} nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ];
0a7de745	74	#define NFS_UC_QUEUE_SLEEPING 0x0001
316670eb	75
c3c9b80d A	76	static LCK_GRP_DECLARE(nfsrv_uc_group, "nfs_upcall_locks");
c3c9b80d A	77	static LCK_MTX_DECLARE(nfsrv_uc_shutdown_lock, &nfsrv_uc_group);
316670eb A	78	static volatile int nfsrv_uc_shutdown = 0;
	79	static int32_t nfsrv_uc_thread_count;
	80
	81	extern kern_return_t thread_terminate(thread_t);
	82
	83	#ifdef NFS_UC_Q_DEBUG
	84	int nfsrv_uc_use_proxy = 1;
	85	uint32_t nfsrv_uc_queue_limit;
	86	uint32_t nfsrv_uc_queue_max_seen;
	87	volatile uint32_t nfsrv_uc_queue_count;
	88	#endif
	89
	90	/*
	91	* Thread that dequeues up-calls and runs the nfsrv_rcv routine
	92	*/
	93	static void
	94	nfsrv_uc_thread(void *arg, wait_result_t wr __unused)
	95	{
	96	int qi = (int)(uintptr_t)arg;
	97	int error;
	98	struct nfsrv_uc_arg *ep = NULL;
	99	struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
	100
	101	DPRINT("nfsrv_uc_thread %d started\n", qi);
	102	while (!nfsrv_uc_shutdown) {
c3c9b80d	103	lck_mtx_lock(&myqueue->ucq_lock);
316670eb A	104
	105	while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) {
	106	myqueue->ucq_flags \|= NFS_UC_QUEUE_SLEEPING;
c3c9b80d	107	error = msleep(myqueue, &myqueue->ucq_lock, PSOCK, "nfsd_upcall_handler", NULL);
316670eb A	108	myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING;
	109	if (error) {
	110	printf("nfsrv_uc_thread received error %d\n", error);
	111	}
	112	}
	113	if (nfsrv_uc_shutdown) {
c3c9b80d	114	lck_mtx_unlock(&myqueue->ucq_lock);
316670eb A	115	break;
	116	}
	117
	118
	119	ep = TAILQ_FIRST(myqueue->ucq_queue);
	120	DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi, ep, myqueue);
	121
	122	TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq);
	123
	124	ep->nua_flags &= ~NFS_UC_QUEUED;
	125
c3c9b80d	126	lck_mtx_unlock(&myqueue->ucq_lock);
316670eb A	127
	128	#ifdef NFS_UC_Q_DEBUG
	129	OSDecrementAtomic(&nfsrv_uc_queue_count);
	130	#endif
	131
	132	DPRINT("calling nfsrv_rcv for %p\n", (void *)ep->nua_slp);
	133	nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag);
	134	}
	135
c3c9b80d	136	lck_mtx_lock(&nfsrv_uc_shutdown_lock);
316670eb A	137	nfsrv_uc_thread_count--;
316670eb A	138	wakeup(&nfsrv_uc_thread_count);
c3c9b80d	139	lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
316670eb A	140
	141	thread_terminate(current_thread());
	142	}
	143
	144	/*
	145	* Dequeue a closed nfsrv_sock if needed from the up-call queue.
	146	* Call from nfsrv_zapsock
	147	*/
	148	void
	149	nfsrv_uc_dequeue(struct nfsrv_sock *slp)
	150	{
	151	struct nfsrv_uc_arg *ap = slp->ns_ua;
	152	struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi];
	153
	154	/*
	155	* We assume that the socket up-calls have been stop and the socket
	156	* is shutting down so no need for acquiring the lock to check that
	157	* the flag is cleared.
	158	*/
0a7de745	159	if (ap == NULL \|\| (ap->nua_flags & NFS_UC_QUEUED) == 0) {
316670eb	160	return;
0a7de745	161	}
316670eb	162	/* If we're queued we might race with nfsrv_uc_thread */
c3c9b80d	163	lck_mtx_lock(&myqueue->ucq_lock);
316670eb A	164	if (ap->nua_flags & NFS_UC_QUEUED) {
	165	printf("nfsrv_uc_dequeue remove %p\n", ap);
	166	TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq);
	167	ap->nua_flags &= ~NFS_UC_QUEUED;
	168	#ifdef NFS_UC_Q_DEBUG
	169	OSDecrementAtomic(&nfsrv_uc_queue_count);
0a7de745	170	#endif
316670eb	171	}
3e170ce0 A	172	FREE(slp->ns_ua, M_TEMP);
3e170ce0 A	173	slp->ns_ua = NULL;
c3c9b80d	174	lck_mtx_unlock(&myqueue->ucq_lock);
316670eb A	175	}
	176
	177	/*
	178	* Allocate and initialize globals for nfsrv_sock up-call support.
	179	*/
	180	void
	181	nfsrv_uc_init(void)
	182	{
c3c9b80d	183	for (int i = 0; i < NFS_UC_HASH_SZ; i++) {
316670eb	184	TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue);
c3c9b80d	185	lck_mtx_init(&nfsrv_uc_queue_tbl[i].ucq_lock, &nfsrv_uc_group, LCK_ATTR_NULL);
316670eb A	186	nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
	187	nfsrv_uc_queue_tbl[i].ucq_flags = 0;
	188	}
316670eb A	189	}
	190
	191	/*
	192	* Start up-call threads to service nfsrv_sock(s)
	193	* Called from the first call of nfsrv_uc_addsock
	194	*/
	195	static void
	196	nfsrv_uc_start(void)
	197	{
	198	int32_t i;
	199	int error;
	200
	201	#ifdef NFS_UC_Q_DEBUG
0a7de745	202	if (!nfsrv_uc_use_proxy) {
316670eb	203	return;
0a7de745	204	}
316670eb A	205	#endif
	206	DPRINT("nfsrv_uc_start\n");
	207
	208	/* Wait until previous shutdown finishes */
c3c9b80d	209	lck_mtx_lock(&nfsrv_uc_shutdown_lock);
0a7de745	210	while (nfsrv_uc_shutdown \|\| nfsrv_uc_thread_count > 0) {
c3c9b80d	211	msleep(&nfsrv_uc_thread_count, &nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_wait", NULL);
0a7de745	212	}
316670eb A	213
	214	/* Start up-call threads */
	215	for (i = 0; i < NFS_UC_HASH_SZ; i++) {
	216	error = kernel_thread_start(nfsrv_uc_thread, (void *)(uintptr_t)i, &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd);
	217	if (!error) {
	218	nfsrv_uc_thread_count++;
	219	} else {
	220	printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error);
	221	}
	222	}
	223	if (nfsrv_uc_thread_count == 0) {
	224	printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
	225	goto out;
	226	}
	227
	228	out:
	229	#ifdef NFS_UC_Q_DEBUG
	230	nfsrv_uc_queue_count = 0ULL;
	231	nfsrv_uc_queue_max_seen = 0ULL;
	232	#endif
c3c9b80d	233	lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
316670eb A	234	}
	235
	236	/*
	237	* Stop the up-call threads.
	238	* Called from nfsrv_uc_cleanup.
	239	*/
	240	static void
	241	nfsrv_uc_stop(void)
	242	{
	243	int32_t i;
	244	int32_t thread_count = nfsrv_uc_thread_count;
	245
	246	DPRINT("Entering nfsrv_uc_stop\n");
	247
	248	/* Signal up-call threads to stop */
	249	nfsrv_uc_shutdown = 1;
	250	for (i = 0; i < thread_count; i++) {
c3c9b80d	251	lck_mtx_lock(&nfsrv_uc_queue_tbl[i].ucq_lock);
316670eb	252	wakeup(&nfsrv_uc_queue_tbl[i]);
c3c9b80d	253	lck_mtx_unlock(&nfsrv_uc_queue_tbl[i].ucq_lock);
316670eb A	254	}
	255
	256	/* Wait until they are done shutting down */
c3c9b80d	257	lck_mtx_lock(&nfsrv_uc_shutdown_lock);
0a7de745	258	while (nfsrv_uc_thread_count > 0) {
c3c9b80d	259	msleep(&nfsrv_uc_thread_count, &nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_stop", NULL);
0a7de745	260	}
316670eb A	261
	262	/* Deallocate old threads */
	263	for (i = 0; i < nfsrv_uc_thread_count; i++) {
0a7de745	264	if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL) {
316670eb	265	thread_deallocate(nfsrv_uc_queue_tbl[i].ucq_thd);
0a7de745	266	}
316670eb A	267	nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
	268	}
	269
	270	/* Enable restarting */
	271	nfsrv_uc_shutdown = 0;
c3c9b80d	272	lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
316670eb A	273	}
	274
	275	/*
	276	* Shutdown up-calls for nfsrv_socks.
	277	* Make sure nothing is queued on the up-call queues
	278	* Shutdown the up-call threads
	279	* Called from nfssvc_cleanup.
	280	*/
	281	void
	282	nfsrv_uc_cleanup(void)
	283	{
	284	int i;
	285
	286	DPRINT("Entering nfsrv_uc_cleanup\n");
	287
	288	/*
	289	* Every thing should be dequeued at this point or will be as sockets are closed
	290	* but to be safe, we'll make sure.
	291	*/
	292	for (i = 0; i < NFS_UC_HASH_SZ; i++) {
	293	struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i];
	294
c3c9b80d	295	lck_mtx_lock(&queue->ucq_lock);
316670eb A	296	while (!TAILQ_EMPTY(queue->ucq_queue)) {
	297	struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue);
	298	TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq);
	299	ep->nua_flags &= ~NFS_UC_QUEUED;
	300	}
c3c9b80d	301	lck_mtx_unlock(&queue->ucq_lock);
316670eb A	302	}
	303
	304	nfsrv_uc_stop();
	305	}
	306
	307	/*
	308	* This is the nfs up-call routine for server sockets.
	309	* We used to set nfsrv_rcv as the up-call routine, but
	310	* recently that seems like we are doing to much work for
	311	* the interface thread, so we just queue the arguments
	312	* that we would have gotten for nfsrv_rcv and let a
	313	* worker thread dequeue them and pass them on to nfsrv_rcv.
	314	*/
	315	static void
	316	nfsrv_uc_proxy(socket_t so, void *arg, int waitflag)
	317	{
	318	struct nfsrv_uc_arg uap = (struct nfsrv_uc_arg )arg;
	319	int qi = uap->nua_qi;
	320	struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
	321
c3c9b80d	322	lck_mtx_lock(&myqueue->ucq_lock);
316670eb A	323	DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap, uap->nua_slp);
316670eb A	324	DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi, myqueue);
3e170ce0	325	if (uap == NULL \|\| uap->nua_flags & NFS_UC_QUEUED) {
c3c9b80d	326	lck_mtx_unlock(&myqueue->ucq_lock);
3e170ce0	327	return; /* Already queued or freed */
316670eb A	328	}
	329
	330	uap->nua_so = so;
	331	uap->nua_waitflag = waitflag;
	332
	333	TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq);
	334
	335	uap->nua_flags \|= NFS_UC_QUEUED;
cb323159	336	if (myqueue->ucq_flags & NFS_UC_QUEUE_SLEEPING) {
316670eb	337	wakeup(myqueue);
0a7de745	338	}
316670eb A	339
	340	#ifdef NFS_UC_Q_DEBUG
	341	{
	342	uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count);
0a7de745	343
316670eb	344	/* This is a bit racey but just for debug */
0a7de745	345	if (count > nfsrv_uc_queue_max_seen) {
316670eb	346	nfsrv_uc_queue_max_seen = count;
0a7de745	347	}
316670eb A	348
	349	if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) {
	350	panic("nfsd up-call queue limit exceeded\n");
	351	}
	352	}
	353	#endif
c3c9b80d	354	lck_mtx_unlock(&myqueue->ucq_lock);
316670eb A	355	}
	356
	357
	358	/*
	359	* Set the up-call routine on the socket associated with the passed in
	360	* nfsrv_sock.
	361	* Assumes nfsd_mutex is held.
	362	*/
	363	void
	364	nfsrv_uc_addsock(struct nfsrv_sock *slp, int start)
	365	{
	366	int on = 1;
	367	struct nfsrv_uc_arg *arg;
	368
0a7de745	369	if (start && nfsrv_uc_thread_count == 0) {
316670eb	370	nfsrv_uc_start();
0a7de745	371	}
316670eb A	372
	373	/*
	374	* We don't take a lock since once we're up nfsrv_uc_thread_count does
	375	* not change until shutdown and then we should not be adding sockets to
	376	* generate up-calls.
	377	*/
	378	if (nfsrv_uc_thread_count) {
0a7de745 A	379	MALLOC(arg, struct nfsrv_uc_arg *, sizeof(struct nfsrv_uc_arg), M_TEMP, M_WAITOK \| M_ZERO);
0a7de745 A	380	if (arg == NULL) {
316670eb	381	goto direct;
0a7de745	382	}
316670eb A	383
	384	slp->ns_ua = arg;
	385	arg->nua_slp = slp;
	386	arg->nua_qi = NFS_UC_HASH(slp);
	387
	388	sock_setupcall(slp->ns_so, nfsrv_uc_proxy, arg);
	389	} else {
	390	direct:
	391	slp->ns_ua = NULL;
	392	DPRINT("setting nfsrv_rcv up-call\n");
	393	sock_setupcall(slp->ns_so, nfsrv_rcv, slp);
	394	}
	395
	396	/* just playin' it safe */
	397	sock_setsockopt(slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, &on, sizeof(on));
	398
	399	return;
	400	}
ea3f0419 A	401
ea3f0419 A	402	#endif /* CONFIG_NFS_SERVER */