]> git.saurik.com Git - apple/xnu.git/blame - bsd/kern/pthread_support.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2050.24.15.tar.gz
[apple/xnu.git] / bsd / kern / pthread_support.c
CommitLineData
2d21ac55 1/*
b0d623f7 2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
2d21ac55
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1995-2005 Apple Computer, Inc. All Rights Reserved */
29/*
30 * pthread_support.c
31 */
32
b0d623f7 33#if PSYNCH
2d21ac55
A		 34
35#include <sys/param.h>
b0d623f7 36#include <sys/queue.h>
2d21ac55
A		 37#include <sys/resourcevar.h>
38#include <sys/proc_internal.h>
39#include <sys/kauth.h>
40#include <sys/systm.h>
41#include <sys/timeb.h>
42#include <sys/times.h>
b0d623f7 43#include <sys/time.h>
2d21ac55 44#include <sys/acct.h>
2d21ac55
A		 45#include <sys/kernel.h>
46#include <sys/wait.h>
47#include <sys/signalvar.h>
48#include <sys/syslog.h>
49#include <sys/stat.h>
50#include <sys/lock.h>
51#include <sys/kdebug.h>
2d21ac55 52#include <sys/sysproto.h>
b0d623f7 53#include <sys/pthread_internal.h>
2d21ac55 54#include <sys/vm.h>
b0d623f7
A		 55#include <sys/user.h>
56
57#include <mach/mach_types.h>
58#include <mach/vm_prot.h>
59#include <mach/semaphore.h>
60#include <mach/sync_policy.h>
61#include <mach/task.h>
62#include <kern/kern_types.h>
63#include <kern/task.h>
64#include <kern/clock.h>
65#include <mach/kern_return.h>
2d21ac55 66#include <kern/thread.h>
b0d623f7
A		 67#include <kern/sched_prim.h>
68#include <kern/thread_call.h>
69#include <kern/kalloc.h>
6d2010ae 70#include <kern/zalloc.h>
b0d623f7
A		 71#include <kern/sched_prim.h>
72#include <kern/processor.h>
73#include <kern/affinity.h>
74#include <kern/wait_queue.h>
6d2010ae 75#include <kern/mach_param.h>
b0d623f7
A		 76#include <mach/mach_vm.h>
77#include <mach/mach_param.h>
78#include <mach/thread_policy.h>
79#include <mach/message.h>
80#include <mach/port.h>
81#include <vm/vm_protos.h>
82#include <vm/vm_map.h>
83#include <mach/vm_region.h>
2d21ac55 84
b0d623f7 85#include <libkern/OSAtomic.h>
2d21ac55 86
6d2010ae
A		 87#include <pexpert/pexpert.h>
88
89#define __PSYNCH_DEBUG__ 0 /* debug panic actions */
316670eb 90#if (KDEBUG && STANDARD_KDEBUG)
6d2010ae 91#define _PSYNCH_TRACE_ 1 /* kdebug trace */
316670eb 92#endif
6d2010ae
A		 93
94#define __TESTMODE__ 2 /* 0 - return error on user error conditions */
95 /* 1 - log error on user error conditions */
96 /* 2 - abort caller on user error conditions */
97 /* 3 - panic on user error conditions */
98static int __test_panics__;
99static int __test_aborts__;
100static int __test_prints__;
101
102static inline void __FAILEDUSERTEST__(const char *str)
103{
104 proc_t p;
105
106 if (__test_panics__ != 0)
107 panic(str);
108
109 if (__test_aborts__ != 0 || __test_prints__ != 0)
110 p = current_proc();
111
112 if (__test_prints__ != 0)
113 printf("PSYNCH: pid[%d]: %s\n", p->p_pid, str);
114
115 if (__test_aborts__ != 0)
116 psignal(p, SIGABRT);
117}
2d21ac55 118
b0d623f7
A		 119#if _PSYNCH_TRACE_
120#define _PSYNCH_TRACE_MLWAIT 0x9000000
121#define _PSYNCH_TRACE_MLDROP 0x9000004
122#define _PSYNCH_TRACE_CVWAIT 0x9000008
123#define _PSYNCH_TRACE_CVSIGNAL 0x900000c
124#define _PSYNCH_TRACE_CVBROAD 0x9000010
125#define _PSYNCH_TRACE_KMDROP 0x9000014
126#define _PSYNCH_TRACE_RWRDLOCK 0x9000018
127#define _PSYNCH_TRACE_RWLRDLOCK 0x900001c
128#define _PSYNCH_TRACE_RWWRLOCK 0x9000020
129#define _PSYNCH_TRACE_RWYWRLOCK 0x9000024
130#define _PSYNCH_TRACE_RWUPGRADE 0x9000028
131#define _PSYNCH_TRACE_RWDOWNGRADE 0x900002c
132#define _PSYNCH_TRACE_RWUNLOCK 0x9000030
133#define _PSYNCH_TRACE_RWUNLOCK2 0x9000034
134#define _PSYNCH_TRACE_RWHANDLEU 0x9000038
135#define _PSYNCH_TRACE_FSEQTILL 0x9000040
6d2010ae
A		 136#define _PSYNCH_TRACE_CLRPRE 0x9000044
137#define _PSYNCH_TRACE_CVHBROAD 0x9000048
138#define _PSYNCH_TRACE_CVSEQ 0x900004c
139#define _PSYNCH_TRACE_THWAKEUP 0x9000050
b0d623f7
A		 140/* user side */
141#define _PSYNCH_TRACE_UM_LOCK 0x9000060
142#define _PSYNCH_TRACE_UM_UNLOCK 0x9000064
143#define _PSYNCH_TRACE_UM_MHOLD 0x9000068
144#define _PSYNCH_TRACE_UM_MDROP 0x900006c
145#define _PSYNCH_TRACE_UM_CVWAIT 0x9000070
146#define _PSYNCH_TRACE_UM_CVSIG 0x9000074
147#define _PSYNCH_TRACE_UM_CVBRD 0x9000078
2d21ac55 148
6d2010ae
A		 149proc_t pthread_debug_proc = PROC_NULL;
150static inline void __PTHREAD_TRACE_DEBUG(uint32_t debugid, uintptr_t arg1,
151 uintptr_t arg2,
152 uintptr_t arg3,
153 uintptr_t arg4,
154 uintptr_t arg5)
155{
156 proc_t p = current_proc();
157
158 if ((pthread_debug_proc != NULL) && (p == pthread_debug_proc))
159 KERNEL_DEBUG_CONSTANT(debugid, arg1, arg2, arg3, arg4, arg5);
160}
161
b0d623f7 162#endif /* _PSYNCH_TRACE_ */
2d21ac55 163
6d2010ae
A		 164#define ECVCERORR 256
165#define ECVPERORR 512
166
b0d623f7
A		 167lck_mtx_t * pthread_list_mlock;
168
169#define PTHHASH(addr) (&pthashtbl[(addr) & pthhash])
170extern LIST_HEAD(pthhashhead, ksyn_wait_queue) *pth_glob_hashtbl;
171struct pthhashhead * pth_glob_hashtbl;
172u_long pthhash;
173
174LIST_HEAD(, ksyn_wait_queue) pth_free_list;
6d2010ae
A		 175int num_total_kwq = 0; /* number of kwq in use currently */
176int num_infreekwq = 0; /* number of kwq in free list */
177int num_freekwq = 0; /* number of kwq actually freed from the free the list */
178int num_reusekwq = 0; /* number of kwq pulled back for reuse from free list */
179int num_addedfreekwq = 0; /* number of added free kwq from the last instance */
180int num_lastfreekwqcount = 0; /* the free count from the last time */
b0d623f7
A		 181
182static int PTH_HASHSIZE = 100;
183
6d2010ae
A		 184static zone_t kwq_zone; /* zone for allocation of ksyn_queue */
185static zone_t kwe_zone; /* zone for allocation of ksyn_waitq_element */
b0d623f7
A		 186
187#define SEQFIT 0
188#define FIRSTFIT 1
189
190struct ksyn_queue {
6d2010ae 191 TAILQ_HEAD(ksynq_kwelist_head, ksyn_waitq_element) ksynq_kwelist;
b0d623f7
A		 192 uint32_t ksynq_count; /* number of entries in queue */
193 uint32_t ksynq_firstnum; /* lowest seq in queue */
194 uint32_t ksynq_lastnum; /* highest seq in queue */
195};
6d2010ae 196typedef struct ksyn_queue * ksyn_queue_t;
b0d623f7
A		 197
198#define KSYN_QUEUE_READ 0
199#define KSYN_QUEUE_LREAD 1
200#define KSYN_QUEUE_WRITER 2
201#define KSYN_QUEUE_YWRITER 3
202#define KSYN_QUEUE_UPGRADE 4
203#define KSYN_QUEUE_MAX 5
204
205struct ksyn_wait_queue {
206 LIST_ENTRY(ksyn_wait_queue) kw_hash;
207 LIST_ENTRY(ksyn_wait_queue) kw_list;
b0d623f7
A		 208 user_addr_t kw_addr;
209 uint64_t kw_owner;
210 uint64_t kw_object; /* object backing in shared mode */
211 uint64_t kw_offset; /* offset inside the object in shared mode */
212 int kw_flags; /* mutex, cvar options/flags */
213 int kw_pflags; /* flags under listlock protection */
214 struct timeval kw_ts; /* timeval need for upkeep before free */
215 int kw_iocount; /* inuse reference */
6d2010ae 216 int kw_dropcount; /* current users unlocking... */
b0d623f7
A		 217
218 int kw_type; /* queue type like mutex, cvar, etc */
219 uint32_t kw_inqueue; /* num of waiters held */
6d2010ae 220 uint32_t kw_fakecount; /* number of error/prepost fakes */
b0d623f7
A		 221 uint32_t kw_highseq; /* highest seq in the queue */
222 uint32_t kw_lowseq; /* lowest seq in the queue */
6d2010ae
A		 223 uint32_t kw_lword; /* L value from userland */
224 uint32_t kw_uword; /* U world value from userland */
225 uint32_t kw_sword; /* S word value from userland */
b0d623f7 226 uint32_t kw_lastunlockseq; /* the last seq that unlocked */
6d2010ae
A		 227/* for CV to be used as the seq kernel has seen so far */
228#define kw_cvkernelseq kw_lastunlockseq
229 uint32_t kw_lastseqword; /* the last seq that unlocked */
230/* for mutex and cvar we need to track I bit values */
231 uint32_t kw_nextseqword; /* the last seq that unlocked; with num of waiters */
232#define kw_initrecv kw_nextseqword /* number of incoming waiters with Ibit seen sofar */
233 uint32_t kw_overlapwatch; /* chance for overlaps */
234#define kw_initcount kw_overlapwatch /* number of incoming waiters with Ibit expected */
235 uint32_t kw_initcountseq; /* highest seq with Ibit on for mutex and cvar*/
b0d623f7
A		 236 uint32_t kw_pre_rwwc; /* prepost count */
237 uint32_t kw_pre_lockseq; /* prepost target seq */
6d2010ae 238 uint32_t kw_pre_sseq; /* prepost target sword, in cvar used for mutexowned */
b0d623f7
A		 239 uint32_t kw_pre_intrcount; /* prepost of missed wakeup due to intrs */
240 uint32_t kw_pre_intrseq; /* prepost of missed wakeup limit seq */
241 uint32_t kw_pre_intrretbits; /* return bits value for missed wakeup threads */
242 uint32_t kw_pre_intrtype; /* type of failed wakueps*/
243
244 int kw_kflags;
b0d623f7
A		 245 struct ksyn_queue kw_ksynqueues[KSYN_QUEUE_MAX]; /* queues to hold threads */
246 lck_mtx_t kw_lock; /* mutex lock protecting this structure */
b0d623f7 247};
b0d623f7
A		 248typedef struct ksyn_wait_queue * ksyn_wait_queue_t;
249
b0d623f7 250#define PTHRW_INC 0x100
6d2010ae 251#define PTHRW_BIT_MASK 0x000000ff
b0d623f7
A		 252
253#define PTHRW_COUNT_SHIFT 8
254#define PTHRW_COUNT_MASK 0xffffff00
255#define PTHRW_MAX_READERS 0xffffff00
256
6d2010ae
A		 257/* New model bits on Lword */
258#define PTH_RWL_KBIT 0x01 /* users cannot acquire in user mode */
259#define PTH_RWL_EBIT 0x02 /* exclusive lock in progress */
260#define PTH_RWL_WBIT 0x04 /* write waiters pending in kernel */
261#define PTH_RWL_PBIT 0x04 /* prepost (cv) pending in kernel */
262#define PTH_RWL_YBIT 0x08 /* yielding write waiters pending in kernel */
263#define PTH_RWL_RETRYBIT 0x08 /* mutex retry wait */
264#define PTH_RWL_LBIT 0x10 /* long read in progress */
265#define PTH_RWL_MTXNONE 0x10 /* indicates the cvwait does not have mutex held */
266#define PTH_RWL_UBIT 0x20 /* upgrade request pending */
267#define PTH_RWL_MTX_WAIT 0x20 /* in cvar in mutex wait */
268#define PTH_RWL_RBIT 0x40 /* reader pending in kernel(not used) */
269#define PTH_RWL_MBIT 0x40 /* overlapping grants from kernel */
270#define PTH_RWL_TRYLKBIT 0x40 /* trylock attempt (mutex only) */
271#define PTH_RWL_IBIT 0x80 /* lcok reset, held untill first succeesful unlock */
272
273
274/* UBIT values for mutex, cvar */
275#define PTH_RWU_SBIT 0x01
276#define PTH_RWU_BBIT 0x02
277
278#define PTHRW_RWL_INIT PTH_RWL_IBIT /* reset state on the lock bits (U)*/
279
280/* New model bits on Sword */
281#define PTH_RWS_SBIT 0x01 /* kernel transition seq not set yet*/
282#define PTH_RWS_IBIT 0x02 /* Sequence is not set on return from kernel */
283#define PTH_RWS_CV_CBIT PTH_RWS_SBIT /* kernel has cleared all info w.r.s.t CV */
284#define PTH_RWS_CV_PBIT PTH_RWS_IBIT /* kernel has prepost/fake structs only,no waiters */
285#define PTH_RWS_CV_MBIT PTH_RWL_MBIT /* to indicate prepost return */
286#define PTH_RWS_WSVBIT 0x04 /* save W bit */
287#define PTH_RWS_USVBIT 0x08 /* save U bit */
288#define PTH_RWS_YSVBIT 0x10 /* save Y bit */
289#define PTHRW_RWS_INIT PTH_RWS_SBIT /* reset on the lock bits (U)*/
290#define PTHRW_RWS_SAVEMASK (PTH_RWS_WSVBIT|PTH_RWS_USVBIT|PTH_RWS_YSVBIT) /*save bits mask*/
291#define PTHRW_SW_Reset_BIT_MASK 0x000000fe /* remove S bit and get rest of the bits */
292
293#define PTHRW_RWS_INIT PTH_RWS_SBIT /* reset on the lock bits (U)*/
294
295
296#define PTHRW_UN_BIT_MASK 0x000000bf /* remove overlap bit */
297
298
299#define PTHREAD_MTX_TID_SWITCHING (uint64_t)-1
300
301/* new L word defns */
302#define is_rwl_readinuser(x) ((((x) & (PTH_RWL_UBIT | PTH_RWL_KBIT)) == 0)||(((x) & PTH_RWL_LBIT) != 0))
303#define is_rwl_ebit_set(x) (((x) & PTH_RWL_EBIT) != 0)
304#define is_rwl_lbit_set(x) (((x) & PTH_RWL_LBIT) != 0)
305#define is_rwl_readoverlap(x) (((x) & PTH_RWL_MBIT) != 0)
306#define is_rw_ubit_set(x) (((x) & PTH_RWL_UBIT) != 0)
307
308/* S word checks */
309#define is_rws_setseq(x) (((x) & PTH_RWS_SBIT))
310#define is_rws_setunlockinit(x) (((x) & PTH_RWS_IBIT))
311
b0d623f7
A		 312/* first contended seq that kernel sees */
313#define KW_MTXFIRST_KSEQ 0x200
314#define KW_CVFIRST_KSEQ 1
315#define KW_RWFIRST_KSEQ 0x200
316
6d2010ae
A		 317int is_seqlower(uint32_t x, uint32_t y);
318int is_seqlower_eq(uint32_t x, uint32_t y);
319int is_seqhigher(uint32_t x, uint32_t y);
320int is_seqhigher_eq(uint32_t x, uint32_t y);
321int find_diff(uint32_t upto, uint32_t lowest);
b0d623f7 322
b0d623f7
A		 323
324static inline int diff_genseq(uint32_t x, uint32_t y) {
325 if (x > y) {
326 return(x-y);
327 } else {
328 return((PTHRW_MAX_READERS - y) + x + PTHRW_INC);
2d21ac55 329 }
2d21ac55
A		 330}
331
b0d623f7
A		 332#define TID_ZERO (uint64_t)0
333
334/* bits needed in handling the rwlock unlock */
335#define PTH_RW_TYPE_READ 0x01
336#define PTH_RW_TYPE_LREAD 0x02
337#define PTH_RW_TYPE_WRITE 0x04
338#define PTH_RW_TYPE_YWRITE 0x08
339#define PTH_RW_TYPE_UPGRADE 0x10
340#define PTH_RW_TYPE_MASK 0xff
341#define PTH_RW_TYPE_SHIFT 8
342
343#define PTH_RWSHFT_TYPE_READ 0x0100
344#define PTH_RWSHFT_TYPE_LREAD 0x0200
345#define PTH_RWSHFT_TYPE_WRITE 0x0400
346#define PTH_RWSHFT_TYPE_YWRITE 0x0800
347#define PTH_RWSHFT_TYPE_MASK 0xff00
348
349/*
350 * Mutex protocol attributes
351 */
352#define PTHREAD_PRIO_NONE 0
353#define PTHREAD_PRIO_INHERIT 1
354#define PTHREAD_PRIO_PROTECT 2
355#define PTHREAD_PROTOCOL_FLAGS_MASK 0x3
356
357/*
358 * Mutex type attributes
359 */
360#define PTHREAD_MUTEX_NORMAL 0
361#define PTHREAD_MUTEX_ERRORCHECK 4
362#define PTHREAD_MUTEX_RECURSIVE 8
363#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL
364#define PTHREAD_TYPE_FLAGS_MASK 0xc
365
366/*
367 * Mutex pshared attributes
368 */
369#define PTHREAD_PROCESS_SHARED 0x10
370#define PTHREAD_PROCESS_PRIVATE 0x20
371#define PTHREAD_PSHARED_FLAGS_MASK 0x30
372
373/*
374 * Mutex policy attributes
375 */
376#define _PTHREAD_MUTEX_POLICY_NONE 0
377#define _PTHREAD_MUTEX_POLICY_FAIRSHARE 0x040 /* 1 */
378#define _PTHREAD_MUTEX_POLICY_FIRSTFIT 0x080 /* 2 */
379#define _PTHREAD_MUTEX_POLICY_REALTIME 0x0c0 /* 3 */
380#define _PTHREAD_MUTEX_POLICY_ADAPTIVE 0x100 /* 4 */
381#define _PTHREAD_MUTEX_POLICY_PRIPROTECT 0x140 /* 5 */
382#define _PTHREAD_MUTEX_POLICY_PRIINHERIT 0x180 /* 6 */
383#define PTHREAD_POLICY_FLAGS_MASK 0x1c0
384
385#define _PTHREAD_MTX_OPT_HOLDLOCK 0x200
6d2010ae
A		 386#define _PTHREAD_MTX_OPT_NOMTX 0x400
387
388#define _PTHREAD_MTX_OPT_NOTIFY 0x1000
389#define _PTHREAD_MTX_OPT_MUTEX 0x2000 /* this is a mutex type */
b0d623f7 390
6d2010ae
A		 391#define _PTHREAD_RWLOCK_UPGRADE_TRY 0x10000
392
393/* pflags */
b0d623f7
A		 394#define KSYN_WQ_INLIST 1
395#define KSYN_WQ_INHASH 2
396#define KSYN_WQ_SHARED 4
6d2010ae 397#define KSYN_WQ_WAITING 8 /* threads waiting for this wq to be available */
b0d623f7
A		 398#define KSYN_WQ_FLIST 0X10 /* in free list to be freed after a short delay */
399
6d2010ae
A		 400/* kflags */
401#define KSYN_KWF_INITCLEARED 1 /* the init status found and preposts cleared */
402#define KSYN_KWF_ZEROEDOUT 2 /* the lword, etc are inited to 0 */
403
b0d623f7
A		 404#define KSYN_CLEANUP_DEADLINE 10
405int psynch_cleanupset;
406thread_call_t psynch_thcall;
2d21ac55 407
b0d623f7 408#define KSYN_WQTYPE_INWAIT 0x1000
6d2010ae 409#define KSYN_WQTYPE_INDROP 0x2000
b0d623f7
A		 410#define KSYN_WQTYPE_MTX 0x1
411#define KSYN_WQTYPE_CVAR 0x2
412#define KSYN_WQTYPE_RWLOCK 0x4
413#define KSYN_WQTYPE_SEMA 0x8
414#define KSYN_WQTYPE_BARR 0x10
6d2010ae 415#define KSYN_WQTYPE_MASK 0x00ff
b0d623f7
A		 416
417#define KSYN_MTX_MAX 0x0fffffff
6d2010ae 418#define KSYN_WQTYPE_MUTEXDROP (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX)
b0d623f7
A		 419
420#define KW_UNLOCK_PREPOST 0x01
421#define KW_UNLOCK_PREPOST_UPGRADE 0x02
422#define KW_UNLOCK_PREPOST_DOWNGRADE 0x04
423#define KW_UNLOCK_PREPOST_READLOCK 0x08
424#define KW_UNLOCK_PREPOST_LREADLOCK 0x10
425#define KW_UNLOCK_PREPOST_WRLOCK 0x20
426#define KW_UNLOCK_PREPOST_YWRLOCK 0x40
427
428#define CLEAR_PREPOST_BITS(kwq) {\
429 kwq->kw_pre_lockseq = 0; \
6d2010ae 430 kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
b0d623f7 431 kwq->kw_pre_rwwc = 0; \
b0d623f7
A		 432 }
433
6d2010ae
A		 434#define CLEAR_INITCOUNT_BITS(kwq) {\
435 kwq->kw_initcount = 0; \
436 kwq->kw_initrecv = 0; \
437 kwq->kw_initcountseq = 0; \
b0d623f7
A		 438 }
439
440#define CLEAR_INTR_PREPOST_BITS(kwq) {\
441 kwq->kw_pre_intrcount = 0; \
442 kwq->kw_pre_intrseq = 0; \
443 kwq->kw_pre_intrretbits = 0; \
444 kwq->kw_pre_intrtype = 0; \
445 }
6d2010ae
A		 446
447#define CLEAR_REINIT_BITS(kwq) {\
448 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) { \
449 if((kwq->kw_inqueue != 0) && (kwq->kw_inqueue != kwq->kw_fakecount)) \
450 panic("CV:entries in queue durinmg reinit %d:%d\n",kwq->kw_inqueue, kwq->kw_fakecount); \
451 };\
452 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_RWLOCK) { \
453 kwq->kw_nextseqword = PTHRW_RWS_INIT; \
454 kwq->kw_overlapwatch = 0; \
455 }; \
456 kwq->kw_pre_lockseq = 0; \
457 kwq->kw_pre_rwwc = 0; \
458 kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
459 kwq->kw_lastunlockseq = PTHRW_RWL_INIT; \
460 kwq->kw_lastseqword = PTHRW_RWS_INIT; \
461 kwq->kw_pre_intrcount = 0; \
462 kwq->kw_pre_intrseq = 0; \
463 kwq->kw_pre_intrretbits = 0; \
464 kwq->kw_pre_intrtype = 0; \
465 kwq->kw_lword = 0; \
466 kwq->kw_uword = 0; \
467 kwq->kw_sword = PTHRW_RWS_INIT; \
468 }
469
b0d623f7
A		 470void pthread_list_lock(void);
471void pthread_list_unlock(void);
472void pthread_list_lock_spin(void);
473void pthread_list_lock_convert_spin(void);
474void ksyn_wqlock(ksyn_wait_queue_t kwq);
475void ksyn_wqunlock(ksyn_wait_queue_t kwq);
476ksyn_wait_queue_t ksyn_wq_hash_lookup(user_addr_t mutex, proc_t p, int flags, uint64_t object, uint64_t offset);
477int ksyn_wqfind(user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int flags, int wqtype , ksyn_wait_queue_t * wq);
6d2010ae 478void ksyn_wqrelease(ksyn_wait_queue_t mkwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype);
b0d623f7 479extern int ksyn_findobj(uint64_t mutex, uint64_t * object, uint64_t * offset);
6d2010ae
A		 480static void UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int wqtype);
481extern thread_t port_name_to_thread(mach_port_name_t port_name);
482
7ddcb079 483kern_return_t ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int log, thread_continue_t, void * parameter);
6d2010ae
A		 484kern_return_t ksyn_wakeup_thread(ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe);
485void ksyn_freeallkwe(ksyn_queue_t kq);
486
487uint32_t psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags);
488int kwq_handle_unlock(ksyn_wait_queue_t, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int *blockp, uint32_t premgen);
489
b0d623f7 490void ksyn_queue_init(ksyn_queue_t kq);
6d2010ae
A		 491int ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int firstfit);
492ksyn_waitq_element_t ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq);
493void ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe);
494int ksyn_queue_move_tofree(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t freeq, int all, int reease);
b0d623f7
A		 495void update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq);
496uint32_t find_nextlowseq(ksyn_wait_queue_t kwq);
497uint32_t find_nexthighseq(ksyn_wait_queue_t kwq);
6d2010ae 498
b0d623f7 499int find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp);
b0d623f7 500uint32_t ksyn_queue_count_tolowest(ksyn_queue_t kq, uint32_t upto);
6d2010ae
A		 501
502ksyn_waitq_element_t ksyn_queue_find_cvpreposeq(ksyn_queue_t kq, uint32_t cgen);
503uint32_t ksyn_queue_cvcount_entries(ksyn_queue_t kq, uint32_t upto, uint32_t from, int * numwaitersp, int * numintrp, int * numprepop);
504void ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t *updatep);
505void ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release);
506ksyn_waitq_element_t ksyn_queue_find_signalseq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t toseq, uint32_t lockseq);
507ksyn_waitq_element_t ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, thread_t th, uint32_t toseq);
7ddcb079
A		 508void psynch_cvcontinue(void *, wait_result_t);
509void psynch_mtxcontinue(void *, wait_result_t);
6d2010ae 510
b0d623f7
A		 511int ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp);
512int kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * type, uint32_t lowest[]);
6d2010ae
A		 513ksyn_waitq_element_t ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove);
514int kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, int flags , int * blockp);
b0d623f7
A		 515int kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, int flags, uint32_t premgen, int * blockp);
516
b0d623f7 517static void
6d2010ae 518UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, __unused uint64_t tid, __unused int wqtype)
2d21ac55 519{
6d2010ae
A		 520 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) {
521 if ((kwq->kw_kflags & KSYN_KWF_ZEROEDOUT) != 0) {
522 /* the values of L,U and S are cleared out due to L==S in previous transition */
523 kwq->kw_lword = mgen;
524 kwq->kw_uword = ugen;
525 kwq->kw_sword = rw_wc;
526 kwq->kw_kflags &= ~KSYN_KWF_ZEROEDOUT;
527 }
528 if (is_seqhigher((mgen & PTHRW_COUNT_MASK), (kwq->kw_lword & PTHRW_COUNT_MASK)) != 0)
529 kwq->kw_lword = mgen;
530 if (is_seqhigher((ugen & PTHRW_COUNT_MASK), (kwq->kw_uword & PTHRW_COUNT_MASK)) != 0)
531 kwq->kw_uword = ugen;
532 if ((rw_wc & PTH_RWS_CV_CBIT) != 0) {
533 if(is_seqlower(kwq->kw_cvkernelseq, (rw_wc & PTHRW_COUNT_MASK)) != 0) {
534 kwq->kw_cvkernelseq = (rw_wc & PTHRW_COUNT_MASK);
535 }
536 if (is_seqhigher((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_sword & PTHRW_COUNT_MASK)) != 0)
537 kwq->kw_sword = rw_wc;
538 }
539 }
b0d623f7 540}
2d21ac55 541
6d2010ae 542
b0d623f7
A		 543/* to protect the hashes, iocounts, freelist */
544void
545pthread_list_lock(void)
546{
547 lck_mtx_lock(pthread_list_mlock);
2d21ac55
A		 548}
549
b0d623f7
A		 550void
551pthread_list_lock_spin(void)
552{
553 lck_mtx_lock_spin(pthread_list_mlock);
554}
2d21ac55
A		 555
556void
b0d623f7 557pthread_list_lock_convert_spin(void)
2d21ac55 558{
b0d623f7
A		 559 lck_mtx_convert_spin(pthread_list_mlock);
560}
561
562
563void
564pthread_list_unlock(void)
565{
566 lck_mtx_unlock(pthread_list_mlock);
567}
568
569/* to protect the indiv queue */
570void
571ksyn_wqlock(ksyn_wait_queue_t kwq)
572{
573
574 lck_mtx_lock(&kwq->kw_lock);
575}
576
577void
578ksyn_wqunlock(ksyn_wait_queue_t kwq)
579{
580 lck_mtx_unlock(&kwq->kw_lock);
581}
582
583
584/* routine to drop the mutex unlocks , used both for mutexunlock system call and drop during cond wait */
6d2010ae 585uint32_t
b0d623f7
A		 586psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags)
587{
6d2010ae 588 uint32_t nextgen, low_writer, updatebits, returnbits = 0;
b0d623f7 589 int firstfit = flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT;
6d2010ae 590 ksyn_waitq_element_t kwe = NULL;
b0d623f7 591 kern_return_t kret = KERN_SUCCESS;
b0d623f7
A		 592
593 nextgen = (ugen + PTHRW_INC);
594
595#if _PSYNCH_TRACE_
6d2010ae 596 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_START, (uint32_t)kwq->kw_addr, lkseq, ugen, flags, 0);
b0d623f7
A		 597#endif /* _PSYNCH_TRACE_ */
598
599 ksyn_wqlock(kwq);
600
601redrive:
6d2010ae 602
b0d623f7 603 if (kwq->kw_inqueue != 0) {
6d2010ae
A		 604 updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_EBIT | PTH_RWL_KBIT);
605 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
b0d623f7
A		 606 if (firstfit != 0)
607 {
b0d623f7 608 /* first fit , pick any one */
6d2010ae
A		 609 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
610 kwe->kwe_psynchretval = updatebits;
611 kwe->kwe_kwqqueue = NULL;
b0d623f7 612
b0d623f7 613#if _PSYNCH_TRACE_
6d2010ae 614 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf1, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
b0d623f7 615#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 616
617 kret = ksyn_wakeup_thread(kwq, kwe);
618#if __TESTPANICS__
b0d623f7
A		 619 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
620 panic("psynch_mutexdrop_internal: panic unable to wakeup firstfit mutex thread\n");
6d2010ae 621#endif /* __TESTPANICS__ */
b0d623f7
A		 622 if (kret == KERN_NOT_WAITING)
623 goto redrive;
624 } else {
625 /* handle fairshare */
626 low_writer = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
627 low_writer &= PTHRW_COUNT_MASK;
628
629 if (low_writer == nextgen) {
b0d623f7 630 /* next seq to be granted found */
6d2010ae 631 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
b0d623f7 632
6d2010ae
A		 633 /* since the grant could be cv, make sure mutex wait is set incase the thread interrupted out */
634 kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
635 kwe->kwe_kwqqueue = NULL;
b0d623f7 636
6d2010ae
A		 637#if _PSYNCH_TRACE_
638 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
639#endif /* _PSYNCH_TRACE_ */
640
641 kret = ksyn_wakeup_thread(kwq, kwe);
642#if __TESTPANICS__
b0d623f7
A		 643 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
644 panic("psynch_mutexdrop_internal: panic unable to wakeup fairshare mutex thread\n");
6d2010ae
A		 645#endif /* __TESTPANICS__ */
646 if (kret == KERN_NOT_WAITING) {
647 /* interrupt post */
648 kwq->kw_pre_intrcount = 1;
649 kwq->kw_pre_intrseq = nextgen;
650 kwq->kw_pre_intrretbits = updatebits;
651 kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
652#if _PSYNCH_TRACE_
653 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfafafaf1, nextgen, kwq->kw_pre_intrretbits, 0);
654#endif /* _PSYNCH_TRACE_ */
655 }
b0d623f7
A		 656
657 } else if (is_seqhigher(low_writer, nextgen) != 0) {
b0d623f7 658 kwq->kw_pre_rwwc++;
6d2010ae
A		 659
660 if (kwq->kw_pre_rwwc > 1) {
661 __FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (1)\n");
662 goto out;
663 }
664
b0d623f7 665 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
b0d623f7 666#if _PSYNCH_TRACE_
6d2010ae 667 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7 668#endif /* _PSYNCH_TRACE_ */
6d2010ae 669 } else {
b0d623f7 670
6d2010ae
A		 671 //__FAILEDUSERTEST__("psynch_mutexdrop_internal: FS mutex unlock sequence higher than the lowest one is queue\n");
672
673 kwe = ksyn_queue_find_seq(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (nextgen & PTHRW_COUNT_MASK), 1);
674 if (kwe != NULL) {
675 /* next seq to be granted found */
676 /* since the grant could be cv, make sure mutex wait is set incase the thread interrupted out */
677 kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
678 kwe->kwe_kwqqueue = NULL;
b0d623f7 679#if _PSYNCH_TRACE_
6d2010ae 680 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
b0d623f7 681#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 682 kret = ksyn_wakeup_thread(kwq, kwe);
683#if __TESTPANICS__
b0d623f7
A		 684 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
685 panic("psynch_mutexdrop_internal: panic unable to wakeup fairshare mutex thread\n");
6d2010ae 686#endif /* __TESTPANICS__ */
b0d623f7
A		 687 if (kret == KERN_NOT_WAITING)
688 goto redrive;
689 } else {
690 /* next seq to be granted not found, prepost */
b0d623f7 691 kwq->kw_pre_rwwc++;
6d2010ae
A		 692
693 if (kwq->kw_pre_rwwc > 1) {
694 __FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (2)\n");
695 goto out;
696 }
697
b0d623f7 698 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
6d2010ae
A		 699#if _PSYNCH_TRACE_
700 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
701#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 702 }
703 }
704 }
705 } else {
6d2010ae
A		 706
707 /* if firstfit the last one could be spurious */
708 if (firstfit == 0) {
709 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
710 kwq->kw_pre_rwwc++;
711
712 if (kwq->kw_pre_rwwc > 1) {
713 __FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (3)\n");
714 goto out;
715 }
716
717 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
b0d623f7 718#if _PSYNCH_TRACE_
6d2010ae 719 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7 720#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 721 } else {
722 /* first fit case */
b0d623f7 723#if _PSYNCH_TRACE_
6d2010ae 724 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_lastunlockseq, kwq->kw_pre_lockseq, 0);
b0d623f7 725#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 726 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
727 /* not set or the new lkseq is higher */
728 if ((kwq->kw_pre_rwwc == 0) || (is_seqlower(kwq->kw_pre_lockseq, lkseq) == 0))
729 kwq->kw_pre_lockseq = (lkseq & PTHRW_COUNT_MASK);
730 kwq->kw_pre_rwwc = 1;
731#if _PSYNCH_TRACE_
732 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
733#endif /* _PSYNCH_TRACE_ */
734
735 /* indicate prepost content in kernel */
736 returnbits = lkseq | PTH_RWL_PBIT;
b0d623f7 737 }
2d21ac55 738 }
b0d623f7 739
6d2010ae 740out:
b0d623f7
A		 741 ksyn_wqunlock(kwq);
742
743#if _PSYNCH_TRACE_
6d2010ae 744 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, 0, 0, 0);
b0d623f7 745#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 746 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX));
747 return(returnbits);
2d21ac55
A		 748}
749
b0d623f7
A		 750/*
751 * psynch_mutexwait: This system call is used for contended psynch mutexes to block.
752 */
2d21ac55 753
b0d623f7
A		 754int
755psynch_mutexwait(__unused proc_t p, struct psynch_mutexwait_args * uap, uint32_t * retval)
2d21ac55 756{
b0d623f7
A		 757 user_addr_t mutex = uap->mutex;
758 uint32_t mgen = uap->mgen;
759 uint32_t ugen = uap->ugen;
760 uint64_t tid = uap->tid;
761 int flags = uap->flags;
762 ksyn_wait_queue_t kwq;
763 int error=0;
6d2010ae 764 int ins_flags, retry;
b0d623f7
A		 765 uthread_t uth;
766 int firstfit = flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT;
6d2010ae
A		 767 uint32_t lockseq, updatebits=0;
768 ksyn_waitq_element_t kwe;
7ddcb079 769 kern_return_t kret;
b0d623f7
A		 770
771#if _PSYNCH_TRACE_
6d2010ae
A		 772 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_START, (uint32_t)mutex, mgen, ugen, flags, 0);
773 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, mgen, ugen, (uint32_t)tid, 0);
b0d623f7
A		 774#endif /* _PSYNCH_TRACE_ */
775
776 uth = current_uthread();
777
6d2010ae
A		 778 kwe = &uth->uu_kwe;
779 kwe->kwe_lockseq = uap->mgen;
780 kwe->kwe_uth = uth;
781 kwe->kwe_psynchretval = 0;
782 kwe->kwe_kwqqueue = NULL;
b0d623f7
A		 783 lockseq = (uap->mgen & PTHRW_COUNT_MASK);
784
785 if (firstfit == 0) {
786 ins_flags = SEQFIT;
787 } else {
788 /* first fit */
789 ins_flags = FIRSTFIT;
790 }
791
792 error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX), &kwq);
793 if (error != 0) {
794#if _PSYNCH_TRACE_
6d2010ae 795 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 1, 0xdeadbeef, error, 0);
b0d623f7
A		 796#endif /* _PSYNCH_TRACE_ */
797 return(error);
798 }
799
800 ksyn_wqlock(kwq);
801
6d2010ae
A		 802
803 if ((mgen & PTH_RWL_RETRYBIT) != 0) {
804 retry = 1;
805 mgen &= ~PTH_RWL_RETRYBIT;
806 }
807
808 /* handle first the missed wakeups */
809 if ((kwq->kw_pre_intrcount != 0) &&
810 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE)) &&
811 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
812 kwq->kw_pre_intrcount--;
813 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
814 if (kwq->kw_pre_intrcount==0)
815 CLEAR_INTR_PREPOST_BITS(kwq);
816 ksyn_wqunlock(kwq);
817 *retval = kwe->kwe_psynchretval;
818#if _PSYNCH_TRACE_
819 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, 0xfafafaf1, kwe->kwe_psynchretval, kwq->kw_pre_intrcount, 0);
820#endif /* _PSYNCH_TRACE_ */
821 goto out;
822 }
823
824 if ((kwq->kw_pre_rwwc != 0) && ((ins_flags == FIRSTFIT) || ((lockseq & PTHRW_COUNT_MASK) == (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK) ))) {
b0d623f7
A		 825 /* got preposted lock */
826 kwq->kw_pre_rwwc--;
827 if (kwq->kw_pre_rwwc == 0) {
828 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 829 kwq->kw_lastunlockseq = PTHRW_RWL_INIT;
830 if (kwq->kw_inqueue == 0) {
831 updatebits = lockseq | (PTH_RWL_KBIT | PTH_RWL_EBIT);
832 } else {
833 updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_KBIT | PTH_RWL_EBIT);
834 }
835 updatebits &= ~PTH_RWL_MTX_WAIT;
836
837 kwe->kwe_psynchretval = updatebits;
838
839 if (updatebits == 0) {
840 __FAILEDUSERTEST__("psynch_mutexwait(prepost): returning 0 lseq in mutexwait with no EBIT \n");
841 }
842 ksyn_wqunlock(kwq);
843 *retval = updatebits;
844#if _PSYNCH_TRACE_
845 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
846#endif /* _PSYNCH_TRACE_ */
847 goto out;
b0d623f7 848 } else {
6d2010ae 849 __FAILEDUSERTEST__("psynch_mutexwait: more than one prepost\n");
b0d623f7 850 kwq->kw_pre_lockseq += PTHRW_INC; /* look for next one */
6d2010ae
A		 851 ksyn_wqunlock(kwq);
852 error = EINVAL;
853 goto out;
b0d623f7 854 }
b0d623f7
A		 855 }
856
6d2010ae
A		 857#if _PSYNCH_TRACE_
858 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfeedfeed, mgen, ins_flags, 0);
859#endif /* _PSYNCH_TRACE_ */
b0d623f7 860
6d2010ae
A		 861 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], mgen, uth, kwe, ins_flags);
862 if (error != 0) {
863 ksyn_wqunlock(kwq);
864#if _PSYNCH_TRACE_
865 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 2, 0xdeadbeef, error, 0);
866#endif /* _PSYNCH_TRACE_ */
867 goto out;
868 }
869
7ddcb079
A		 870 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, psynch_mtxcontinue, (void *)kwq);
871
872 psynch_mtxcontinue((void *)kwq, kret);
873
874 /* not expected to return from unix_syscall_return */
875 panic("psynch_mtxcontinue returned from unix_syscall_return");
876
877out:
878 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
879#if _PSYNCH_TRACE_
880 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 0xeeeeeeed, updatebits, error, 0);
881#endif /* _PSYNCH_TRACE_ */
882
883 return(error);
884}
885
886void
887psynch_mtxcontinue(void * parameter, wait_result_t result)
888{
889 int error = 0;
890 uint32_t updatebits = 0;
891 uthread_t uth = current_uthread();
892 ksyn_wait_queue_t kwq = (ksyn_wait_queue_t)parameter;
893 ksyn_waitq_element_t kwe;
894
895 kwe = &uth->uu_kwe;
896
897 switch (result) {
898 case THREAD_TIMED_OUT:
899 error = ETIMEDOUT;
900 break;
901 case THREAD_INTERRUPTED:
902 error = EINTR;
903 break;
904 default:
905 error = 0;
906 break;
907 }
b0d623f7
A		 908
909 if (error != 0) {
910 ksyn_wqlock(kwq);
6d2010ae 911
b0d623f7 912#if _PSYNCH_TRACE_
7ddcb079 913 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, 0xdeadbeef, error, 0);
b0d623f7 914#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 915 if (kwe->kwe_kwqqueue != NULL)
916 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
b0d623f7
A		 917 ksyn_wqunlock(kwq);
918 } else {
6d2010ae
A		 919 updatebits = kwe->kwe_psynchretval;
920 updatebits &= ~PTH_RWL_MTX_WAIT;
7ddcb079 921 uth->uu_rval[0] = updatebits;
6d2010ae
A		 922
923 if (updatebits == 0)
924 __FAILEDUSERTEST__("psynch_mutexwait: returning 0 lseq in mutexwait with no EBIT \n");
b0d623f7 925 }
6d2010ae 926 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
b0d623f7 927#if _PSYNCH_TRACE_
7ddcb079 928 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, updatebits, error, 0);
b0d623f7
A		 929#endif /* _PSYNCH_TRACE_ */
930
7ddcb079 931 unix_syscall_return(error);
2d21ac55
A		 932}
933
b0d623f7
A		 934/*
935 * psynch_mutexdrop: This system call is used for unlock postings on contended psynch mutexes.
936 */
937int
6d2010ae 938psynch_mutexdrop(__unused proc_t p, struct psynch_mutexdrop_args * uap, uint32_t * retval)
b0d623f7
A		 939{
940 user_addr_t mutex = uap->mutex;
941 uint32_t mgen = uap->mgen;
b0d623f7
A		 942 uint32_t ugen = uap->ugen;
943 uint64_t tid = uap->tid;
944 int flags = uap->flags;
945 ksyn_wait_queue_t kwq;
6d2010ae 946 uint32_t updateval;
b0d623f7
A		 947 int error=0;
948
6d2010ae 949 error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
b0d623f7 950 if (error != 0) {
b0d623f7
A		 951 return(error);
952 }
6d2010ae
A		 953
954 updateval = psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
b0d623f7 955 /* drops the kwq reference */
6d2010ae
A		 956
957 *retval = updateval;
b0d623f7
A		 958 return(0);
959
960}
2d21ac55 961
b0d623f7
A		 962/*
963 * psynch_cvbroad: This system call is used for broadcast posting on blocked waiters of psynch cvars.
964 */
965int
6d2010ae 966psynch_cvbroad(__unused proc_t p, struct psynch_cvbroad_args * uap, uint32_t * retval)
2d21ac55 967{
b0d623f7 968 user_addr_t cond = uap->cv;
6d2010ae
A		 969 uint64_t cvlsgen = uap->cvlsgen;
970 uint64_t cvudgen = uap->cvudgen;
971 uint32_t cgen, cugen, csgen, diffgen;
972 uint32_t uptoseq, fromseq;
b0d623f7 973 int flags = uap->flags;
6d2010ae 974 ksyn_wait_queue_t ckwq;
b0d623f7 975 int error=0;
6d2010ae
A		 976 uint32_t updatebits = 0;
977 uint32_t count;
978 struct ksyn_queue kfreeq;
979
980 csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
981 cgen = ((uint32_t)(cvlsgen & 0xffffffff));
982 cugen = (uint32_t)((cvudgen >> 32) & 0xffffffff);
983 diffgen = ((uint32_t)(cvudgen & 0xffffffff));
984 count = (diffgen >> PTHRW_COUNT_SHIFT);
2d21ac55 985
b0d623f7 986#if _PSYNCH_TRACE_
6d2010ae
A		 987 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
988 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_NONE, (uint32_t)cond, 0xcbcbcbc1, diffgen,flags, 0);
b0d623f7 989#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 990
991 uptoseq = cgen & PTHRW_COUNT_MASK;
992 fromseq = (cugen & PTHRW_COUNT_MASK) + PTHRW_INC;
993
994 if (is_seqhigher(fromseq, uptoseq) || is_seqhigher((csgen & PTHRW_COUNT_MASK), uptoseq)) {
995 __FAILEDUSERTEST__("cvbroad: invalid L, U and S values\n");
996 return EINVAL;
997 }
998 if (count > (uint32_t)task_threadmax) {
999 __FAILEDUSERTEST__("cvbroad: difference greater than maximum possible thread count\n");
1000 return EBUSY;
1001 }
1002
1003 ckwq = NULL;
1004
1005 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
b0d623f7
A		 1006 if (error != 0) {
1007#if _PSYNCH_TRACE_
6d2010ae 1008 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
b0d623f7
A		 1009#endif /* _PSYNCH_TRACE_ */
1010 return(error);
1011 }
2d21ac55 1012
6d2010ae 1013 *retval = 0;
b0d623f7
A		 1014
1015 ksyn_wqlock(ckwq);
b0d623f7 1016
6d2010ae
A		 1017 /* update L, U and S... */
1018 UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
b0d623f7 1019
6d2010ae
A		 1020 /* broadcast wakeups/prepost handling */
1021 ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
b0d623f7 1022
6d2010ae
A		 1023 /* set C or P bits and free if needed */
1024 ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1025 ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1026 ksyn_wqunlock(ckwq);
1027
1028 *retval = updatebits;
1029
1030 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
b0d623f7 1031#if _PSYNCH_TRACE_
6d2010ae 1032 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, (uint32_t)*retval, error, 0);
b0d623f7
A		 1033#endif /* _PSYNCH_TRACE_ */
1034
1035 return(error);
2d21ac55
A		 1036}
1037
6d2010ae
A		 1038ksyn_waitq_element_t
1039ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, __unused ksyn_queue_t kq, thread_t th, uint32_t upto)
1040{
1041 uthread_t uth = get_bsdthread_info(th);
1042 ksyn_waitq_element_t kwe = &uth->uu_kwe;
1043
1044 if (kwe->kwe_kwqqueue != ckwq ||
1045 is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) {
1046 /* the thread is not waiting in the cv (or wasn't when the wakeup happened) */
1047 return NULL;
1048 }
1049 return kwe;
1050}
1051
b0d623f7
A		 1052/*
1053 * psynch_cvsignal: This system call is used for signalling the blocked waiters of psynch cvars.
1054 */
1055int
6d2010ae 1056psynch_cvsignal(__unused proc_t p, struct psynch_cvsignal_args * uap, uint32_t * retval)
b0d623f7
A		 1057{
1058 user_addr_t cond = uap->cv;
6d2010ae
A		 1059 uint64_t cvlsgen = uap->cvlsgen;
1060 uint32_t cgen, csgen, signalseq, uptoseq;
b0d623f7 1061 uint32_t cugen = uap->cvugen;
b0d623f7
A		 1062 int threadport = uap->thread_port;
1063 int flags = uap->flags;
6d2010ae
A		 1064 ksyn_wait_queue_t ckwq = NULL;
1065 ksyn_waitq_element_t kwe, nkwe = NULL;
1066 ksyn_queue_t kq;
1067 int error=0;
b0d623f7 1068 thread_t th = THREAD_NULL;
6d2010ae
A		 1069 uint32_t updatebits = 0;
1070 kern_return_t kret;
1071 struct ksyn_queue kfreeq;
b0d623f7
A		 1072
1073
6d2010ae
A		 1074 csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
1075 cgen = ((uint32_t)(cvlsgen & 0xffffffff));
b0d623f7 1076
b0d623f7 1077#if _PSYNCH_TRACE_
6d2010ae 1078 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, threadport, 0);
b0d623f7 1079#endif /* _PSYNCH_TRACE_ */
b0d623f7 1080
6d2010ae
A		 1081 uptoseq = cgen & PTHRW_COUNT_MASK;
1082 signalseq = (cugen & PTHRW_COUNT_MASK) + PTHRW_INC;
b0d623f7 1083
6d2010ae
A		 1084 /* validate sane L, U, and S values */
1085 if (((threadport == 0) && (is_seqhigher(signalseq, uptoseq))) || is_seqhigher((csgen & PTHRW_COUNT_MASK), uptoseq)) {
1086 __FAILEDUSERTEST__("psync_cvsignal; invalid sequence numbers\n");
1087 error = EINVAL;
1088 goto out;
b0d623f7 1089 }
b0d623f7 1090
6d2010ae 1091 /* If we are looking for a specific thread, grab a reference for it */
b0d623f7
A		 1092 if (threadport != 0) {
1093 th = (thread_t)port_name_to_thread((mach_port_name_t)threadport);
1094 if (th == THREAD_NULL) {
b0d623f7
A		 1095 error = ESRCH;
1096 goto out;
1097 }
1098 }
1099
6d2010ae
A		 1100 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1101 if (error != 0) {
b0d623f7 1102#if _PSYNCH_TRACE_
6d2010ae
A		 1103 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
1104#endif /* _PSYNCH_TRACE_ */
1105 goto out;
1106 }
1107
1108 ksyn_wqlock(ckwq);
1109
1110 /* update L, U and S... */
1111 UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1112
1113 kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
1114
1115retry:
1116 /* Only bother if we aren't already balanced */
1117 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
1118
1119 kwe = (th != NULL) ? ksyn_queue_find_threadseq(ckwq, kq, th, uptoseq) :
1120 ksyn_queue_find_signalseq(ckwq, kq, uptoseq, signalseq);
1121 if (kwe != NULL) {
1122 switch (kwe->kwe_flags) {
1123
1124 case KWE_THREAD_BROADCAST:
1125 /* broadcasts swallow our signal */
1126 break;
1127
1128 case KWE_THREAD_PREPOST:
1129 /* merge in with existing prepost at our same uptoseq */
1130 kwe->kwe_count += 1;
1131 break;
1132
1133 case KWE_THREAD_INWAIT:
1134 if (is_seqlower((kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq)) {
1135 /*
1136 * A valid thread in our range, but lower than our signal.
1137 * Matching it may leave our match with nobody to wake it if/when
1138 * it arrives (the signal originally meant for this thread might
1139 * not successfully wake it).
1140 *
1141 * Convert to broadcast - may cause some spurious wakeups
1142 * (allowed by spec), but avoids starvation (better choice).
1143 */
1144#if _PSYNCH_TRACE_
1145 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc1c1c1c1, uptoseq, 0, 0);
b0d623f7 1146#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1147 ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1148 } else {
1149 ksyn_queue_removeitem(ckwq, kq, kwe);
1150 kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
1151 kwe->kwe_kwqqueue = NULL;
b0d623f7 1152#if _PSYNCH_TRACE_
6d2010ae 1153 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
b0d623f7 1154#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1155 kret = ksyn_wakeup_thread(ckwq, kwe);
1156#if __TESTPANICS__
1157 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
1158 panic("ksyn_wakeup_thread: panic waking up condition waiter\n");
1159#endif /* __TESTPANICS__ */
1160 updatebits += PTHRW_INC;
b0d623f7 1161 }
6d2010ae
A		 1162
1163 ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1164 break;
1165
1166 default:
1167 panic("unknown kweflags\n");
1168 break;
b0d623f7 1169 }
6d2010ae
A		 1170
1171 } else if (th != NULL) {
1172 /*
1173 * Could not find the thread, post a broadcast,
1174 * otherwise the waiter will be stuck. Use to send
1175 * ESRCH here, did lead to rare hangs.
1176 */
1177 ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1178 ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1179 } else if (nkwe == NULL) {
b0d623f7 1180 ksyn_wqunlock(ckwq);
6d2010ae
A		 1181 nkwe = (ksyn_waitq_element_t)zalloc(kwe_zone);
1182 ksyn_wqlock(ckwq);
1183 goto retry;
1184
1185 } else {
1186 /* no eligible entries - add prepost */
1187 bzero(nkwe, sizeof(struct ksyn_waitq_element));
1188 nkwe->kwe_kwqqueue = ckwq;
1189 nkwe->kwe_flags = KWE_THREAD_PREPOST;
1190 nkwe->kwe_lockseq = uptoseq;
1191 nkwe->kwe_count = 1;
1192 nkwe->kwe_uth = NULL;
1193 nkwe->kwe_psynchretval = 0;
1194
b0d623f7 1195#if _PSYNCH_TRACE_
6d2010ae 1196 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfefe, uptoseq, 0, 0);
b0d623f7 1197#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1198
1199 (void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], uptoseq, NULL, nkwe, SEQFIT);
1200 ckwq->kw_fakecount++;
1201 nkwe = NULL;
b0d623f7 1202 }
6d2010ae
A		 1203
1204 /* set C or P bits and free if needed */
1205 ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
b0d623f7 1206 }
6d2010ae
A		 1207
1208 ksyn_wqunlock(ckwq);
1209 if (nkwe != NULL)
1210 zfree(kwe_zone, nkwe);
1211
1212 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
1213
b0d623f7 1214out:
6d2010ae 1215 if (th != NULL)
b0d623f7 1216 thread_deallocate(th);
6d2010ae
A		 1217 if (error == 0)
1218 *retval = updatebits;
b0d623f7 1219#if _PSYNCH_TRACE_
6d2010ae 1220 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, updatebits, error, 0);
b0d623f7
A		 1221#endif /* _PSYNCH_TRACE_ */
1222
1223 return(error);
1224}
2d21ac55 1225
b0d623f7
A		 1226/*
1227 * psynch_cvwait: This system call is used for psynch cvar waiters to block in kernel.
1228 */
1229int
1230psynch_cvwait(__unused proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval)
2d21ac55 1231{
b0d623f7 1232 user_addr_t cond = uap->cv;
6d2010ae
A		 1233 uint64_t cvlsgen = uap->cvlsgen;
1234 uint32_t cgen, csgen;
b0d623f7
A		 1235 uint32_t cugen = uap->cvugen;
1236 user_addr_t mutex = uap->mutex;
6d2010ae
A		 1237 uint64_t mugen = uap->mugen;
1238 uint32_t mgen, ugen;
1239 int flags = uap->flags;
b0d623f7 1240 ksyn_wait_queue_t kwq, ckwq;
6d2010ae 1241 int error=0, local_error = 0;
b0d623f7 1242 uint64_t abstime = 0;
6d2010ae 1243 uint32_t lockseq, updatebits=0;
b0d623f7
A		 1244 struct timespec ts;
1245 uthread_t uth;
6d2010ae
A		 1246 ksyn_waitq_element_t kwe, nkwe = NULL;
1247 struct ksyn_queue *kq, kfreeq;
7ddcb079 1248 kern_return_t kret;
6d2010ae 1249
b0d623f7
A		 1250 /* for conformance reasons */
1251 __pthread_testcancel(0);
1252
6d2010ae
A		 1253 csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
1254 cgen = ((uint32_t)(cvlsgen & 0xffffffff));
1255 ugen = (uint32_t)((mugen >> 32) & 0xffffffff);
1256 mgen = ((uint32_t)(mugen & 0xffffffff));
1257
b0d623f7 1258#if _PSYNCH_TRACE_
6d2010ae
A		 1259 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
1260 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)mutex, mgen, ugen, flags, 0);
b0d623f7 1261#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1262
1263 lockseq = (cgen & PTHRW_COUNT_MASK);
1264 /*
1265 * In cvwait U word can be out of range as cond could be used only for
1266 * timeouts. However S word needs to be within bounds and validated at
1267 * user level as well.
1268 */
1269 if (is_seqhigher_eq((csgen & PTHRW_COUNT_MASK), lockseq) != 0) {
1270 __FAILEDUSERTEST__("psync_cvwait; invalid sequence numbers\n");
1271 return EINVAL;
b0d623f7 1272 }
6d2010ae
A		 1273
1274 ckwq = kwq = NULL;
1275 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INWAIT, &ckwq);
b0d623f7
A		 1276 if (error != 0) {
1277#if _PSYNCH_TRACE_
6d2010ae 1278 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)cond, 1, 0xdeadbeef, error, 0);
b0d623f7
A		 1279#endif /* _PSYNCH_TRACE_ */
1280 return(error);
1281 }
1282
b0d623f7 1283
6d2010ae
A		 1284 if (mutex != (user_addr_t)0) {
1285 error = ksyn_wqfind(mutex, mgen, ugen, 0, 0, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
1286 if (error != 0) {
1287 local_error = error;
1288#if _PSYNCH_TRACE_
1289 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)mutex, 2, 0xdeadbeef, error, 0);
1290#endif /* _PSYNCH_TRACE_ */
b0d623f7 1291 goto out;
2d21ac55 1292 }
b0d623f7 1293
6d2010ae 1294 (void)psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
b0d623f7 1295 /* drops kwq reference */
6d2010ae 1296 kwq = NULL;
2d21ac55 1297 }
b0d623f7 1298
6d2010ae 1299 if (uap->sec != 0 || (uap->nsec & 0x3fffffff) != 0) {
b0d623f7 1300 ts.tv_sec = uap->sec;
6d2010ae
A		 1301 ts.tv_nsec = (uap->nsec & 0x3fffffff);
1302 nanoseconds_to_absolutetime((uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec, &abstime );
1303 clock_absolutetime_interval_to_deadline( abstime, &abstime );
b0d623f7 1304 }
6d2010ae 1305
b0d623f7 1306 ksyn_wqlock(ckwq);
6d2010ae
A		 1307
1308 /* update L, U and S... */
1309 UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1310
1311 /* Look for the sequence for prepost (or conflicting thread */
1312 kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
1313 kwe = ksyn_queue_find_cvpreposeq(kq, lockseq);
1314
1315 if (kwe != NULL) {
1316 switch (kwe->kwe_flags) {
1317
1318 case KWE_THREAD_INWAIT:
1319 ksyn_wqunlock(ckwq);
1320 __FAILEDUSERTEST__("cvwait: thread entry with same sequence already present\n");
1321 local_error = EBUSY;
1322 goto out;
1323
1324 case KWE_THREAD_BROADCAST:
1325 break;
1326
1327 case KWE_THREAD_PREPOST:
1328 if ((kwe->kwe_lockseq & PTHRW_COUNT_MASK) == lockseq) {
1329 /* we can safely consume a reference, so do so */
1330 if (--kwe->kwe_count == 0) {
1331 ksyn_queue_removeitem(ckwq, kq, kwe);
1332 ckwq->kw_fakecount--;
1333 nkwe = kwe;
1334 }
1335 } else {
1336 /*
1337 * consuming a prepost higher than our lock sequence is valid, but
1338 * can leave the higher thread without a match. Convert the entry
1339 * to a broadcast to compensate for this.
1340 */
b0d623f7 1341#if _PSYNCH_TRACE_
6d2010ae 1342 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc2c2c2c2, kwe->kwe_lockseq, 0, 0);
b0d623f7 1343#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1344
1345 ksyn_handle_cvbroad(ckwq, kwe->kwe_lockseq, &updatebits);
1346#if __TESTPANICS__
1347 if (updatebits != 0)
1348 panic("psync_cvwait: convert pre-post to broadcast: woke up %d threads that shouldn't be there\n",
1349 updatebits);
1350#endif /* __TESTPANICS__ */
1351 }
1352
1353 break;
b0d623f7 1354
6d2010ae
A		 1355 default:
1356 panic("psync_cvwait: unexpected wait queue element type\n");
1357 }
1358
1359#if _PSYNCH_TRACE_
1360 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfefefefe, kwe->kwe_lockseq, 0, 0);
1361#endif /* _PSYNCH_TRACE_ */
1362
1363
1364 updatebits = PTHRW_INC;
1365 ckwq->kw_sword += PTHRW_INC;
1366
1367 /* set C or P bits and free if needed */
1368 ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1369
b0d623f7 1370 error = 0;
6d2010ae
A		 1371 local_error = 0;
1372
1373 *retval = updatebits;
1374
b0d623f7 1375 ksyn_wqunlock(ckwq);
6d2010ae
A		 1376
1377 if (nkwe != NULL)
1378 zfree(kwe_zone, nkwe);
1379
b0d623f7 1380 goto out;
6d2010ae
A		 1381
1382 }
b0d623f7 1383
6d2010ae
A		 1384 uth = current_uthread();
1385 kwe = &uth->uu_kwe;
1386 kwe->kwe_kwqqueue = ckwq;
1387 kwe->kwe_flags = KWE_THREAD_INWAIT;
1388 kwe->kwe_lockseq = lockseq;
1389 kwe->kwe_count = 1;
1390 kwe->kwe_uth = uth;
1391 kwe->kwe_psynchretval = 0;
1392
b0d623f7 1393#if _PSYNCH_TRACE_
6d2010ae 1394 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, cgen, 0, 0);
b0d623f7 1395#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1396
1397 error = ksyn_queue_insert(ckwq, kq, cgen, uth, kwe, SEQFIT);
1398 if (error != 0) {
1399 ksyn_wqunlock(ckwq);
1400 local_error = error;
1401 goto out;
b0d623f7 1402 }
6d2010ae 1403
7ddcb079 1404 kret = ksyn_block_thread_locked(ckwq, abstime, kwe, 1, psynch_cvcontinue, (void *)ckwq);
6d2010ae
A		 1405 /* lock dropped */
1406
7ddcb079
A		 1407 psynch_cvcontinue(ckwq, kret);
1408 /* not expected to return from unix_syscall_return */
1409 panic("psynch_cvcontinue returned from unix_syscall_return");
1410
1411out:
1412#if _PSYNCH_TRACE_
1413 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, (uint32_t)*retval, local_error, 0);
1414#endif /* _PSYNCH_TRACE_ */
1415 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
1416 return(local_error);
1417}
1418
1419
1420void
1421psynch_cvcontinue(void * parameter, wait_result_t result)
1422{
1423 int error = 0, local_error = 0;
1424 uthread_t uth = current_uthread();
1425 ksyn_wait_queue_t ckwq = (ksyn_wait_queue_t)parameter;
1426 ksyn_waitq_element_t kwe;
1427 struct ksyn_queue kfreeq;
1428
1429 switch (result) {
1430 case THREAD_TIMED_OUT:
1431 error = ETIMEDOUT;
1432 break;
1433 case THREAD_INTERRUPTED:
1434 error = EINTR;
1435 break;
1436 default:
1437 error = 0;
1438 break;
1439 }
1440#if _PSYNCH_TRACE_
1441 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf4f3f2f1, (uintptr_t)uth, result, 0, 0);
1442#endif /* _PSYNCH_TRACE_ */
1443
6d2010ae 1444 local_error = error;
7ddcb079
A		 1445 kwe = &uth->uu_kwe;
1446
b0d623f7
A		 1447 if (error != 0) {
1448 ksyn_wqlock(ckwq);
6d2010ae 1449 /* just in case it got woken up as we were granting */
7ddcb079 1450 uth->uu_rval[0] = kwe->kwe_psynchretval;
6d2010ae
A		 1451
1452#if __TESTPANICS__
1453 if ((kwe->kwe_kwqqueue != NULL) && (kwe->kwe_kwqqueue != ckwq))
1454 panic("cvwait waiting on some other kwq\n");
1455
1456#endif /* __TESTPANICS__ */
1457
1458
1459 if (kwe->kwe_kwqqueue != NULL) {
1460 ksyn_queue_removeitem(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
1461 kwe->kwe_kwqqueue = NULL;
1462 }
1463 if ((kwe->kwe_psynchretval & PTH_RWL_MTX_WAIT) != 0) {
7ddcb079 1464 /* the condition var granted.
6d2010ae
A		 1465 * reset the error so that the thread returns back.
1466 */
1467 local_error = 0;
1468 /* no need to set any bits just return as cvsig/broad covers this */
1469 ksyn_wqunlock(ckwq);
6d2010ae
A		 1470 goto out;
1471 }
1472
1473 ckwq->kw_sword += PTHRW_INC;
1474
7ddcb079 1475 /* set C and P bits, in the local error */
6d2010ae 1476 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
6d2010ae
A		 1477 local_error |= ECVCERORR;
1478 if (ckwq->kw_inqueue != 0) {
7ddcb079 1479 (void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (ckwq->kw_lword & PTHRW_COUNT_MASK), &kfreeq, 1, 1);
6d2010ae
A		 1480 }
1481 ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
1482 ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
1483 } else {
1484 /* everythig in the queue is a fake entry ? */
1485 if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
6d2010ae
A		 1486 local_error |= ECVPERORR;
1487 }
1488 }
1489 ksyn_wqunlock(ckwq);
1490
1491 } else {
1492 /* PTH_RWL_MTX_WAIT is removed */
1493 if ((kwe->kwe_psynchretval & PTH_RWS_CV_MBIT) != 0)
7ddcb079 1494 uth->uu_rval[0] = PTHRW_INC | PTH_RWS_CV_CBIT;
6d2010ae 1495 else
7ddcb079 1496 uth->uu_rval[0] = 0;
6d2010ae
A		 1497 local_error = 0;
1498 }
1499out:
1500#if _PSYNCH_TRACE_
7ddcb079 1501 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)ckwq->kw_addr, 0xeeeeeeed, uth->uu_rval[0], local_error, 0);
6d2010ae
A		 1502#endif /* _PSYNCH_TRACE_ */
1503 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
7ddcb079
A		 1504
1505 unix_syscall_return(local_error);
1506
6d2010ae
A		 1507}
1508
1509/*
1510 * psynch_cvclrprepost: This system call clears pending prepost if present.
1511 */
1512int
1513psynch_cvclrprepost(__unused proc_t p, struct psynch_cvclrprepost_args * uap, __unused int * retval)
1514{
1515 user_addr_t cond = uap->cv;
1516 uint32_t cgen = uap->cvgen;
1517 uint32_t cugen = uap->cvugen;
1518 uint32_t csgen = uap->cvsgen;
1519 uint32_t pseq = uap->preposeq;
1520 uint32_t flags = uap->flags;
1521 int error;
1522 ksyn_wait_queue_t ckwq = NULL;
1523 struct ksyn_queue kfreeq;
1524
b0d623f7 1525#if _PSYNCH_TRACE_
6d2010ae
A		 1526 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
1527 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_NONE, (uint32_t)cond, 0xcececece, pseq, flags, 0);
b0d623f7 1528#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1529
1530 if ((flags & _PTHREAD_MTX_OPT_MUTEX) == 0) {
1531 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1532 if (error != 0) {
1533 *retval = 0;
1534#if _PSYNCH_TRACE_
1535 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
1536#endif /* _PSYNCH_TRACE_ */
1537 return(error);
b0d623f7 1538 }
6d2010ae
A		 1539
1540 ksyn_wqlock(ckwq);
1541 (void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (pseq & PTHRW_COUNT_MASK), &kfreeq, 0, 1);
b0d623f7 1542 ksyn_wqunlock(ckwq);
6d2010ae
A		 1543 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP));
1544 } else {
1545 /* mutex type */
1546 error = ksyn_wqfind(cond, cgen, cugen, 0, 0, flags, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP), &ckwq);
1547 if (error != 0) {
1548 *retval = 0;
1549#if _PSYNCH_TRACE_
1550 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
1551#endif /* _PSYNCH_TRACE_ */
1552 return(error);
1553 }
b0d623f7 1554
6d2010ae
A		 1555 ksyn_wqlock(ckwq);
1556 if (((flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT) != 0) && (ckwq->kw_pre_rwwc != 0)) {
1557 if (is_seqlower_eq(ckwq->kw_pre_lockseq, cgen) != 0) {
1558 /* clear prepost */
1559 ckwq->kw_pre_rwwc = 0;
1560 ckwq->kw_pre_lockseq = 0;
1561 }
1562 }
1563 ksyn_wqunlock(ckwq);
1564 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP));
b0d623f7 1565 }
6d2010ae 1566
b0d623f7 1567#if _PSYNCH_TRACE_
6d2010ae 1568 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, 0, 0, 0);
b0d623f7 1569#endif /* _PSYNCH_TRACE_ */
6d2010ae 1570 return(0);
2d21ac55
A		 1571}
1572
b0d623f7
A		 1573/* ***************** pthread_rwlock ************************ */
1574/*
1575 * psynch_rw_rdlock: This system call is used for psync rwlock readers to block.
1576 */
1577int
1578psynch_rw_rdlock(__unused proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval)
1579{
1580 user_addr_t rwlock = uap->rwlock;
1581 uint32_t lgen = uap->lgenval;
1582 uint32_t ugen = uap->ugenval;
1583 uint32_t rw_wc = uap->rw_wc;
1584 //uint64_t tid = uap->tid;
1585 int flags = uap->flags;
1586 int error = 0, block;
6d2010ae 1587 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
b0d623f7
A		 1588 ksyn_wait_queue_t kwq;
1589 uthread_t uth;
6d2010ae
A		 1590 int isinit = lgen & PTHRW_RWL_INIT;
1591 uint32_t returnbits = 0;
1592 ksyn_waitq_element_t kwe;
7ddcb079 1593 kern_return_t kret;
2d21ac55 1594
b0d623f7 1595#if _PSYNCH_TRACE_
6d2010ae 1596 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 1597#endif /* _PSYNCH_TRACE_ */
1598 uth = current_uthread();
1599
1600 /* preserve the seq number */
6d2010ae
A		 1601 kwe = &uth->uu_kwe;
1602 kwe->kwe_lockseq = lgen;
1603 kwe->kwe_uth = uth;
1604 kwe->kwe_psynchretval = 0;
1605 kwe->kwe_kwqqueue = NULL;
1606
b0d623f7
A		 1607 lockseq = lgen & PTHRW_COUNT_MASK;
1608
6d2010ae 1609
b0d623f7
A		 1610 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1611 if (error != 0) {
1612#if _PSYNCH_TRACE_
6d2010ae 1613 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1614#endif /* _PSYNCH_TRACE_ */
1615 return(error);
1616 }
1617
1618 ksyn_wqlock(kwq);
1619
6d2010ae
A		 1620 if (isinit != 0) {
1621 lgen &= ~PTHRW_RWL_INIT;
1622 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1623 /* first to notice the reset of the lock, clear preposts */
1624 CLEAR_REINIT_BITS(kwq);
1625 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1626#if _PSYNCH_TRACE_
1627 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
1628#endif /* _PSYNCH_TRACE_ */
1629 }
1630 }
1631
b0d623f7
A		 1632 /* handle first the missed wakeups */
1633 if ((kwq->kw_pre_intrcount != 0) &&
1634 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
1635 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1636
1637 kwq->kw_pre_intrcount--;
6d2010ae 1638 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 1639 if (kwq->kw_pre_intrcount==0)
1640 CLEAR_INTR_PREPOST_BITS(kwq);
1641 ksyn_wqunlock(kwq);
1642 goto out;
1643 }
1644
6d2010ae
A		 1645 /* handle overlap first as they are not counted against pre_rwwc */
1646
1647 /* check for overlap and if no pending W bit (indicates writers) */
1648 if ((kwq->kw_overlapwatch != 0) && ((rw_wc & PTHRW_RWS_SAVEMASK) == 0) && ((lgen & PTH_RWL_WBIT) == 0)) {
b0d623f7 1649#if _PSYNCH_TRACE_
6d2010ae 1650 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 10, kwq->kw_nextseqword, kwq->kw_lastseqword, 0);
b0d623f7 1651#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1652 error = kwq_handle_overlap(kwq, lgen, ugen, rw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block);
1653#if __TESTPANICS__
1654 if (error != 0)
1655 panic("rw_rdlock: kwq_handle_overlap failed %d\n",error);
1656#endif /* __TESTPANICS__ */
1657 if (block == 0) {
1658 error = 0;
1659 kwe->kwe_psynchretval = updatebits;
1660#if _PSYNCH_TRACE_
1661 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xff, updatebits, 0xee, 0);
1662#endif /* _PSYNCH_TRACE_ */
1663 ksyn_wqunlock(kwq);
1664 goto out;
1665 }
b0d623f7
A		 1666 }
1667
1668 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1669#if _PSYNCH_TRACE_
6d2010ae 1670 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 1671#endif /* _PSYNCH_TRACE_ */
1672 kwq->kw_pre_rwwc--;
1673 if (kwq->kw_pre_rwwc == 0) {
1674 preseq = kwq->kw_pre_lockseq;
6d2010ae 1675 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 1676 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 1677 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1678 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1679#if _PSYNCH_TRACE_
1680 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
1681#endif /* _PSYNCH_TRACE_ */
1682 }
1683 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1684#if __TESTPANICS__
b0d623f7 1685 if (error != 0)
6d2010ae
A		 1686 panic("rw_rdlock: kwq_handle_unlock failed %d\n",error);
1687#endif /* __TESTPANICS__ */
b0d623f7
A		 1688 if (block == 0) {
1689 ksyn_wqunlock(kwq);
1690 goto out;
1691 }
1692 /* insert to q and proceed as ususal */
1693 }
1694 }
1695
6d2010ae 1696
b0d623f7 1697#if _PSYNCH_TRACE_
6d2010ae 1698 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 1699#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1700 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], lgen, uth, kwe, SEQFIT);
1701#if __TESTPANICS__
b0d623f7
A		 1702 if (error != 0)
1703 panic("psynch_rw_rdlock: failed to enqueue\n");
6d2010ae 1704#endif /* __TESTPANICS__ */
7ddcb079 1705 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 1706 /* drops the kwq lock */
7ddcb079
A		 1707 switch (kret) {
1708 case THREAD_TIMED_OUT:
1709 error = ETIMEDOUT;
1710 break;
1711 case THREAD_INTERRUPTED:
1712 error = EINTR;
1713 break;
1714 default:
1715 error = 0;
1716 break;
1717 }
b0d623f7
A		 1718
1719out:
1720 if (error != 0) {
1721#if _PSYNCH_TRACE_
6d2010ae 1722 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 1723#endif /* _PSYNCH_TRACE_ */
1724 ksyn_wqlock(kwq);
6d2010ae
A		 1725 if (kwe->kwe_kwqqueue != NULL)
1726 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwe);
b0d623f7
A		 1727 ksyn_wqunlock(kwq);
1728 } else {
1729 /* update bits */
6d2010ae
A		 1730 *retval = kwe->kwe_psynchretval;
1731 returnbits = kwe->kwe_psynchretval;
b0d623f7 1732 }
6d2010ae 1733 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
b0d623f7 1734#if _PSYNCH_TRACE_
6d2010ae 1735 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
b0d623f7
A		 1736#endif /* _PSYNCH_TRACE_ */
1737 return(error);
1738}
1739
1740/*
1741 * psynch_rw_longrdlock: This system call is used for psync rwlock long readers to block.
1742 */
1743int
316670eb
A		 1744#ifdef NOTYET
1745psynch_rw_longrdlock(__unused proc_t p, struct psynch_rw_longrdlock_args * uap, __unused uint32_t * retval)
1746#else /* NOTYET */
6d2010ae 1747psynch_rw_longrdlock(__unused proc_t p, __unused struct psynch_rw_longrdlock_args * uap, __unused uint32_t * retval)
316670eb 1748#endif /* NOTYET */
2d21ac55 1749{
316670eb 1750#ifdef NOTYET
b0d623f7
A		 1751 user_addr_t rwlock = uap->rwlock;
1752 uint32_t lgen = uap->lgenval;
1753 uint32_t ugen = uap->ugenval;
1754 uint32_t rw_wc = uap->rw_wc;
1755 //uint64_t tid = uap->tid;
1756 int flags = uap->flags;
6d2010ae
A		 1757 int isinit = lgen & PTHRW_RWL_INIT;
1758 uint32_t returnbits=0;
1759 ksyn_waitq_element_t kwe;
7ddcb079 1760 kern_return_t kret;
b0d623f7
A		 1761
1762 ksyn_wait_queue_t kwq;
1763 int error=0, block = 0 ;
1764 uthread_t uth;
6d2010ae 1765 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
b0d623f7
A		 1766
1767#if _PSYNCH_TRACE_
6d2010ae 1768 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 1769#endif /* _PSYNCH_TRACE_ */
1770 uth = current_uthread();
6d2010ae
A		 1771 kwe = &uth->uu_kwe;
1772 kwe->kwe_lockseq = lgen;
1773 kwe->kwe_uth = uth;
1774 kwe->kwe_psynchretval = 0;
1775 kwe->kwe_kwqqueue = NULL;
b0d623f7 1776 lockseq = (lgen & PTHRW_COUNT_MASK);
6d2010ae 1777
b0d623f7
A		 1778 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1779 if (error != 0) {
1780#if _PSYNCH_TRACE_
6d2010ae 1781 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1782#endif /* _PSYNCH_TRACE_ */
1783 return(error);
2d21ac55 1784 }
b0d623f7
A		 1785
1786 ksyn_wqlock(kwq);
1787
6d2010ae
A		 1788 if (isinit != 0) {
1789 lgen &= ~PTHRW_RWL_INIT;
1790 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1791 /* first to notice the reset of the lock, clear preposts */
1792 CLEAR_REINIT_BITS(kwq);
1793 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1794#if _PSYNCH_TRACE_
1795 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
1796#endif /* _PSYNCH_TRACE_ */
1797 }
1798 }
1799
b0d623f7
A		 1800 /* handle first the missed wakeups */
1801 if ((kwq->kw_pre_intrcount != 0) &&
1802 (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD) &&
1803 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1804
1805 kwq->kw_pre_intrcount--;
6d2010ae 1806 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 1807 if (kwq->kw_pre_intrcount==0)
1808 CLEAR_INTR_PREPOST_BITS(kwq);
1809 ksyn_wqunlock(kwq);
1810 goto out;
1811 }
1812
b0d623f7
A		 1813
1814 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1815#if _PSYNCH_TRACE_
6d2010ae 1816 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 1817#endif /* _PSYNCH_TRACE_ */
1818 kwq->kw_pre_rwwc--;
1819 if (kwq->kw_pre_rwwc == 0) {
1820 preseq = kwq->kw_pre_lockseq;
6d2010ae 1821 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 1822 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 1823 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1824 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1825#if _PSYNCH_TRACE_
1826 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
1827#endif /* _PSYNCH_TRACE_ */
1828 }
1829 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_LREADLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1830#if __TESTPANICS__
b0d623f7
A		 1831 if (error != 0)
1832 panic("kwq_handle_unlock failed %d\n",error);
6d2010ae 1833#endif /* __TESTPANICS__ */
b0d623f7
A		 1834 if (block == 0) {
1835 ksyn_wqunlock(kwq);
1836 goto out;
1837 }
1838 /* insert to q and proceed as ususal */
1839 }
1840 }
1841
1842#if _PSYNCH_TRACE_
6d2010ae 1843 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 1844#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1845 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], lgen, uth, kwe, SEQFIT);
1846#if __TESTPANICS__
b0d623f7
A		 1847 if (error != 0)
1848 panic("psynch_rw_longrdlock: failed to enqueue\n");
6d2010ae 1849#endif /* __TESTPANICS__ */
b0d623f7 1850
7ddcb079 1851 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 1852 /* drops the kwq lock */
7ddcb079
A		 1853 switch (kret) {
1854 case THREAD_TIMED_OUT:
1855 error = ETIMEDOUT;
1856 break;
1857 case THREAD_INTERRUPTED:
1858 error = EINTR;
1859 break;
1860 default:
1861 error = 0;
1862 break;
1863 }
b0d623f7
A		 1864out:
1865 if (error != 0) {
1866#if _PSYNCH_TRACE_
6d2010ae 1867 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1868#endif /* _PSYNCH_TRACE_ */
1869 ksyn_wqlock(kwq);
6d2010ae
A		 1870 if (kwe->kwe_kwqqueue != NULL)
1871 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwe);
b0d623f7
A		 1872 ksyn_wqunlock(kwq);
1873 } else {
1874 /* update bits */
6d2010ae
A		 1875 *retval = kwe->kwe_psynchretval;
1876 returnbits = kwe->kwe_psynchretval;
b0d623f7
A		 1877 }
1878
6d2010ae 1879 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 1880
1881#if _PSYNCH_TRACE_
6d2010ae 1882 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, returnbits, error, 0);
b0d623f7
A		 1883#endif /* _PSYNCH_TRACE_ */
1884 return(error);
316670eb
A		 1885#else /* NOTYET */
1886 return(ESRCH);
1887#endif /* NOTYET */
2d21ac55
A		 1888}
1889
316670eb 1890
b0d623f7
A		 1891/*
1892 * psynch_rw_wrlock: This system call is used for psync rwlock writers to block.
1893 */
1894int
1895psynch_rw_wrlock(__unused proc_t p, struct psynch_rw_wrlock_args * uap, uint32_t * retval)
1896{
1897 user_addr_t rwlock = uap->rwlock;
1898 uint32_t lgen = uap->lgenval;
1899 uint32_t ugen = uap->ugenval;
1900 uint32_t rw_wc = uap->rw_wc;
1901 //uint64_t tid = uap->tid;
1902 int flags = uap->flags;
1903 int block;
1904 ksyn_wait_queue_t kwq;
1905 int error=0;
1906 uthread_t uth;
6d2010ae
A		 1907 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
1908 int isinit = lgen & PTHRW_RWL_INIT;
1909 uint32_t returnbits = 0;
1910 ksyn_waitq_element_t kwe;
7ddcb079 1911 kern_return_t kret;
2d21ac55 1912
b0d623f7 1913#if _PSYNCH_TRACE_
6d2010ae 1914 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 1915#endif /* _PSYNCH_TRACE_ */
1916 uth = current_uthread();
6d2010ae
A		 1917 kwe = &uth->uu_kwe;
1918 kwe->kwe_lockseq = lgen;
1919 kwe->kwe_uth = uth;
1920 kwe->kwe_psynchretval = 0;
1921 kwe->kwe_kwqqueue = NULL;
b0d623f7
A		 1922 lockseq = (lgen & PTHRW_COUNT_MASK);
1923
1924 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1925 if (error != 0) {
1926#if _PSYNCH_TRACE_
6d2010ae 1927 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1928#endif /* _PSYNCH_TRACE_ */
1929 return(error);
1930 }
1931
1932 ksyn_wqlock(kwq);
1933
6d2010ae
A		 1934
1935 if (isinit != 0) {
1936 lgen &= ~PTHRW_RWL_INIT;
1937 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1938 /* first to notice the reset of the lock, clear preposts */
1939 CLEAR_REINIT_BITS(kwq);
1940 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1941#if _PSYNCH_TRACE_
1942 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
1943#endif /* _PSYNCH_TRACE_ */
1944 }
1945 }
1946
1947
b0d623f7
A		 1948 /* handle first the missed wakeups */
1949 if ((kwq->kw_pre_intrcount != 0) &&
1950 (kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE) &&
1951 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1952
1953 kwq->kw_pre_intrcount--;
6d2010ae 1954 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 1955 if (kwq->kw_pre_intrcount==0)
1956 CLEAR_INTR_PREPOST_BITS(kwq);
1957 ksyn_wqunlock(kwq);
1958 goto out;
1959 }
1960
b0d623f7
A		 1961
1962 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1963#if _PSYNCH_TRACE_
6d2010ae 1964 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 1965#endif /* _PSYNCH_TRACE_ */
1966 kwq->kw_pre_rwwc--;
1967 if (kwq->kw_pre_rwwc == 0) {
1968 preseq = kwq->kw_pre_lockseq;
6d2010ae 1969 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 1970 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 1971 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1972 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1973#if _PSYNCH_TRACE_
1974 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
1975#endif /* _PSYNCH_TRACE_ */
1976 }
1977 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_WRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1978#if __TESTPANICS__
b0d623f7 1979 if (error != 0)
6d2010ae
A		 1980 panic("rw_wrlock: kwq_handle_unlock failed %d\n",error);
1981#endif /* __TESTPANICS__ */
b0d623f7
A		 1982 if (block == 0) {
1983 ksyn_wqunlock(kwq);
6d2010ae
A		 1984 *retval = updatebits;
1985 goto out1;
b0d623f7
A		 1986 }
1987 /* insert to q and proceed as ususal */
1988 }
1989 }
1990
6d2010ae
A		 1991 /* No overlap watch needed go ahead and block */
1992
b0d623f7 1993#if _PSYNCH_TRACE_
6d2010ae 1994 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 1995#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1996 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], lgen, uth, kwe, SEQFIT);
1997#if __TESTPANICS__
b0d623f7
A		 1998 if (error != 0)
1999 panic("psynch_rw_wrlock: failed to enqueue\n");
6d2010ae 2000#endif /* __TESTPANICS__ */
b0d623f7 2001
7ddcb079 2002 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 2003 /* drops the wq lock */
7ddcb079
A		 2004 switch (kret) {
2005 case THREAD_TIMED_OUT:
2006 error = ETIMEDOUT;
2007 break;
2008 case THREAD_INTERRUPTED:
2009 error = EINTR;
2010 break;
2011 default:
2012 error = 0;
2013 break;
2014 }
b0d623f7
A		 2015
2016out:
2017 if (error != 0) {
2018#if _PSYNCH_TRACE_
6d2010ae 2019 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 2020#endif /* _PSYNCH_TRACE_ */
2021 ksyn_wqlock(kwq);
6d2010ae
A		 2022 if (kwe->kwe_kwqqueue != NULL)
2023 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
b0d623f7
A		 2024 ksyn_wqunlock(kwq);
2025 } else {
2026 /* update bits */
6d2010ae
A		 2027 *retval = kwe->kwe_psynchretval;
2028 returnbits = kwe->kwe_psynchretval;
b0d623f7 2029 }
6d2010ae
A		 2030out1:
2031 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 2032
2033#if _PSYNCH_TRACE_
6d2010ae 2034 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
b0d623f7
A		 2035#endif /* _PSYNCH_TRACE_ */
2036 return(error);
2037}
2038
2039/*
2040 * psynch_rw_yieldwrlock: This system call is used for psync rwlock yielding writers to block.
2041 */
2042int
316670eb 2043#ifdef NOTYET
6d2010ae 2044psynch_rw_yieldwrlock(__unused proc_t p, __unused struct psynch_rw_yieldwrlock_args * uap, __unused uint32_t * retval)
316670eb
A		 2045#else /* NOTYET */
2046psynch_rw_yieldwrlock(__unused proc_t p, __unused struct __unused psynch_rw_yieldwrlock_args * uap, __unused uint32_t * retval)
2047#endif /* NOTYET */
2d21ac55 2048{
316670eb 2049#ifdef NOTYET
b0d623f7
A		 2050 user_addr_t rwlock = uap->rwlock;
2051 uint32_t lgen = uap->lgenval;
2052 uint32_t ugen = uap->ugenval;
2053 uint32_t rw_wc = uap->rw_wc;
2054 //uint64_t tid = uap->tid;
2055 int flags = uap->flags;
2056 int block;
2057 ksyn_wait_queue_t kwq;
2058 int error=0;
6d2010ae 2059 int isinit = lgen & PTHRW_RWL_INIT;
b0d623f7 2060 uthread_t uth;
6d2010ae
A		 2061 uint32_t returnbits=0;
2062 ksyn_waitq_element_t kwe;
7ddcb079 2063 kern_return_t kret;
b0d623f7
A		 2064
2065#if _PSYNCH_TRACE_
6d2010ae 2066 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7 2067#endif /* _PSYNCH_TRACE_ */
6d2010ae 2068 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
b0d623f7
A		 2069
2070 uth = current_uthread();
6d2010ae
A		 2071 kwe = &uth->uu_kwe;
2072 kwe->kwe_lockseq = lgen;
2073 kwe->kwe_uth = uth;
2074 kwe->kwe_psynchretval = 0;
2075 kwe->kwe_kwqqueue = NULL;
b0d623f7
A		 2076 lockseq = (lgen & PTHRW_COUNT_MASK);
2077
2078 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
2079 if (error != 0) {
2080#if _PSYNCH_TRACE_
6d2010ae 2081 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2082#endif /* _PSYNCH_TRACE_ */
2083 return(error);
2084 }
2085
2086 ksyn_wqlock(kwq);
2087
6d2010ae
A		 2088 if (isinit != 0) {
2089 lgen &= ~PTHRW_RWL_INIT;
2090 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
2091 /* first to notice the reset of the lock, clear preposts */
2092 CLEAR_REINIT_BITS(kwq);
2093 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2094#if _PSYNCH_TRACE_
2095 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2096#endif /* _PSYNCH_TRACE_ */
2097 }
2098 }
2099
b0d623f7
A		 2100 /* handle first the missed wakeups */
2101 if ((kwq->kw_pre_intrcount != 0) &&
2102 (kwq->kw_pre_intrtype == PTH_RW_TYPE_YWRITE) &&
2103 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
2104
2105 kwq->kw_pre_intrcount--;
6d2010ae 2106 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 2107 if (kwq->kw_pre_intrcount==0)
2108 CLEAR_INTR_PREPOST_BITS(kwq);
2109 ksyn_wqunlock(kwq);
2110 goto out;
2111 }
2112
b0d623f7
A		 2113 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
2114#if _PSYNCH_TRACE_
6d2010ae 2115 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 2116#endif /* _PSYNCH_TRACE_ */
2117 kwq->kw_pre_rwwc--;
2118 if (kwq->kw_pre_rwwc == 0) {
2119 preseq = kwq->kw_pre_lockseq;
6d2010ae 2120 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 2121 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 2122 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
2123 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2124#if _PSYNCH_TRACE_
2125 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2126#endif /* _PSYNCH_TRACE_ */
2127 }
2128 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_YWRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
2129#if __TESTPANICS__
b0d623f7
A		 2130 if (error != 0)
2131 panic("kwq_handle_unlock failed %d\n",error);
6d2010ae 2132#endif /* __TESTPANICS__ */
b0d623f7
A		 2133 if (block == 0) {
2134 ksyn_wqunlock(kwq);
6d2010ae 2135 *retval = updatebits;
b0d623f7
A		 2136 goto out;
2137 }
2138 /* insert to q and proceed as ususal */
2139 }
2140 }
2141
2142#if _PSYNCH_TRACE_
6d2010ae 2143 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 2144#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2145 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], lgen, uth, kwe, SEQFIT);
2146#if __TESTPANICS__
b0d623f7
A		 2147 if (error != 0)
2148 panic("psynch_rw_yieldwrlock: failed to enqueue\n");
6d2010ae 2149#endif /* __TESTPANICS__ */
b0d623f7 2150
7ddcb079
A		 2151 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2152 switch (kret) {
2153 case THREAD_TIMED_OUT:
2154 error = ETIMEDOUT;
2155 break;
2156 case THREAD_INTERRUPTED:
2157 error = EINTR;
2158 break;
2159 default:
2160 error = 0;
2161 break;
2162 }
b0d623f7
A		 2163
2164out:
2165 if (error != 0) {
2166#if _PSYNCH_TRACE_
6d2010ae 2167 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 2168#endif /* _PSYNCH_TRACE_ */
2169 ksyn_wqlock(kwq);
6d2010ae
A		 2170 if (kwe->kwe_kwqqueue != NULL)
2171 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwe);
b0d623f7
A		 2172 ksyn_wqunlock(kwq);
2173 } else {
2174 /* update bits */
6d2010ae
A		 2175 *retval = kwe->kwe_psynchretval;
2176 returnbits = kwe->kwe_psynchretval;
b0d623f7
A		 2177 }
2178
6d2010ae 2179 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 2180
2181#if _PSYNCH_TRACE_
6d2010ae 2182 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
b0d623f7
A		 2183#endif /* _PSYNCH_TRACE_ */
2184 return(error);
316670eb
A		 2185#else /* NOTYET */
2186 return(ESRCH);
2187#endif /* NOTYET */
b0d623f7
A		 2188}
2189
6d2010ae 2190#if NOTYET
b0d623f7
A		 2191/*
2192 * psynch_rw_downgrade: This system call is used for wakeup blocked readers who are eligible to run due to downgrade.
2193 */
2194int
2195psynch_rw_downgrade(__unused proc_t p, struct psynch_rw_downgrade_args * uap, __unused int * retval)
2196{
2197 user_addr_t rwlock = uap->rwlock;
2198 uint32_t lgen = uap->lgenval;
2199 uint32_t ugen = uap->ugenval;
2200 uint32_t rw_wc = uap->rw_wc;
2201 //uint64_t tid = uap->tid;
2202 int flags = uap->flags;
2203 uint32_t count = 0;
6d2010ae 2204 int isinit = lgen & PTHRW_RWL_INIT;
b0d623f7
A		 2205 ksyn_wait_queue_t kwq;
2206 int error=0;
2207 uthread_t uth;
2208 uint32_t curgen = 0;
2209
2210#if _PSYNCH_TRACE_
6d2010ae 2211 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 2212#endif /* _PSYNCH_TRACE_ */
2213 uth = current_uthread();
2214
2215 curgen = (lgen & PTHRW_COUNT_MASK);
2216
6d2010ae 2217 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
b0d623f7
A		 2218 if (error != 0) {
2219#if _PSYNCH_TRACE_
6d2010ae 2220 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2221#endif /* _PSYNCH_TRACE_ */
2222 return(error);
2223 }
2224
2225 ksyn_wqlock(kwq);
2226
6d2010ae
A		 2227 if ((lgen & PTHRW_RWL_INIT) != 0) {
2228 lgen &= ~PTHRW_RWL_INIT;
2229 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
2230 CLEAR_REINIT_BITS(kwq);
2231 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2232#if _PSYNCH_TRACE_
2233 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2234#endif /* _PSYNCH_TRACE_ */
2235 }
2236 isinit = 1;
2237 }
2238
2239 /* if lastunlock seq is set, ensure the current one is not lower than that, as it would be spurious */
2240 if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
b0d623f7 2241 /* spurious updatebits?? */
6d2010ae 2242 error = 0;
b0d623f7
A		 2243 goto out;
2244 }
b0d623f7 2245
6d2010ae
A		 2246
2247
2248 /* If L-U != num of waiters, then it needs to be preposted or spr */
2249 diff = find_diff(lgen, ugen);
2250 /* take count of the downgrade thread itself */
2251 diff--;
2252
2253
2254#if _PSYNCH_TRACE_
2255 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
2256#endif /* _PSYNCH_TRACE_ */
2257 if (find_seq_till(kwq, curgen, diff, &count) == 0) {
2258 if (count < (uint32_t)diff)
2259 goto prepost;
b0d623f7 2260 }
6d2010ae
A		 2261
2262 /* no prepost and all threads are in place, reset the bit */
2263 if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
2264 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2265#if _PSYNCH_TRACE_
2266 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2267#endif /* _PSYNCH_TRACE_ */
b0d623f7 2268 }
6d2010ae
A		 2269
2270 /* can handle unlock now */
b0d623f7 2271
6d2010ae
A		 2272 CLEAR_PREPOST_BITS(kwq);
2273
b0d623f7
A		 2274dounlock:
2275#if _PSYNCH_TRACE_
6d2010ae 2276 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7
A		 2277#endif /* _PSYNCH_TRACE_ */
2278 error = kwq_handle_downgrade(kwq, lgen, 0, 0, NULL);
2279
6d2010ae 2280#if __TESTPANICS__
b0d623f7
A		 2281 if (error != 0)
2282 panic("psynch_rw_downgrade: failed to wakeup\n");
6d2010ae 2283#endif /* __TESTPANICS__ */
b0d623f7
A		 2284
2285out:
2286 ksyn_wqunlock(kwq);
2287#if _PSYNCH_TRACE_
6d2010ae 2288 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_END, (uint32_t)rwlock, 0, 0, error, 0);
b0d623f7 2289#endif /* _PSYNCH_TRACE_ */
6d2010ae 2290 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 2291
2292 return(error);
2293
2294prepost:
2295 kwq->kw_pre_rwwc = (rw_wc - count);
2296 kwq->kw_pre_lockseq = lgen;
2297#if _PSYNCH_TRACE_
6d2010ae 2298 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 2299#endif /* _PSYNCH_TRACE_ */
2300 error = 0;
2301 goto out;
2302}
2303
2304
2305/*
2306 * psynch_rw_upgrade: This system call is used by an reader to block waiting for upgrade to be granted.
2307 */
2308int
2309psynch_rw_upgrade(__unused proc_t p, struct psynch_rw_upgrade_args * uap, uint32_t * retval)
2310{
2311 user_addr_t rwlock = uap->rwlock;
2312 uint32_t lgen = uap->lgenval;
2313 uint32_t ugen = uap->ugenval;
2314 uint32_t rw_wc = uap->rw_wc;
2315 //uint64_t tid = uap->tid;
2316 int flags = uap->flags;
2317 int block;
2318 ksyn_wait_queue_t kwq;
2319 int error=0;
2320 uthread_t uth;
2321 uint32_t lockseq = 0, updatebits = 0, preseq = 0;
6d2010ae
A		 2322 int isinit = lgen & PTHRW_RWL_INIT;
2323 ksyn_waitq_element_t kwe;
7ddcb079 2324 kern_return_t kret;
b0d623f7
A		 2325
2326#if _PSYNCH_TRACE_
6d2010ae 2327 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 2328#endif /* _PSYNCH_TRACE_ */
2329 uth = current_uthread();
6d2010ae
A		 2330 kwe = &uth->uu_kwe;
2331 kwe->kwe_lockseq = lgen;
2332 kwe->kwe_uth = uth;
2333 kwe->kwe_psynchretval = 0;
2334 kwe->kwe_kwqqueue = NULL;
b0d623f7 2335 lockseq = (lgen & PTHRW_COUNT_MASK);
6d2010ae
A		 2336
2337 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK), &kwq);
b0d623f7
A		 2338 if (error != 0) {
2339#if _PSYNCH_TRACE_
6d2010ae 2340 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2341#endif /* _PSYNCH_TRACE_ */
2342 return(error);
2343 }
2344
2345 ksyn_wqlock(kwq);
6d2010ae
A		 2346
2347 if (isinit != 0) {
2348 lgen &= ~PTHRW_RWL_INIT;
2349 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
2350 /* first to notice the reset of the lock, clear preposts */
2351 CLEAR_REINIT_BITS(kwq);
2352 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2353#if _PSYNCH_TRACE_
2354 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2355#endif /* _PSYNCH_TRACE_ */
2356 }
2357 }
2358
b0d623f7
A		 2359 /* handle first the missed wakeups */
2360 if ((kwq->kw_pre_intrcount != 0) &&
6d2010ae 2361 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
b0d623f7
A		 2362 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
2363
2364 kwq->kw_pre_intrcount--;
6d2010ae 2365 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 2366 if (kwq->kw_pre_intrcount==0)
2367 CLEAR_INTR_PREPOST_BITS(kwq);
2368 ksyn_wqunlock(kwq);
2369 goto out;
2370 }
2371
2372 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
2373#if _PSYNCH_TRACE_
6d2010ae 2374 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 2375#endif /* _PSYNCH_TRACE_ */
2376 kwq->kw_pre_rwwc--;
2377 if (kwq->kw_pre_rwwc == 0) {
2378 preseq = kwq->kw_pre_lockseq;
6d2010ae 2379 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 2380 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 2381 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
2382 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2383#if _PSYNCH_TRACE_
2384 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2385#endif /* _PSYNCH_TRACE_ */
2386 }
2387 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_UPGRADE|KW_UNLOCK_PREPOST), &block, lgen);
2388#if __TESTPANICS__
b0d623f7 2389 if (error != 0)
6d2010ae
A		 2390 panic("rw_rdlock: kwq_handle_unlock failed %d\n",error);
2391#endif /* __TESTPANICS__ */
b0d623f7
A		 2392 if (block == 0) {
2393 ksyn_wqunlock(kwq);
2394 goto out;
2395 }
2396 /* insert to q and proceed as ususal */
2397 }
2398 }
2399
2400
2401#if _PSYNCH_TRACE_
6d2010ae 2402 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 2403#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2404 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], lgen, uth, kwe, SEQFIT);
2405#if __TESTPANICS__
b0d623f7
A		 2406 if (error != 0)
2407 panic("psynch_rw_upgrade: failed to enqueue\n");
6d2010ae 2408#endif /* __TESTPANICS__ */
b0d623f7
A		 2409
2410
7ddcb079 2411 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 2412 /* drops the lock */
7ddcb079
A		 2413 switch (kret) {
2414 case THREAD_TIMED_OUT:
2415 error = ETIMEDOUT;
2416 break;
2417 case THREAD_INTERRUPTED:
2418 error = EINTR;
2419 break;
2420 default:
2421 error = 0;
2422 break;
2423 }
b0d623f7
A		 2424
2425out:
2426 if (error != 0) {
2427#if _PSYNCH_TRACE_
6d2010ae 2428 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 2429#endif /* _PSYNCH_TRACE_ */
2430 ksyn_wqlock(kwq);
6d2010ae
A		 2431 if (kwe->kwe_kwqqueue != NULL)
2432 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwe);
b0d623f7
A		 2433 ksyn_wqunlock(kwq);
2434 } else {
2435 /* update bits */
6d2010ae 2436 *retval = kwe->kwe_psynchretval;
b0d623f7
A		 2437 }
2438
6d2010ae 2439 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
b0d623f7 2440#if _PSYNCH_TRACE_
6d2010ae 2441 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7 2442#endif /* _PSYNCH_TRACE_ */
6d2010ae 2443
b0d623f7
A		 2444 return(error);
2445}
2446
6d2010ae
A		 2447#else /* NOTYET */
2448int
2449psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t * retval)
2450{
2451 return(0);
2452}
2453int
2454psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int * retval)
2455{
2456 return(0);
2457}
2458#endif /* NOTYET */
b0d623f7
A		 2459/*
2460 * psynch_rw_unlock: This system call is used for unlock state postings. This will grant appropriate
2461 * reader/writer variety lock.
2462 */
2463
2464int
2465psynch_rw_unlock(__unused proc_t p, struct psynch_rw_unlock_args * uap, uint32_t * retval)
2466{
2467 user_addr_t rwlock = uap->rwlock;
2468 uint32_t lgen = uap->lgenval;
2469 uint32_t ugen = uap->ugenval;
2470 uint32_t rw_wc = uap->rw_wc;
2471 uint32_t curgen;
2472 //uint64_t tid = uap->tid;
2473 int flags = uap->flags;
2474 uthread_t uth;
2475 ksyn_wait_queue_t kwq;
2476 uint32_t updatebits = 0;
6d2010ae 2477 int error=0, diff;
b0d623f7 2478 uint32_t count = 0;
6d2010ae 2479 int isinit = 0;
b0d623f7
A		 2480
2481
2482#if _PSYNCH_TRACE_
6d2010ae 2483 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 2484#endif /* _PSYNCH_TRACE_ */
2485 uth = current_uthread();
2486
6d2010ae 2487 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
b0d623f7
A		 2488 if (error != 0) {
2489#if _PSYNCH_TRACE_
6d2010ae 2490 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2491#endif /* _PSYNCH_TRACE_ */
2492 return(error);
2493 }
2494
2495 curgen = lgen & PTHRW_COUNT_MASK;
2496
2497 ksyn_wqlock(kwq);
2498
6d2010ae
A		 2499 if ((lgen & PTHRW_RWL_INIT) != 0) {
2500 lgen &= ~PTHRW_RWL_INIT;
2501 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
2502 CLEAR_REINIT_BITS(kwq);
2503 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2504#if _PSYNCH_TRACE_
2505 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2506#endif /* _PSYNCH_TRACE_ */
2507 }
2508 isinit = 1;
2509 }
2510
2511 /* if lastunlock seq is set, ensure the current one is not lower than that, as it would be spurious */
2512 if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
2513#if _PSYNCH_TRACE_
2514 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, (uint32_t)0xeeeeeeee, rw_wc, kwq->kw_lastunlockseq, 0);
2515#endif /* _PSYNCH_TRACE_ */
2516 error = 0;
b0d623f7
A		 2517 goto out;
2518 }
2519
6d2010ae
A		 2520 /* If L-U != num of waiters, then it needs to be preposted or spr */
2521 diff = find_diff(lgen, ugen);
b0d623f7
A		 2522
2523#if _PSYNCH_TRACE_
6d2010ae 2524 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
b0d623f7 2525#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2526 if (find_seq_till(kwq, curgen, diff, &count) == 0) {
2527 if ((count == 0) || (count < (uint32_t)diff))
b0d623f7
A		 2528 goto prepost;
2529 }
2530
6d2010ae
A		 2531 /* no prepost and all threads are in place, reset the bit */
2532 if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
2533 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2534#if _PSYNCH_TRACE_
2535 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2536#endif /* _PSYNCH_TRACE_ */
2537 }
b0d623f7
A		 2538
2539 /* can handle unlock now */
2540
2541 CLEAR_PREPOST_BITS(kwq);
b0d623f7
A		 2542
2543#if _PSYNCH_TRACE_
6d2010ae 2544 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, 0, 0, 0);
b0d623f7 2545#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2546 error = kwq_handle_unlock(kwq, lgen, rw_wc, &updatebits, 0, NULL, 0);
2547#if __TESTPANICS__
b0d623f7
A		 2548 if (error != 0)
2549 panic("psynch_rw_unlock: kwq_handle_unlock failed %d\n",error);
6d2010ae 2550#endif /* __TESTPANICS__ */
b0d623f7
A		 2551out:
2552 if (error == 0) {
2553 /* update bits?? */
2554 *retval = updatebits;
2555 }
6d2010ae
A		 2556
2557
b0d623f7
A		 2558 ksyn_wqunlock(kwq);
2559
6d2010ae 2560 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
b0d623f7 2561#if _PSYNCH_TRACE_
6d2010ae 2562 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, updatebits, error, 0);
b0d623f7
A		 2563#endif /* _PSYNCH_TRACE_ */
2564
2565 return(error);
2566
2567prepost:
6d2010ae
A		 2568 /* update if the new seq is higher than prev prepost, or first set */
2569 if ((is_rws_setseq(kwq->kw_pre_sseq) != 0) ||
2570 (is_seqhigher_eq((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_pre_sseq & PTHRW_COUNT_MASK)) != 0)) {
2571 kwq->kw_pre_rwwc = (diff - count);
2572 kwq->kw_pre_lockseq = curgen;
2573 kwq->kw_pre_sseq = rw_wc;
b0d623f7 2574#if _PSYNCH_TRACE_
6d2010ae
A		 2575 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, rw_wc, count, 0);
2576 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7 2577#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2578 updatebits = lgen; /* let this not do unlock handling */
2579 }
b0d623f7
A		 2580 error = 0;
2581 goto out;
2582}
2583
2584
2585/*
2586 * psynch_rw_unlock2: This system call is used to wakeup pending readers when unlock grant frm kernel
2587 * to new reader arrival races
2588 */
2589int
6d2010ae 2590psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args * uap, __unused uint32_t * retval)
b0d623f7 2591{
6d2010ae 2592 return(ENOTSUP);
b0d623f7
A		 2593}
2594
2595
2596/* ************************************************************************** */
2597void
2598pth_global_hashinit()
2599{
6d2010ae
A		 2600 int arg;
2601
b0d623f7 2602 pth_glob_hashtbl = hashinit(PTH_HASHSIZE * 4, M_PROC, &pthhash);
6d2010ae
A		 2603
2604 /*
2605 * pthtest={0,1,2,3} (override default aborting behavior on pthread sync failures)
2606 * 0 - just return errors
2607 * 1 - print and return errors
2608 * 2 - abort user, print and return errors
2609 * 3 - panic
2610 */
2611 if (!PE_parse_boot_argn("pthtest", &arg, sizeof(arg)))
2612 arg = __TESTMODE__;
2613
2614 if (arg == 3) {
2615 __test_panics__ = 1;
2616 printf("Pthread support PANICS when sync kernel primitives misused\n");
2617 } else if (arg == 2) {
2618 __test_aborts__ = 1;
2619 __test_prints__ = 1;
2620 printf("Pthread support ABORTS when sync kernel primitives misused\n");
2621 } else if (arg == 1) {
2622 __test_prints__ = 1;
2623 printf("Pthread support LOGS when sync kernel primitives misused\n");
2624 }
b0d623f7
A		 2625}
2626
2627void
2628pth_proc_hashinit(proc_t p)
2629{
2630 p->p_pthhash = hashinit(PTH_HASHSIZE, M_PROC, &pthhash);
2631 if (p->p_pthhash == NULL)
2632 panic("pth_proc_hashinit: hash init returned 0\n");
2633}
2634
2635
2636ksyn_wait_queue_t
2637ksyn_wq_hash_lookup(user_addr_t mutex, proc_t p, int flags, uint64_t object, uint64_t objoffset)
2638{
2639 ksyn_wait_queue_t kwq;
2640 struct pthhashhead * hashptr;
2641
2642 if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
2643 {
2644 hashptr = pth_glob_hashtbl;
2645 kwq = (&hashptr[object & pthhash])->lh_first;
2646 if (kwq != 0) {
2647 for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
2648 if ((kwq->kw_object == object) &&(kwq->kw_offset == objoffset)) {
2649 return (kwq);
2650 }
2651 }
2652 }
2653 } else {
2654 hashptr = p->p_pthhash;
2655 kwq = (&hashptr[mutex & pthhash])->lh_first;
2656 if (kwq != 0)
2657 for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
2658 if (kwq->kw_addr == mutex) {
2659 return (kwq);
2660 }
2661 }
2662 }
2663 return(NULL);
2664}
2665
2666void
2667pth_proc_hashdelete(proc_t p)
2668{
2669 struct pthhashhead * hashptr;
2670 ksyn_wait_queue_t kwq;
2671 int hashsize = pthhash + 1;
2672 int i;
2673
6d2010ae
A		 2674#if _PSYNCH_TRACE_
2675 if ((pthread_debug_proc != NULL) && (p == pthread_debug_proc))
2676 pthread_debug_proc = PROC_NULL;
2677#endif /* _PSYNCH_TRACE_ */
b0d623f7 2678 hashptr = p->p_pthhash;
316670eb 2679 p->p_pthhash = NULL;
b0d623f7
A		 2680 if (hashptr == NULL)
2681 return;
2682
316670eb 2683 pthread_list_lock();
b0d623f7
A		 2684 for(i= 0; i < hashsize; i++) {
2685 while ((kwq = LIST_FIRST(&hashptr[i])) != NULL) {
b0d623f7
A		 2686 if ((kwq->kw_pflags & KSYN_WQ_INHASH) != 0) {
2687 kwq->kw_pflags &= ~KSYN_WQ_INHASH;
2688 LIST_REMOVE(kwq, kw_hash);
2689 }
2690 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2691 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
2692 LIST_REMOVE(kwq, kw_list);
6d2010ae 2693 num_infreekwq--;
b0d623f7 2694 }
6d2010ae 2695 num_freekwq++;
b0d623f7 2696 pthread_list_unlock();
6d2010ae
A		 2697 /* release fake entries if present for cvars */
2698 if (((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) && (kwq->kw_inqueue != 0))
2699 ksyn_freeallkwe(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER]);
b0d623f7 2700 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
6d2010ae 2701 zfree(kwq_zone, kwq);
316670eb 2702 pthread_list_lock();
b0d623f7
A		 2703 }
2704 }
316670eb
A		 2705 pthread_list_unlock();
2706 FREE(hashptr, M_PROC);
b0d623f7
A		 2707}
2708
6d2010ae
A		 2709/* no lock held for this as the waitqueue is getting freed */
2710void
2711ksyn_freeallkwe(ksyn_queue_t kq)
2712{
2713 ksyn_waitq_element_t kwe;
2714
2715 /* free all the fake entries, dequeue rest */
2716 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
2717 while (kwe != NULL) {
2718 if (kwe->kwe_flags != KWE_THREAD_INWAIT) {
2719 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
2720 zfree(kwe_zone, kwe);
2721 } else {
2722 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
2723 }
2724 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
2725 }
2726}
b0d623f7
A		 2727
2728/* find kernel waitqueue, if not present create one. Grants a reference */
2729int
2730ksyn_wqfind(user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int flags, int wqtype, ksyn_wait_queue_t * kwqp)
2731{
2732 ksyn_wait_queue_t kwq;
2733 ksyn_wait_queue_t nkwq;
2734 struct pthhashhead * hashptr;
2735 uint64_t object = 0, offset = 0;
2736 uint64_t hashhint;
2737 proc_t p = current_proc();
6d2010ae
A		 2738 int retry = mgen & PTH_RWL_RETRYBIT;
2739 struct ksyn_queue kfreeq;
b0d623f7
A		 2740 int i;
2741
2742 if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
2743 {
2744 (void)ksyn_findobj(mutex, &object, &offset);
2745 hashhint = object;
2746 hashptr = pth_glob_hashtbl;
2747 } else {
2748 hashptr = p->p_pthhash;
2749 }
2750
6d2010ae
A		 2751 ksyn_queue_init(&kfreeq);
2752
2753 if (((wqtype & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_MTX) && (retry != 0))
2754 mgen &= ~PTH_RWL_RETRYBIT;
2755
2756loop:
b0d623f7
A		 2757 //pthread_list_lock_spin();
2758 pthread_list_lock();
2759
2760 kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
2761
2762 if (kwq != NULL) {
b0d623f7
A		 2763 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2764 LIST_REMOVE(kwq, kw_list);
2765 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
6d2010ae
A		 2766 num_infreekwq--;
2767 num_reusekwq++;
2768 }
2769 if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
2770 if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
2771 if (kwq->kw_iocount == 0) {
2772 kwq->kw_addr = mutex;
2773 kwq->kw_flags = flags;
2774 kwq->kw_object = object;
2775 kwq->kw_offset = offset;
2776 kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
2777 CLEAR_REINIT_BITS(kwq);
2778 CLEAR_INTR_PREPOST_BITS(kwq);
2779 CLEAR_PREPOST_BITS(kwq);
2780 kwq->kw_lword = mgen;
2781 kwq->kw_uword = ugen;
2782 kwq->kw_sword = rw_wc;
2783 kwq->kw_owner = tid;
2784 } else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
2785 /* if all users are unlockers then wait for it to finish */
2786 kwq->kw_pflags |= KSYN_WQ_WAITING;
2787 /* wait for the wq to be free */
2788 (void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
2789 /* does not have list lock */
2790 goto loop;
2791 } else {
2792 __FAILEDUSERTEST__("address already known to kernel for another (busy) synchronizer type\n");
2793 pthread_list_unlock();
2794 return EBUSY;
2795 }
2796 } else {
2797 __FAILEDUSERTEST__("address already known to kernel for another (busy) synchronizer type(1)\n");
2798 pthread_list_unlock();
2799 return EBUSY;
2800 }
b0d623f7 2801 }
6d2010ae
A		 2802 kwq->kw_iocount++;
2803 if (wqtype == KSYN_WQTYPE_MUTEXDROP)
2804 kwq->kw_dropcount++;
b0d623f7
A		 2805 if (kwqp != NULL)
2806 *kwqp = kwq;
2807 pthread_list_unlock();
2808 return (0);
2809 }
2810
2811 pthread_list_unlock();
2812
6d2010ae 2813 nkwq = (ksyn_wait_queue_t)zalloc(kwq_zone);
b0d623f7
A		 2814 bzero(nkwq, sizeof(struct ksyn_wait_queue));
2815 nkwq->kw_addr = mutex;
2816 nkwq->kw_flags = flags;
2817 nkwq->kw_iocount = 1;
6d2010ae
A		 2818 if (wqtype == KSYN_WQTYPE_MUTEXDROP)
2819 nkwq->kw_dropcount++;
b0d623f7
A		 2820 nkwq->kw_object = object;
2821 nkwq->kw_offset = offset;
2822 nkwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
6d2010ae
A		 2823 nkwq->kw_lastseqword = PTHRW_RWS_INIT;
2824 if (nkwq->kw_type == KSYN_WQTYPE_RWLOCK)
2825 nkwq->kw_nextseqword = PTHRW_RWS_INIT;
2826
2827 nkwq->kw_pre_sseq = PTHRW_RWS_INIT;
2828
2829 CLEAR_PREPOST_BITS(nkwq);
2830 CLEAR_INTR_PREPOST_BITS(nkwq);
2831 CLEAR_REINIT_BITS(nkwq);
2832 nkwq->kw_lword = mgen;
2833 nkwq->kw_uword = ugen;
2834 nkwq->kw_sword = rw_wc;
2835 nkwq->kw_owner = tid;
2836
b0d623f7
A		 2837
2838 for (i=0; i< KSYN_QUEUE_MAX; i++)
2839 ksyn_queue_init(&nkwq->kw_ksynqueues[i]);
2840
b0d623f7
A		 2841 lck_mtx_init(&nkwq->kw_lock, pthread_lck_grp, pthread_lck_attr);
2842
2843 //pthread_list_lock_spin();
2844 pthread_list_lock();
2845 /* see whether it is alread allocated */
2846 kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
2847
2848 if (kwq != NULL) {
b0d623f7
A		 2849 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2850 LIST_REMOVE(kwq, kw_list);
2851 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
6d2010ae
A		 2852 num_infreekwq--;
2853 num_reusekwq++;
2854 }
2855 if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
2856 if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
2857 if (kwq->kw_iocount == 0) {
2858 kwq->kw_addr = mutex;
2859 kwq->kw_flags = flags;
2860 kwq->kw_object = object;
2861 kwq->kw_offset = offset;
2862 kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
2863 CLEAR_REINIT_BITS(kwq);
2864 CLEAR_INTR_PREPOST_BITS(kwq);
2865 CLEAR_PREPOST_BITS(kwq);
2866 kwq->kw_lword = mgen;
2867 kwq->kw_uword = ugen;
2868 kwq->kw_sword = rw_wc;
2869 kwq->kw_owner = tid;
2870 } else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
2871 kwq->kw_pflags |= KSYN_WQ_WAITING;
2872 /* wait for the wq to be free */
2873 (void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
2874
2875 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
2876 zfree(kwq_zone, nkwq);
2877 /* will acquire lock again */
2878
2879 goto loop;
2880 } else {
2881 __FAILEDUSERTEST__("address already known to kernel for another [busy] synchronizer type(2)\n");
2882 pthread_list_unlock();
2883 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
2884 zfree(kwq_zone, nkwq);
2885 return EBUSY;
2886 }
2887 } else {
2888 __FAILEDUSERTEST__("address already known to kernel for another [busy] synchronizer type(3)\n");
2889 pthread_list_unlock();
2890 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
2891 zfree(kwq_zone, nkwq);
2892 return EBUSY;
2893 }
b0d623f7 2894 }
6d2010ae
A		 2895 kwq->kw_iocount++;
2896 if (wqtype == KSYN_WQTYPE_MUTEXDROP)
2897 kwq->kw_dropcount++;
b0d623f7
A		 2898 if (kwqp != NULL)
2899 *kwqp = kwq;
2900 pthread_list_unlock();
2901 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
6d2010ae 2902 zfree(kwq_zone, nkwq);
b0d623f7
A		 2903 return (0);
2904 }
2905 kwq = nkwq;
2906
6d2010ae
A		 2907#if _PSYNCH_TRACE_
2908 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, kwq->kw_lword, kwq->kw_uword, kwq->kw_sword, 0xffff, 0);
2909#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 2910 if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
2911 {
2912 kwq->kw_pflags |= KSYN_WQ_SHARED;
2913 LIST_INSERT_HEAD(&hashptr[kwq->kw_object & pthhash], kwq, kw_hash);
2914 } else
2915 LIST_INSERT_HEAD(&hashptr[mutex & pthhash], kwq, kw_hash);
2916
2917 kwq->kw_pflags |= KSYN_WQ_INHASH;
6d2010ae 2918 num_total_kwq++;
b0d623f7
A		 2919
2920 pthread_list_unlock();
2921
2922 if (kwqp != NULL)
2923 *kwqp = kwq;
2924 return (0);
2925}
2926
2927/* Reference from find is dropped here. Starts the free process if needed */
2928void
6d2010ae 2929ksyn_wqrelease(ksyn_wait_queue_t kwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype)
b0d623f7
A		 2930{
2931 uint64_t deadline;
2932 struct timeval t;
2933 int sched = 0;
6d2010ae
A		 2934 ksyn_wait_queue_t free_elem = NULL;
2935 ksyn_wait_queue_t free_elem1 = NULL;
b0d623f7
A		 2936
2937 //pthread_list_lock_spin();
2938 pthread_list_lock();
2939 kwq->kw_iocount--;
6d2010ae
A		 2940 if (wqtype == KSYN_WQTYPE_MUTEXDROP) {
2941 kwq->kw_dropcount--;
2942 }
b0d623f7 2943 if (kwq->kw_iocount == 0) {
6d2010ae
A		 2944 if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
2945 /* some one is waiting for the waitqueue, wake them up */
2946 kwq->kw_pflags &= ~KSYN_WQ_WAITING;
2947 wakeup(&kwq->kw_pflags);
b0d623f7 2948 }
6d2010ae
A		 2949
2950 if ((kwq->kw_pre_rwwc == 0) && (kwq->kw_inqueue == 0) && (kwq->kw_pre_intrcount == 0)) {
2951 if (qfreenow == 0) {
2952 microuptime(&kwq->kw_ts);
2953 LIST_INSERT_HEAD(&pth_free_list, kwq, kw_list);
2954 kwq->kw_pflags |= KSYN_WQ_FLIST;
2955 num_infreekwq++;
2956 free_elem = NULL;
2957 } else {
2958 /* remove from the only list it is in ie hash */
2959 kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
2960 LIST_REMOVE(kwq, kw_hash);
2961 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
2962 num_total_kwq--;
2963 num_freekwq++;
2964 free_elem = kwq;
2965 }
2966 } else
2967 free_elem = NULL;
2968 if (qfreenow == 0)
2969 sched = 1;
b0d623f7 2970 }
6d2010ae
A		 2971
2972 if (ckwq != NULL) {
b0d623f7 2973 ckwq->kw_iocount--;
6d2010ae
A		 2974 if (wqtype == KSYN_WQTYPE_MUTEXDROP) {
2975 kwq->kw_dropcount--;
2976 }
b0d623f7 2977 if ( ckwq->kw_iocount == 0) {
6d2010ae
A		 2978 if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
2979 /* some one is waiting for the waitqueue, wake them up */
2980 kwq->kw_pflags &= ~KSYN_WQ_WAITING;
2981 wakeup(&kwq->kw_pflags);
b0d623f7 2982 }
6d2010ae
A		 2983 if ((ckwq->kw_pre_rwwc == 0) && (ckwq->kw_inqueue == 0) && (ckwq->kw_pre_intrcount == 0)) {
2984 if (qfreenow == 0) {
2985 /* mark for free if we can */
2986 microuptime(&ckwq->kw_ts);
2987 LIST_INSERT_HEAD(&pth_free_list, ckwq, kw_list);
2988 ckwq->kw_pflags |= KSYN_WQ_FLIST;
2989 num_infreekwq++;
2990 free_elem1 = NULL;
2991 } else {
2992 /* remove from the only list it is in ie hash */
2993 ckwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
2994 LIST_REMOVE(ckwq, kw_hash);
2995 lck_mtx_destroy(&ckwq->kw_lock, pthread_lck_grp);
2996 num_total_kwq--;
2997 num_freekwq++;
2998 free_elem1 = ckwq;
2999 }
3000 } else
3001 free_elem1 = NULL;
3002 if (qfreenow == 0)
3003 sched = 1;
b0d623f7
A		 3004 }
3005 }
3006
3007 if (sched == 1 && psynch_cleanupset == 0) {
3008 psynch_cleanupset = 1;
3009 microuptime(&t);
3010 t.tv_sec += KSYN_CLEANUP_DEADLINE;
3011
3012 deadline = tvtoabstime(&t);
3013 thread_call_enter_delayed(psynch_thcall, deadline);
3014 }
3015 pthread_list_unlock();
6d2010ae
A		 3016 if (free_elem != NULL)
3017 zfree(kwq_zone, free_elem);
3018 if (free_elem1 != NULL)
3019 zfree(kwq_zone, free_elem1);
b0d623f7
A		 3020}
3021
3022/* responsible to free the waitqueues */
3023void
3024psynch_wq_cleanup(__unused void * param, __unused void * param1)
3025{
3026 ksyn_wait_queue_t kwq;
3027 struct timeval t;
3028 LIST_HEAD(, ksyn_wait_queue) freelist = {NULL};
3029 int count = 0, delayed = 0, diff;
3030 uint64_t deadline = 0;
3031
3032 //pthread_list_lock_spin();
3033 pthread_list_lock();
3034
6d2010ae
A		 3035 num_addedfreekwq = num_infreekwq - num_lastfreekwqcount;
3036 num_lastfreekwqcount = num_infreekwq;
b0d623f7
A		 3037 microuptime(&t);
3038
3039 LIST_FOREACH(kwq, &pth_free_list, kw_list) {
6d2010ae
A		 3040 if ((kwq->kw_iocount != 0) || (kwq->kw_pre_rwwc != 0) || (kwq->kw_inqueue != 0) || (kwq->kw_pre_intrcount != 0)) {
3041 /* still in use */
b0d623f7
A		 3042 continue;
3043 }
3044 diff = t.tv_sec - kwq->kw_ts.tv_sec;
3045 if (diff < 0)
3046 diff *= -1;
3047 if (diff >= KSYN_CLEANUP_DEADLINE) {
3048 /* out of hash */
3049 kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
6d2010ae
A		 3050 num_infreekwq--;
3051 num_freekwq++;
b0d623f7
A		 3052 LIST_REMOVE(kwq, kw_hash);
3053 LIST_REMOVE(kwq, kw_list);
3054 LIST_INSERT_HEAD(&freelist, kwq, kw_list);
3055 count ++;
6d2010ae 3056 num_total_kwq--;
b0d623f7
A		 3057 } else {
3058 delayed = 1;
3059 }
3060
3061 }
3062 if (delayed != 0) {
3063 t.tv_sec += KSYN_CLEANUP_DEADLINE;
3064
3065 deadline = tvtoabstime(&t);
3066 thread_call_enter_delayed(psynch_thcall, deadline);
3067 psynch_cleanupset = 1;
3068 } else
3069 psynch_cleanupset = 0;
3070
3071 pthread_list_unlock();
3072
3073
3074 while ((kwq = LIST_FIRST(&freelist)) != NULL) {
3075 LIST_REMOVE(kwq, kw_list);
3076 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
6d2010ae 3077 zfree(kwq_zone, kwq);
b0d623f7
A		 3078 }
3079}
3080
3081
7ddcb079
A		 3082kern_return_t
3083#if _PSYNCH_TRACE_
3084ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int mylog, thread_continue_t continuation, void * parameter)
3085#else
3086ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, __unused int mylog, thread_continue_t continuation, void * parameter)
3087#endif
b0d623f7
A		 3088{
3089 kern_return_t kret;
6d2010ae 3090#if _PSYNCH_TRACE_
316670eb 3091 int error = 0;
6d2010ae
A		 3092 uthread_t uth = NULL;
3093#endif /* _PSYNCH_TRACE_ */
b0d623f7 3094
6d2010ae
A		 3095 kwe->kwe_kwqqueue = (void *)kwq;
3096 assert_wait_deadline(&kwe->kwe_psynchretval, THREAD_ABORTSAFE, abstime);
b0d623f7
A		 3097 ksyn_wqunlock(kwq);
3098
7ddcb079
A		 3099 if (continuation == THREAD_CONTINUE_NULL)
3100 kret = thread_block(NULL);
3101 else
3102 kret = thread_block_parameter(continuation, parameter);
3103
3104#if _PSYNCH_TRACE_
b0d623f7
A		 3105 switch (kret) {
3106 case THREAD_TIMED_OUT:
3107 error = ETIMEDOUT;
3108 break;
3109 case THREAD_INTERRUPTED:
3110 error = EINTR;
3111 break;
3112 }
6d2010ae
A		 3113 uth = current_uthread();
3114#if defined(__i386__)
3115 if (mylog != 0)
3116 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf4f3f2f1, (uint32_t)uth, kret, 0, 0);
3117#else
3118 if (mylog != 0)
3119 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xeeeeeeee, kret, error, 0xeeeeeeee, 0);
3120#endif
3121#endif /* _PSYNCH_TRACE_ */
3122
7ddcb079 3123 return(kret);
b0d623f7
A		 3124}
3125
3126kern_return_t
6d2010ae 3127ksyn_wakeup_thread(__unused ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe)
b0d623f7 3128{
b0d623f7 3129 kern_return_t kret;
6d2010ae
A		 3130#if _PSYNCH_TRACE_
3131 uthread_t uth = NULL;
3132#endif /* _PSYNCH_TRACE_ */
b0d623f7 3133
6d2010ae 3134 kret = thread_wakeup_one((caddr_t)&kwe->kwe_psynchretval);
b0d623f7
A		 3135
3136 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3137 panic("ksyn_wakeup_thread: panic waking up thread %x\n", kret);
6d2010ae
A		 3138#if _PSYNCH_TRACE_
3139 uth = kwe->kwe_uth;
3140#if defined(__i386__)
3141 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf1f2f3f4, (uint32_t)uth, kret, 0, 0);
3142#endif
3143#endif /* _PSYNCH_TRACE_ */
b0d623f7 3144
b0d623f7
A		 3145 return(kret);
3146}
3147
b0d623f7
A		 3148/* find the true shared obect/offset for shared mutexes */
3149int
3150ksyn_findobj(uint64_t mutex, uint64_t * objectp, uint64_t * offsetp)
3151{
3152 vm_page_info_basic_data_t info;
3153 kern_return_t kret;
3154 mach_msg_type_number_t count = VM_PAGE_INFO_BASIC_COUNT;
3155
3156 kret = vm_map_page_info(current_map(), mutex, VM_PAGE_INFO_BASIC,
3157 (vm_page_info_t)&info, &count);
3158
3159 if (kret != KERN_SUCCESS)
3160 return(EINVAL);
3161
3162 if (objectp != NULL)
3163 *objectp = (uint64_t)info.object_id;
3164 if (offsetp != NULL)
3165 *offsetp = (uint64_t)info.offset;
3166
3167 return(0);
3168}
3169
3170
3171/* lowest of kw_fr, kw_flr, kw_fwr, kw_fywr */
3172int
3173kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * typep, uint32_t lowest[])
3174{
3175
3176 uint32_t kw_fr, kw_flr, kw_fwr, kw_fywr, low;
3177 int type = 0, lowtype, typenum[4];
3178 uint32_t numbers[4];
3179 int count = 0, i;
3180
3181
3182 if ((kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
3183 type |= PTH_RWSHFT_TYPE_READ;
3184 /* read entries are present */
3185 if (kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) {
3186 kw_fr = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_firstnum;
3187 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, kw_fr) != 0))
3188 kw_fr = premgen;
3189 } else
3190 kw_fr = premgen;
3191
3192 lowest[KSYN_QUEUE_READ] = kw_fr;
3193 numbers[count]= kw_fr;
3194 typenum[count] = PTH_RW_TYPE_READ;
3195 count++;
3196 } else
3197 lowest[KSYN_QUEUE_READ] = 0;
3198
3199 if ((kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0)) {
3200 type |= PTH_RWSHFT_TYPE_LREAD;
3201 /* read entries are present */
3202 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) {
3203 kw_flr = kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum;
3204 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) && (is_seqlower(premgen, kw_flr) != 0))
3205 kw_flr = premgen;
3206 } else
3207 kw_flr = premgen;
3208
3209 lowest[KSYN_QUEUE_LREAD] = kw_flr;
3210 numbers[count]= kw_flr;
3211 typenum[count] = PTH_RW_TYPE_LREAD;
3212 count++;
3213 } else
3214 lowest[KSYN_QUEUE_LREAD] = 0;
3215
3216
3217 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0)) {
3218 type |= PTH_RWSHFT_TYPE_WRITE;
3219 /* read entries are present */
3220 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) {
3221 kw_fwr = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
3222 if (((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) && (is_seqlower(premgen, kw_fwr) != 0))
3223 kw_fwr = premgen;
3224 } else
3225 kw_fwr = premgen;
3226
3227 lowest[KSYN_QUEUE_WRITER] = kw_fwr;
3228 numbers[count]= kw_fwr;
3229 typenum[count] = PTH_RW_TYPE_WRITE;
3230 count++;
3231 } else
3232 lowest[KSYN_QUEUE_WRITER] = 0;
3233
3234 if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0)) {
3235 type |= PTH_RWSHFT_TYPE_YWRITE;
3236 /* read entries are present */
3237 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) {
3238 kw_fywr = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
3239 if (((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) && (is_seqlower(premgen, kw_fywr) != 0))
3240 kw_fywr = premgen;
3241 } else
3242 kw_fywr = premgen;
3243
3244 lowest[KSYN_QUEUE_YWRITER] = kw_fywr;
3245 numbers[count]= kw_fywr;
3246 typenum[count] = PTH_RW_TYPE_YWRITE;
3247 count++;
3248 } else
3249 lowest[KSYN_QUEUE_YWRITER] = 0;
3250
b0d623f7 3251
6d2010ae 3252#if __TESTPANICS__
b0d623f7
A		 3253 if (count == 0)
3254 panic("nothing in the queue???\n");
6d2010ae 3255#endif /* __TESTPANICS__ */
b0d623f7 3256
6d2010ae 3257 low = numbers[0];
b0d623f7
A		 3258 lowtype = typenum[0];
3259 if (count > 1) {
3260 for (i = 1; i< count; i++) {
3261 if(is_seqlower(numbers[i] , low) != 0) {
3262 low = numbers[i];
3263 lowtype = typenum[i];
3264 }
3265 }
3266 }
3267 type |= lowtype;
3268
3269 if (typep != 0)
3270 *typep = type;
3271 return(0);
3272}
3273
3274/* wakeup readers and longreaders to upto the writer limits */
3275int
3276ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp)
3277{
6d2010ae 3278 ksyn_waitq_element_t kwe = NULL;
b0d623f7
A		 3279 ksyn_queue_t kq;
3280 int failedwakeup = 0;
3281 int numwoken = 0;
3282 kern_return_t kret = KERN_SUCCESS;
b0d623f7
A		 3283 uint32_t lbits = 0;
3284
3285 lbits = updatebits;
3286 if (longreadset != 0) {
3287 /* clear all read and longreads */
6d2010ae
A		 3288 while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwq)) != NULL) {
3289 kwe->kwe_psynchretval = lbits;
3290 kwe->kwe_kwqqueue = NULL;
3291
b0d623f7 3292 numwoken++;
6d2010ae
A		 3293 kret = ksyn_wakeup_thread(kwq, kwe);
3294#if __TESTPANICS__
b0d623f7
A		 3295 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3296 panic("ksyn_wakeupreaders: panic waking up readers\n");
6d2010ae 3297#endif /* __TESTPANICS__ */
b0d623f7
A		 3298 if (kret == KERN_NOT_WAITING) {
3299 failedwakeup++;
3300 }
3301 }
6d2010ae
A		 3302 while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwq)) != NULL) {
3303 kwe->kwe_psynchretval = lbits;
3304 kwe->kwe_kwqqueue = NULL;
b0d623f7 3305 numwoken++;
6d2010ae
A		 3306 kret = ksyn_wakeup_thread(kwq, kwe);
3307#if __TESTPANICS__
b0d623f7
A		 3308 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3309 panic("ksyn_wakeupreaders: panic waking up lreaders\n");
6d2010ae 3310#endif /* __TESTPANICS__ */
b0d623f7
A		 3311 if (kret == KERN_NOT_WAITING) {
3312 failedwakeup++;
3313 }
3314 }
3315 } else {
3316 kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
3317 while ((kq->ksynq_count != 0) && (allreaders || (is_seqlower(kq->ksynq_firstnum, limitread) != 0))) {
6d2010ae
A		 3318 kwe = ksyn_queue_removefirst(kq, kwq);
3319 kwe->kwe_psynchretval = lbits;
3320 kwe->kwe_kwqqueue = NULL;
b0d623f7 3321 numwoken++;
6d2010ae
A		 3322 kret = ksyn_wakeup_thread(kwq, kwe);
3323#if __TESTPANICS__
b0d623f7
A		 3324 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3325 panic("ksyn_wakeupreaders: panic waking up readers\n");
6d2010ae 3326#endif /* __TESTPANICS__ */
b0d623f7
A		 3327 if (kret == KERN_NOT_WAITING) {
3328 failedwakeup++;
3329 }
3330 }
3331 }
3332
3333 if (wokenp != NULL)
3334 *wokenp = numwoken;
3335 return(failedwakeup);
3336}
3337
3338
3339/* This handles the unlock grants for next set on rw_unlock() or on arrival of all preposted waiters */
3340int
6d2010ae 3341kwq_handle_unlock(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int * blockp, uint32_t premgen)
b0d623f7
A		 3342{
3343 uint32_t low_reader, low_writer, low_ywriter, low_lreader,limitrdnum;
3344 int rwtype, error=0;
3345 int longreadset = 0, allreaders, failed;
6d2010ae 3346 uint32_t updatebits=0, numneeded = 0;;
b0d623f7
A		 3347 int prepost = flags & KW_UNLOCK_PREPOST;
3348 thread_t preth = THREAD_NULL;
6d2010ae 3349 ksyn_waitq_element_t kwe;
b0d623f7
A		 3350 uthread_t uth;
3351 thread_t th;
3352 int woken = 0;
3353 int block = 1;
6d2010ae 3354 uint32_t lowest[KSYN_QUEUE_MAX]; /* np need for upgrade as it is handled separately */
b0d623f7 3355 kern_return_t kret = KERN_SUCCESS;
6d2010ae
A		 3356 ksyn_queue_t kq;
3357 int curthreturns = 0;
b0d623f7
A		 3358
3359#if _PSYNCH_TRACE_
6d2010ae 3360 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_START, (uint32_t)kwq->kw_addr, mgen, premgen, rw_wc, 0);
b0d623f7
A		 3361#endif /* _PSYNCH_TRACE_ */
3362 if (prepost != 0) {
3363 preth = current_thread();
3364 }
3365
6d2010ae
A		 3366 kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
3367 kwq->kw_lastseqword = rw_wc;
3368 kwq->kw_lastunlockseq = (rw_wc & PTHRW_COUNT_MASK);
3369 kwq->kw_overlapwatch = 0;
3370
b0d623f7
A		 3371 /* upgrade pending */
3372 if (is_rw_ubit_set(mgen)) {
6d2010ae
A		 3373#if __TESTPANICS__
3374 panic("NO UBIT SHOULD BE SET\n");
3375 updatebits = PTH_RWL_EBIT | PTH_RWL_KBIT;
3376 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
3377 updatebits |= PTH_RWL_WBIT;
3378 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
3379 updatebits |= PTH_RWL_YBIT;
b0d623f7
A		 3380 if (prepost != 0) {
3381 if((flags & KW_UNLOCK_PREPOST_UPGRADE) != 0) {
3382 /* upgrade thread calling the prepost */
3383 /* upgrade granted */
3384 block = 0;
3385 goto out;
3386 }
3387
3388 }
3389 if (kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE].ksynq_count > 0) {
6d2010ae
A		 3390 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwq);
3391
3392 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3393 kwe->kwe_psynchretval = updatebits;
3394 kwe->kwe_kwqqueue = NULL;
3395 kret = ksyn_wakeup_thread(kwq, kwe);
b0d623f7
A		 3396 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3397 panic("kwq_handle_unlock: panic waking up the upgrade thread \n");
3398 if (kret == KERN_NOT_WAITING) {
3399 kwq->kw_pre_intrcount = 1; /* actually a count */
3400 kwq->kw_pre_intrseq = mgen;
6d2010ae 3401 kwq->kw_pre_intrretbits = kwe->kwe_psynchretval;
b0d623f7
A		 3402 kwq->kw_pre_intrtype = PTH_RW_TYPE_UPGRADE;
3403 }
3404 error = 0;
3405 } else {
3406 panic("panic unable to find the upgrade thread\n");
3407 }
6d2010ae 3408#endif /* __TESTPANICS__ */
b0d623f7
A		 3409 ksyn_wqunlock(kwq);
3410 goto out;
3411 }
3412
3413 error = kwq_find_rw_lowest(kwq, flags, premgen, &rwtype, lowest);
6d2010ae 3414#if __TESTPANICS__
b0d623f7
A		 3415 if (error != 0)
3416 panic("rwunlock: cannot fails to slot next round of threads");
6d2010ae 3417#endif /* __TESTPANICS__ */
b0d623f7
A		 3418
3419#if _PSYNCH_TRACE_
6d2010ae 3420 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 1, rwtype, 0, 0);
b0d623f7
A		 3421#endif /* _PSYNCH_TRACE_ */
3422 low_reader = lowest[KSYN_QUEUE_READ];
3423 low_lreader = lowest[KSYN_QUEUE_LREAD];
3424 low_writer = lowest[KSYN_QUEUE_WRITER];
3425 low_ywriter = lowest[KSYN_QUEUE_YWRITER];
3426
3427
b0d623f7
A		 3428 longreadset = 0;
3429 allreaders = 0;
6d2010ae
A		 3430 updatebits = 0;
3431
3432
b0d623f7
A		 3433 switch (rwtype & PTH_RW_TYPE_MASK) {
3434 case PTH_RW_TYPE_LREAD:
3435 longreadset = 1;
6d2010ae 3436
b0d623f7 3437 case PTH_RW_TYPE_READ: {
6d2010ae
A		 3438 /* what about the preflight which is LREAD or READ ?? */
3439 if ((rwtype & PTH_RWSHFT_TYPE_MASK) != 0) {
3440 if (rwtype & PTH_RWSHFT_TYPE_WRITE)
3441 updatebits |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
3442 if (rwtype & PTH_RWSHFT_TYPE_YWRITE)
3443 updatebits |= PTH_RWL_YBIT;
3444 }
b0d623f7
A		 3445 limitrdnum = 0;
3446 if (longreadset == 0) {
3447 switch (rwtype & (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE)) {
3448 case PTH_RWSHFT_TYPE_WRITE:
3449 limitrdnum = low_writer;
3450 if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
6d2010ae
A		 3451 (is_seqlower(low_lreader, limitrdnum) != 0)) {
3452 longreadset = 1;
3453 }
3454 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
3455 (is_seqlower(premgen, limitrdnum) != 0)) {
b0d623f7
A		 3456 longreadset = 1;
3457 }
b0d623f7
A		 3458 break;
3459 case PTH_RWSHFT_TYPE_YWRITE:
3460 /* all read ? */
3461 if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
3462 (is_seqlower(low_lreader, low_ywriter) != 0)) {
3463 longreadset = 1;
3464 } else
3465 allreaders = 1;
6d2010ae
A		 3466 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
3467 (is_seqlower(premgen, low_ywriter) != 0)) {
3468 longreadset = 1;
3469 allreaders = 0;
3470 }
3471
3472
b0d623f7
A		 3473 break;
3474 case (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE):
6d2010ae
A		 3475 if (is_seqlower(low_ywriter, low_writer) != 0) {
3476 limitrdnum = low_ywriter;
3477 } else
3478 limitrdnum = low_writer;
b0d623f7 3479 if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
6d2010ae
A		 3480 (is_seqlower(low_lreader, limitrdnum) != 0)) {
3481 longreadset = 1;
3482 }
3483 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
3484 (is_seqlower(premgen, limitrdnum) != 0)) {
b0d623f7
A		 3485 longreadset = 1;
3486 }
3487 break;
3488 default: /* no writers at all */
3489 if ((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0)
3490 longreadset = 1;
3491 else
3492 allreaders = 1;
3493 };
3494
3495 }
6d2010ae
A		 3496 numneeded = 0;
3497 if (longreadset != 0) {
3498 updatebits |= PTH_RWL_LBIT;
3499 updatebits &= ~PTH_RWL_KBIT;
3500 if ((flags & (KW_UNLOCK_PREPOST_READLOCK | KW_UNLOCK_PREPOST_LREADLOCK)) != 0)
3501 numneeded += 1;
3502 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3503 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count;
3504 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
3505 kwq->kw_overlapwatch = 1;
3506 } else {
3507 /* no longread, evaluate number of readers */
b0d623f7 3508
6d2010ae
A		 3509 switch (rwtype & (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE)) {
3510 case PTH_RWSHFT_TYPE_WRITE:
3511 limitrdnum = low_writer;
3512 numneeded = ksyn_queue_count_tolowest(kq, limitrdnum);
3513 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, limitrdnum) != 0)) {
3514 curthreturns = 1;
3515 numneeded += 1;
3516 }
3517 break;
3518 case PTH_RWSHFT_TYPE_YWRITE:
3519 /* all read ? */
3520 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3521 if ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) {
3522 curthreturns = 1;
3523 numneeded += 1;
3524 }
3525 break;
3526 case (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE):
3527 limitrdnum = low_writer;
3528 numneeded = ksyn_queue_count_tolowest(kq, limitrdnum);
3529 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, limitrdnum) != 0)) {
3530 curthreturns = 1;
3531 numneeded += 1;
3532 }
3533 break;
3534 default: /* no writers at all */
3535 /* no other waiters only readers */
3536 kwq->kw_overlapwatch = 1;
3537 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3538 if ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) {
3539 curthreturns = 1;
3540 numneeded += 1;
3541 }
3542 };
3543
3544 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
3545 }
3546 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
b0d623f7 3547
6d2010ae
A		 3548 if (curthreturns != 0) {
3549 block = 0;
3550 uth = current_uthread();
3551 kwe = &uth->uu_kwe;
3552 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3553 }
3554
b0d623f7
A		 3555
3556 failed = ksyn_wakeupreaders(kwq, limitrdnum, longreadset, allreaders, updatebits, &woken);
6d2010ae
A		 3557#if _PSYNCH_TRACE_
3558 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3559#endif /* _PSYNCH_TRACE_ */
3560
b0d623f7
A		 3561 if (failed != 0) {
3562 kwq->kw_pre_intrcount = failed; /* actually a count */
3563 kwq->kw_pre_intrseq = limitrdnum;
3564 kwq->kw_pre_intrretbits = updatebits;
3565 if (longreadset)
3566 kwq->kw_pre_intrtype = PTH_RW_TYPE_LREAD;
3567 else
3568 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3569 }
3570
b0d623f7
A		 3571 error = 0;
3572
6d2010ae
A		 3573 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) && ((updatebits & PTH_RWL_WBIT) == 0))
3574 panic("kwq_handle_unlock: writer pending but no writebit set %x\n", updatebits);
b0d623f7
A		 3575 }
3576 break;
3577
3578 case PTH_RW_TYPE_WRITE: {
6d2010ae
A		 3579
3580 /* only one thread is goin to be granted */
3581 updatebits |= (PTHRW_INC);
3582 updatebits |= PTH_RWL_KBIT| PTH_RWL_EBIT;
3583
b0d623f7
A		 3584 if (((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) && (low_writer == premgen)) {
3585 block = 0;
3586 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
6d2010ae
A		 3587 updatebits |= PTH_RWL_WBIT;
3588 if ((rwtype & PTH_RWSHFT_TYPE_YWRITE) != 0)
3589 updatebits |= PTH_RWL_YBIT;
b0d623f7
A		 3590 th = preth;
3591 uth = get_bsdthread_info(th);
6d2010ae
A		 3592 kwe = &uth->uu_kwe;
3593 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3594 } else {
3595 /* we are not granting writelock to the preposting thread */
6d2010ae 3596 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
b0d623f7
A		 3597
3598 /* if there are writers present or the preposting write thread then W bit is to be set */
3599 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) )
6d2010ae
A		 3600 updatebits |= PTH_RWL_WBIT;
3601 if ((rwtype & PTH_RWSHFT_TYPE_YWRITE) != 0)
3602 updatebits |= PTH_RWL_YBIT;
3603 kwe->kwe_psynchretval = updatebits;
3604 kwe->kwe_kwqqueue = NULL;
b0d623f7 3605 /* setup next in the queue */
6d2010ae
A		 3606 kret = ksyn_wakeup_thread(kwq, kwe);
3607#if _PSYNCH_TRACE_
3608 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
3609#endif /* _PSYNCH_TRACE_ */
3610#if __TESTPANICS__
b0d623f7
A		 3611 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3612 panic("kwq_handle_unlock: panic waking up writer\n");
6d2010ae 3613#endif /* __TESTPANICS__ */
b0d623f7
A		 3614 if (kret == KERN_NOT_WAITING) {
3615 kwq->kw_pre_intrcount = 1; /* actually a count */
3616 kwq->kw_pre_intrseq = low_writer;
3617 kwq->kw_pre_intrretbits = updatebits;
3618 kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
3619 }
3620 error = 0;
3621 }
6d2010ae
A		 3622 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3623 if ((updatebits & (PTH_RWL_KBIT | PTH_RWL_EBIT)) != (PTH_RWL_KBIT | PTH_RWL_EBIT))
3624 panic("kwq_handle_unlock: writer lock granted but no ke set %x\n", updatebits);
b0d623f7
A		 3625
3626 }
3627 break;
3628
3629 case PTH_RW_TYPE_YWRITE: {
3630 /* can reader locks be granted ahead of this write? */
3631 if ((rwtype & PTH_RWSHFT_TYPE_READ) != 0) {
6d2010ae
A		 3632 if ((rwtype & PTH_RWSHFT_TYPE_MASK) != 0) {
3633 if (rwtype & PTH_RWSHFT_TYPE_WRITE)
3634 updatebits |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
3635 if (rwtype & PTH_RWSHFT_TYPE_YWRITE)
3636 updatebits |= PTH_RWL_YBIT;
3637 }
b0d623f7
A		 3638
3639 if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0) {
3640 /* is lowest reader less than the low writer? */
3641 if (is_seqlower(low_reader,low_writer) == 0)
3642 goto yielditis;
6d2010ae
A		 3643
3644 numneeded = ksyn_queue_count_tolowest(kq, low_writer);
3645 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
b0d623f7
A		 3646 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, low_writer) != 0)) {
3647 uth = current_uthread();
6d2010ae
A		 3648 kwe = &uth->uu_kwe;
3649 /* add one more */
3650 updatebits += PTHRW_INC;
3651 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3652 block = 0;
3653 }
b0d623f7 3654
6d2010ae
A		 3655 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3656
b0d623f7
A		 3657 /* there will be readers to wakeup , no need to check for woken */
3658 failed = ksyn_wakeupreaders(kwq, low_writer, 0, 0, updatebits, NULL);
6d2010ae
A		 3659#if _PSYNCH_TRACE_
3660 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3661#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 3662 if (failed != 0) {
3663 kwq->kw_pre_intrcount = failed; /* actually a count */
3664 kwq->kw_pre_intrseq = low_writer;
3665 kwq->kw_pre_intrretbits = updatebits;
3666 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3667 }
3668 error = 0;
3669 } else {
3670 /* wakeup all readers */
6d2010ae
A		 3671 numneeded = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3672 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
b0d623f7
A		 3673 if ((prepost != 0) && ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
3674 uth = current_uthread();
6d2010ae
A		 3675 kwe = &uth->uu_kwe;
3676 updatebits += PTHRW_INC;
3677 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3678 block = 0;
3679 }
6d2010ae 3680 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
b0d623f7 3681 failed = ksyn_wakeupreaders(kwq, low_writer, 0, 1, updatebits, &woken);
6d2010ae
A		 3682#if _PSYNCH_TRACE_
3683 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3684#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 3685 if (failed != 0) {
3686 kwq->kw_pre_intrcount = failed; /* actually a count */
3687 kwq->kw_pre_intrseq = kwq->kw_highseq;
3688 kwq->kw_pre_intrretbits = updatebits;
3689 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3690 }
b0d623f7
A		 3691 error = 0;
3692 }
3693 } else {
3694yielditis:
3695 /* no reads, so granting yeilding writes */
6d2010ae
A		 3696 updatebits |= PTHRW_INC;
3697 updatebits |= PTH_RWL_KBIT| PTH_RWL_EBIT;
b0d623f7
A		 3698
3699 if (((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) && (low_writer == premgen)) {
3700 /* preposting yielding write thread is being granted exclusive lock */
3701
3702 block = 0;
3703
3704 if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0)
6d2010ae 3705 updatebits |= PTH_RWL_WBIT;
b0d623f7 3706 else if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
6d2010ae 3707 updatebits |= PTH_RWL_YBIT;
b0d623f7
A		 3708
3709 th = preth;
3710 uth = get_bsdthread_info(th);
6d2010ae
A		 3711 kwe = &uth->uu_kwe;
3712 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3713 } else {
3714 /* we are granting yield writelock to some other thread */
6d2010ae 3715 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwq);
b0d623f7
A		 3716
3717 if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0)
6d2010ae 3718 updatebits |= PTH_RWL_WBIT;
b0d623f7
A		 3719 /* if there are ywriters present or the preposting ywrite thread then W bit is to be set */
3720 else if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) )
6d2010ae 3721 updatebits |= PTH_RWL_YBIT;
b0d623f7 3722
6d2010ae
A		 3723 kwe->kwe_psynchretval = updatebits;
3724 kwe->kwe_kwqqueue = NULL;
b0d623f7 3725
6d2010ae
A		 3726 kret = ksyn_wakeup_thread(kwq, kwe);
3727#if _PSYNCH_TRACE_
3728 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
3729#endif /* _PSYNCH_TRACE_ */
3730#if __TESTPANICS__
b0d623f7
A		 3731 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3732 panic("kwq_handle_unlock : panic waking up readers\n");
6d2010ae 3733#endif /* __TESTPANICS__ */
b0d623f7
A		 3734 if (kret == KERN_NOT_WAITING) {
3735 kwq->kw_pre_intrcount = 1; /* actually a count */
3736 kwq->kw_pre_intrseq = low_ywriter;
3737 kwq->kw_pre_intrretbits = updatebits;
3738 kwq->kw_pre_intrtype = PTH_RW_TYPE_YWRITE;
3739 }
3740 error = 0;
3741 }
6d2010ae 3742 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
b0d623f7
A		 3743 }
3744 }
3745 break;
3746
3747 default:
3748 panic("rwunlock: invalid type for lock grants");
3749
3750 };
3751
b0d623f7
A		 3752
3753out:
6d2010ae
A		 3754 if (updatep != NULL)
3755 *updatep = updatebits;
b0d623f7
A		 3756 if (blockp != NULL)
3757 *blockp = block;
3758#if _PSYNCH_TRACE_
6d2010ae 3759 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0, updatebits, block, 0);
b0d623f7
A		 3760#endif /* _PSYNCH_TRACE_ */
3761 return(error);
3762}
3763
6d2010ae
A		 3764int
3765kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, __unused uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, __unused int flags , int * blockp)
3766{
3767 uint32_t highword = kwq->kw_nextseqword & PTHRW_COUNT_MASK;
3768 uint32_t lowword = kwq->kw_lastseqword & PTHRW_COUNT_MASK;
3769 uint32_t val=0;
3770 int withinseq;
3771
3772
3773 /* overlap is set, so no need to check for valid state for overlap */
3774
3775 withinseq = ((is_seqlower_eq(rw_wc, highword) != 0) || (is_seqhigher_eq(lowword, rw_wc) != 0));
3776
3777 if (withinseq != 0) {
3778 if ((kwq->kw_nextseqword & PTH_RWL_LBIT) == 0) {
3779 /* if no writers ahead, overlap granted */
3780 if ((lgenval & PTH_RWL_WBIT) == 0) {
3781 goto grantoverlap;
3782 }
3783 } else {
3784 /* Lbit is set, and writers ahead does not count */
3785 goto grantoverlap;
3786 }
3787 }
3788
3789 *blockp = 1;
3790 return(0);
3791
3792grantoverlap:
3793 /* increase the next expected seq by one */
3794 kwq->kw_nextseqword += PTHRW_INC;
3795 /* set count by one & bits from the nextseq and add M bit */
3796 val = PTHRW_INC;
3797 val |= ((kwq->kw_nextseqword & PTHRW_BIT_MASK) | PTH_RWL_MBIT);
3798 *updatebitsp = val;
3799 *blockp = 0;
3800 return(0);
3801}
b0d623f7 3802
6d2010ae 3803#if NOTYET
b0d623f7
A		 3804/* handle downgrade actions */
3805int
3806kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, __unused int flags, __unused uint32_t premgen, __unused int * blockp)
3807{
3808 uint32_t updatebits, lowriter = 0;
3809 int longreadset, allreaders, count;
3810
3811 /* can handle downgrade now */
3812 updatebits = mgen;
3813
3814 longreadset = 0;
3815 allreaders = 0;
3816 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count > 0) {
3817 lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
3818 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
3819 if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
3820 longreadset = 1;
3821 }
3822 } else {
3823 allreaders = 1;
3824 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count > 0) {
3825 lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
3826 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
3827 if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
3828 longreadset = 1;
3829 }
3830 }
3831 }
3832
3833 count = ksyn_wakeupreaders(kwq, lowriter, longreadset, allreaders, updatebits, NULL);
3834 if (count != 0) {
3835 kwq->kw_pre_limrd = count;
3836 kwq->kw_pre_limrdseq = lowriter;
3837 kwq->kw_pre_limrdbits = lowriter;
3838 /* need to handle prepost */
3839 }
3840 return(0);
3841}
6d2010ae
A		 3842
3843#endif /* NOTYET */
3844
b0d623f7
A		 3845/************* Indiv queue support routines ************************/
3846void
3847ksyn_queue_init(ksyn_queue_t kq)
3848{
6d2010ae 3849 TAILQ_INIT(&kq->ksynq_kwelist);
b0d623f7
A		 3850 kq->ksynq_count = 0;
3851 kq->ksynq_firstnum = 0;
3852 kq->ksynq_lastnum = 0;
3853}
3854
b0d623f7 3855int
6d2010ae 3856ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int fit)
b0d623f7
A		 3857{
3858 uint32_t lockseq = mgen & PTHRW_COUNT_MASK;
6d2010ae
A		 3859 ksyn_waitq_element_t q_kwe, r_kwe;
3860 int res = 0;
3861 uthread_t nuth = NULL;
b0d623f7
A		 3862
3863 if (kq->ksynq_count == 0) {
6d2010ae 3864 TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3865 kq->ksynq_firstnum = lockseq;
3866 kq->ksynq_lastnum = lockseq;
3867 goto out;
3868 }
3869
3870 if (fit == FIRSTFIT) {
6d2010ae 3871 /* TBD: if retry bit is set for mutex, add it to the head */
b0d623f7 3872 /* firstfit, arriving order */
6d2010ae 3873 TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3874 if (is_seqlower (lockseq, kq->ksynq_firstnum) != 0)
3875 kq->ksynq_firstnum = lockseq;
3876 if (is_seqhigher (lockseq, kq->ksynq_lastnum) != 0)
3877 kq->ksynq_lastnum = lockseq;
3878 goto out;
3879 }
3880
6d2010ae
A		 3881 if ((lockseq == kq->ksynq_firstnum) || (lockseq == kq->ksynq_lastnum)) {
3882 /* During prepost when a thread is getting cancelled, we could have two with same seq */
3883 if (kwe->kwe_flags == KWE_THREAD_PREPOST) {
3884 q_kwe = ksyn_queue_find_seq(kwq, kq, lockseq, 0);
3885 if ((q_kwe != NULL) && ((nuth = (uthread_t)q_kwe->kwe_uth) != NULL) &&
3886 ((nuth->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL)) {
3887 TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
3888 goto out;
3889
3890 } else {
3891 __FAILEDUSERTEST__("ksyn_queue_insert: two threads with same lockseq ");
3892 res = EBUSY;
3893 goto out1;
3894 }
3895 } else {
3896 __FAILEDUSERTEST__("ksyn_queue_insert: two threads with same lockseq ");
3897 res = EBUSY;
3898 goto out1;
3899 }
3900 }
b0d623f7
A		 3901
3902 /* check for next seq one */
3903 if (is_seqlower(kq->ksynq_lastnum, lockseq) != 0) {
6d2010ae 3904 TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3905 kq->ksynq_lastnum = lockseq;
3906 goto out;
3907 }
3908
3909 if (is_seqlower(lockseq, kq->ksynq_firstnum) != 0) {
6d2010ae 3910 TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3911 kq->ksynq_firstnum = lockseq;
3912 goto out;
3913 }
3914
3915 /* goto slow insert mode */
6d2010ae
A		 3916 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
3917 if (is_seqhigher(q_kwe->kwe_lockseq, lockseq) != 0) {
3918 TAILQ_INSERT_BEFORE(q_kwe, kwe, kwe_list);
b0d623f7
A		 3919 goto out;
3920 }
3921 }
3922
6d2010ae 3923#if __TESTPANICS__
b0d623f7 3924 panic("failed to insert \n");
6d2010ae
A		 3925#endif /* __TESTPANICS__ */
3926
b0d623f7 3927out:
6d2010ae
A		 3928 if (uth != NULL)
3929 kwe->kwe_uth = uth;
b0d623f7
A		 3930 kq->ksynq_count++;
3931 kwq->kw_inqueue++;
3932 update_low_high(kwq, lockseq);
6d2010ae
A		 3933out1:
3934 return(res);
b0d623f7
A		 3935}
3936
6d2010ae 3937ksyn_waitq_element_t
b0d623f7
A		 3938ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq)
3939{
6d2010ae
A		 3940 ksyn_waitq_element_t kwe = NULL;
3941 ksyn_waitq_element_t q_kwe;
b0d623f7
A		 3942 uint32_t curseq;
3943
3944 if (kq->ksynq_count != 0) {
6d2010ae
A		 3945 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3946 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
3947 curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
b0d623f7
A		 3948 kq->ksynq_count--;
3949 kwq->kw_inqueue--;
3950
3951 if(kq->ksynq_count != 0) {
6d2010ae
A		 3952 q_kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3953 kq->ksynq_firstnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
b0d623f7
A		 3954 } else {
3955 kq->ksynq_firstnum = 0;
3956 kq->ksynq_lastnum = 0;
3957
3958 }
3959 if (kwq->kw_inqueue == 0) {
3960 kwq->kw_lowseq = 0;
3961 kwq->kw_highseq = 0;
3962 } else {
3963 if (kwq->kw_lowseq == curseq)
3964 kwq->kw_lowseq = find_nextlowseq(kwq);
3965 if (kwq->kw_highseq == curseq)
3966 kwq->kw_highseq = find_nexthighseq(kwq);
3967 }
3968 }
6d2010ae 3969 return(kwe);
b0d623f7
A		 3970}
3971
3972void
6d2010ae 3973ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe)
b0d623f7 3974{
6d2010ae 3975 ksyn_waitq_element_t q_kwe;
b0d623f7
A		 3976 uint32_t curseq;
3977
3978 if (kq->ksynq_count > 0) {
6d2010ae 3979 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3980 kq->ksynq_count--;
3981 if(kq->ksynq_count != 0) {
6d2010ae
A		 3982 q_kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3983 kq->ksynq_firstnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
3984 q_kwe = TAILQ_LAST(&kq->ksynq_kwelist, ksynq_kwelist_head);
3985 kq->ksynq_lastnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
b0d623f7
A		 3986 } else {
3987 kq->ksynq_firstnum = 0;
3988 kq->ksynq_lastnum = 0;
3989
3990 }
3991 kwq->kw_inqueue--;
6d2010ae 3992 curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
b0d623f7
A		 3993 if (kwq->kw_inqueue == 0) {
3994 kwq->kw_lowseq = 0;
3995 kwq->kw_highseq = 0;
3996 } else {
3997 if (kwq->kw_lowseq == curseq)
3998 kwq->kw_lowseq = find_nextlowseq(kwq);
3999 if (kwq->kw_highseq == curseq)
4000 kwq->kw_highseq = find_nexthighseq(kwq);
4001 }
4002 }
4003}
4004
6d2010ae
A		 4005/* find the thread and removes from the queue */
4006ksyn_waitq_element_t
4007ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove)
4008{
4009 ksyn_waitq_element_t q_kwe, r_kwe;
4010
4011 /* TBD: bail out if higher seq is seen */
4012 /* case where wrap in the tail of the queue exists */
4013 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
4014 if ((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK) == seq) {
4015 if (remove != 0)
4016 ksyn_queue_removeitem(kwq, kq, q_kwe);
4017 return(q_kwe);
4018 }
4019 }
4020 return(NULL);
4021}
4022
4023
4024/* find the thread at the target sequence (or a broadcast/prepost at or above) */
4025ksyn_waitq_element_t
4026ksyn_queue_find_cvpreposeq(ksyn_queue_t kq, uint32_t cgen)
4027{
4028 ksyn_waitq_element_t q_kwe, r_kwe;
4029 uint32_t lgen = (cgen & PTHRW_COUNT_MASK);
4030
4031 /* case where wrap in the tail of the queue exists */
4032 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
4033
4034 /* skip the lower entries */
4035 if (is_seqlower((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), cgen) != 0)
4036 continue;
4037
4038 switch (q_kwe->kwe_flags) {
4039
4040 case KWE_THREAD_INWAIT:
4041 if ((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK) != lgen)
4042 break;
4043 /* fall thru */
4044
4045 case KWE_THREAD_BROADCAST:
4046 case KWE_THREAD_PREPOST:
4047 return (q_kwe);
4048 }
4049 }
4050 return(NULL);
4051}
4052
4053/* look for a thread at lockseq, a */
4054ksyn_waitq_element_t
4055ksyn_queue_find_signalseq(__unused ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t uptoseq, uint32_t signalseq)
4056{
4057 ksyn_waitq_element_t q_kwe, r_kwe, t_kwe = NULL;
4058
4059 /* case where wrap in the tail of the queue exists */
4060 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
4061
4062 switch (q_kwe->kwe_flags) {
4063
4064 case KWE_THREAD_PREPOST:
4065 if (is_seqhigher((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
4066 return t_kwe;
4067 /* fall thru */
4068
4069 case KWE_THREAD_BROADCAST:
4070 /* match any prepost at our same uptoseq or any broadcast above */
4071 if (is_seqlower((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
4072 continue;
4073 return q_kwe;
4074
4075 case KWE_THREAD_INWAIT:
4076 /*
4077 * Match any (non-cancelled) thread at or below our upto sequence -
4078 * but prefer an exact match to our signal sequence (if present) to
4079 * keep exact matches happening.
4080 */
4081 if (is_seqhigher((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
4082 return t_kwe;
4083
4084 if (q_kwe->kwe_kwqqueue == kwq) {
4085 uthread_t ut = q_kwe->kwe_uth;
4086 if ((ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) != UT_CANCEL) {
4087 /* if equal or higher than our signal sequence, return this one */
4088 if (is_seqhigher_eq((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq))
4089 return q_kwe;
4090
4091 /* otherwise, just remember this eligible thread and move on */
4092 if (t_kwe == NULL)
4093 t_kwe = q_kwe;
4094 }
4095 }
4096 break;
4097
4098 default:
4099 panic("ksyn_queue_find_signalseq(): unknow wait queue element type (%d)\n", q_kwe->kwe_flags);
4100 break;
4101 }
4102 }
4103 return t_kwe;
4104}
4105
4106
4107int
4108ksyn_queue_move_tofree(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t kfreeq, int all, int release)
4109{
4110 ksyn_waitq_element_t kwe;
4111 int count = 0;
4112 uint32_t tseq = upto & PTHRW_COUNT_MASK;
4113#if _PSYNCH_TRACE_
4114 uthread_t ut;
4115#endif /* _PSYNCH_TRACE_ */
4116
4117 ksyn_queue_init(kfreeq);
4118
4119 /* free all the entries, must be only fakes.. */
4120 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
4121 while (kwe != NULL) {
4122 if ((all == 0) && (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), tseq) != 0))
4123 break;
4124 if (kwe->kwe_flags == KWE_THREAD_INWAIT) {
4125 /*
4126 * This scenario is typically noticed when the cvar is
4127 * reinited and the new waiters are waiting. We can
4128 * return them as spurious wait so the cvar state gets
4129 * reset correctly.
4130 */
4131#if _PSYNCH_TRACE_
4132 ut = (uthread_t)kwe->kwe_uth;
4133#endif /* _PSYNCH_TRACE_ */
4134
4135 /* skip canceled ones */
4136 /* wake the rest */
4137 ksyn_queue_removeitem(ckwq, kq, kwe);
4138 /* set M bit to indicate to waking CV to retun Inc val */
4139 kwe->kwe_psynchretval = PTHRW_INC | (PTH_RWS_CV_MBIT | PTH_RWL_MTX_WAIT);
4140 kwe->kwe_kwqqueue = NULL;
4141#if _PSYNCH_TRACE_
4142 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4143#endif /* _PSYNCH_TRACE_ */
4144 (void)ksyn_wakeup_thread(ckwq, kwe);
4145 } else {
4146 ksyn_queue_removeitem(ckwq, kq, kwe);
4147 TAILQ_INSERT_TAIL(&kfreeq->ksynq_kwelist, kwe, kwe_list);
4148 ckwq->kw_fakecount--;
4149 count++;
4150 }
4151 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
4152 }
4153
4154 if ((release != 0) && (count != 0)) {
4155 kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
4156 while (kwe != NULL) {
4157 TAILQ_REMOVE(&kfreeq->ksynq_kwelist, kwe, kwe_list);
4158 zfree(kwe_zone, kwe);
4159 kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
4160 }
4161 }
4162
4163 return(count);
4164}
4165
4166/*************************************************************************/
b0d623f7
A		 4167
4168void
4169update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq)
4170{
4171 if (kwq->kw_inqueue == 1) {
4172 kwq->kw_lowseq = lockseq;
4173 kwq->kw_highseq = lockseq;
4174 } else {
4175 if (is_seqlower(lockseq, kwq->kw_lowseq) != 0)
4176 kwq->kw_lowseq = lockseq;
4177 if (is_seqhigher(lockseq, kwq->kw_highseq) != 0)
4178 kwq->kw_highseq = lockseq;
4179 }
4180}
4181
4182uint32_t
4183find_nextlowseq(ksyn_wait_queue_t kwq)
4184{
316670eb 4185 uint32_t numbers[KSYN_QUEUE_MAX];
b0d623f7
A		 4186 int count = 0, i;
4187 uint32_t lowest;
4188
4189 for(i = 0; i< KSYN_QUEUE_MAX; i++) {
4190 if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
4191 numbers[count]= kwq->kw_ksynqueues[i].ksynq_firstnum;
4192 count++;
4193 }
4194 }
4195
4196 if (count == 0)
4197 return(0);
4198 lowest = numbers[0];
4199 if (count > 1) {
4200 for (i = 1; i< count; i++) {
4201 if(is_seqlower(numbers[i] , lowest) != 0)
4202 lowest = numbers[count];
4203
4204 }
4205 }
4206 return(lowest);
4207}
4208
4209uint32_t
4210find_nexthighseq(ksyn_wait_queue_t kwq)
4211{
316670eb 4212 uint32_t numbers[KSYN_QUEUE_MAX];
b0d623f7
A		 4213 int count = 0, i;
4214 uint32_t highest;
4215
4216 for(i = 0; i< KSYN_QUEUE_MAX; i++) {
4217 if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
4218 numbers[count]= kwq->kw_ksynqueues[i].ksynq_lastnum;
4219 count++;
4220 }
4221 }
4222
4223
4224
4225 if (count == 0)
4226 return(0);
4227 highest = numbers[0];
4228 if (count > 1) {
4229 for (i = 1; i< count; i++) {
4230 if(is_seqhigher(numbers[i], highest) != 0)
4231 highest = numbers[i];
4232
4233 }
4234 }
4235 return(highest);
4236}
4237
6d2010ae
A		 4238int
4239is_seqlower(uint32_t x, uint32_t y)
4240{
4241 if (x < y) {
4242 if ((y-x) < (PTHRW_MAX_READERS/2))
4243 return(1);
4244 } else {
4245 if ((x-y) > (PTHRW_MAX_READERS/2))
4246 return(1);
4247 }
4248 return(0);
4249}
4250
4251int
4252is_seqlower_eq(uint32_t x, uint32_t y)
4253{
4254 if (x==y)
4255 return(1);
4256 else
4257 return(is_seqlower(x,y));
4258}
4259
4260int
4261is_seqhigher(uint32_t x, uint32_t y)
4262{
4263 if (x > y) {
4264 if ((x-y) < (PTHRW_MAX_READERS/2))
4265 return(1);
4266 } else {
4267 if ((y-x) > (PTHRW_MAX_READERS/2))
4268 return(1);
4269 }
4270 return(0);
4271}
4272
4273int
4274is_seqhigher_eq(uint32_t x, uint32_t y)
4275{
4276 if (x==y)
4277 return(1);
4278 else
4279 return(is_seqhigher(x,y));
4280}
4281
4282
b0d623f7
A		 4283int
4284find_diff(uint32_t upto, uint32_t lowest)
4285{
4286 uint32_t diff;
4287
4288 if (upto == lowest)
4289 return(0);
6d2010ae 4290#if 0
b0d623f7 4291 diff = diff_genseq(upto, lowest);
6d2010ae
A		 4292#else
4293 if (is_seqlower(upto, lowest) != 0)
4294 diff = diff_genseq(lowest, upto);
4295 else
4296 diff = diff_genseq(upto, lowest);
4297#endif
b0d623f7
A		 4298 diff = (diff >> PTHRW_COUNT_SHIFT);
4299 return(diff);
4300}
4301
4302
4303int
4304find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp)
4305{
4306 int i;
4307 uint32_t count = 0;
4308
4309
4310#if _PSYNCH_TRACE_
6d2010ae 4311 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_START, 0, 0, upto, nwaiters, 0);
b0d623f7
A		 4312#endif /* _PSYNCH_TRACE_ */
4313
4314 for (i= 0; i< KSYN_QUEUE_MAX; i++) {
4315 count += ksyn_queue_count_tolowest(&kwq->kw_ksynqueues[i], upto);
4316#if _PSYNCH_TRACE_
6d2010ae 4317 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_NONE, 0, 1, i, count, 0);
b0d623f7
A		 4318#endif /* _PSYNCH_TRACE_ */
4319 if (count >= nwaiters) {
4320 break;
4321 }
4322 }
4323
4324 if (countp != NULL) {
4325 *countp = count;
4326 }
4327#if _PSYNCH_TRACE_
6d2010ae 4328 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_END, 0, 0, count, nwaiters, 0);
b0d623f7 4329#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 4330 if (count == 0)
4331 return(0);
4332 else if (count >= nwaiters)
b0d623f7
A		 4333 return(1);
4334 else
4335 return(0);
4336}
4337
4338
4339uint32_t
4340ksyn_queue_count_tolowest(ksyn_queue_t kq, uint32_t upto)
4341{
4342 uint32_t i = 0;
6d2010ae 4343 ksyn_waitq_element_t kwe, newkwe;
b0d623f7
A		 4344 uint32_t curval;
4345
4346 /* if nothing or the first num is greater than upto, return none */
4347 if ((kq->ksynq_count == 0) || (is_seqhigher(kq->ksynq_firstnum, upto) != 0))
4348 return(0);
4349 if (upto == kq->ksynq_firstnum)
4350 return(1);
4351
6d2010ae
A		 4352 TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
4353 curval = (kwe->kwe_lockseq & PTHRW_COUNT_MASK);
b0d623f7
A		 4354 if (upto == curval) {
4355 i++;
4356 break;
4357 } else if (is_seqhigher(curval, upto) != 0) {
4358 break;
4359 } else {
4360 /* seq is lower */
4361 i++;
4362 }
4363 }
4364 return(i);
4365}
4366
6d2010ae
A		 4367
4368/* handles the cond broadcast of cvar and returns number of woken threads and bits for syscall return */
4369void
4370ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t * updatep)
4371{
4372 kern_return_t kret;
4373 ksyn_queue_t kq;
4374 ksyn_waitq_element_t kwe, newkwe;
4375 uint32_t updatebits = 0;
4376 struct ksyn_queue kfreeq;
4377 uthread_t ut;
4378
4379#if _PSYNCH_TRACE_
4380 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_START, 0xcbcbcbc2, upto, 0, 0, 0);
4381#endif /* _PSYNCH_TRACE_ */
4382
4383 ksyn_queue_init(&kfreeq);
4384 kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
4385
4386 retry:
4387 TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
4388
4389 if (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) /* outside our range */
4390 break;
4391
4392 /* now handle the one we found (inside the range) */
4393 switch (kwe->kwe_flags) {
4394
4395 case KWE_THREAD_INWAIT:
4396 ut = (uthread_t)kwe->kwe_uth;
4397
4398 /* skip canceled ones */
4399 if (kwe->kwe_kwqqueue != ckwq ||
4400 (ut->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL)
4401 break;
4402
4403 /* wake the rest */
4404 ksyn_queue_removeitem(ckwq, kq, kwe);
4405 kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
4406 kwe->kwe_kwqqueue = NULL;
4407#if _PSYNCH_TRACE_
4408 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4409#endif /* _PSYNCH_TRACE_ */
4410 kret = ksyn_wakeup_thread(ckwq, kwe);
4411#if __TESTPANICS__
4412 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
4413 panic("ksyn_wakeupreaders: panic waking up readers\n");
4414#endif /* __TESTPANICS__ */
4415 updatebits += PTHRW_INC;
4416 break;
4417
4418 case KWE_THREAD_BROADCAST:
4419 case KWE_THREAD_PREPOST:
4420 ksyn_queue_removeitem(ckwq, kq, kwe);
4421 TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, kwe, kwe_list);
4422 ckwq->kw_fakecount--;
4423 break;
4424
4425 default:
4426 panic("unknown kweflags\n");
4427 break;
b0d623f7 4428 }
6d2010ae
A		 4429 }
4430
4431 /* Need to enter a broadcast in the queue (if not already at L == S) */
4432
4433 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
4434
4435 newkwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4436 if (newkwe == NULL) {
4437 ksyn_wqunlock(ckwq);
4438 newkwe = (ksyn_waitq_element_t)zalloc(kwe_zone);
4439 TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, newkwe, kwe_list);
4440 ksyn_wqlock(ckwq);
4441 goto retry;
4442 }
4443
4444 TAILQ_REMOVE(&kfreeq.ksynq_kwelist, newkwe, kwe_list);
4445 bzero(newkwe, sizeof(struct ksyn_waitq_element));
4446 newkwe->kwe_kwqqueue = ckwq;
4447 newkwe->kwe_flags = KWE_THREAD_BROADCAST;
4448 newkwe->kwe_lockseq = upto;
4449 newkwe->kwe_count = 0;
4450 newkwe->kwe_uth = NULL;
4451 newkwe->kwe_psynchretval = 0;
4452
4453#if _PSYNCH_TRACE_
4454 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, upto, 0, 0);
4455#endif /* _PSYNCH_TRACE_ */
4456
4457 (void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], upto, NULL, newkwe, SEQFIT);
4458 ckwq->kw_fakecount++;
4459 }
4460
4461 /* free up any remaining things stumbled across above */
4462 kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4463 while (kwe != NULL) {
4464 TAILQ_REMOVE(&kfreeq.ksynq_kwelist, kwe, kwe_list);
4465 zfree(kwe_zone, kwe);
4466 kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4467 }
4468
4469 if (updatep != NULL)
4470 *updatep = updatebits;
4471
4472#if _PSYNCH_TRACE_
4473 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_END, 0xeeeeeeed, updatebits, 0, 0, 0);
4474#endif /* _PSYNCH_TRACE_ */
2d21ac55
A		 4475}
4476
6d2010ae
A		 4477void
4478ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release)
4479{
4480 uint32_t updatebits = 0;
4481
4482 if (updatep != NULL)
4483 updatebits = *updatep;
4484 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
4485 updatebits |= PTH_RWS_CV_CBIT;
4486 if (ckwq->kw_inqueue != 0) {
4487 /* FREE THE QUEUE */
4488 ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], ckwq->kw_lword, kfreeq, 0, release);
4489#if __TESTPANICS__
4490 if (ckwq->kw_inqueue != 0)
4491 panic("ksyn_cvupdate_fixup: L == S, but entries in queue beyond S");
4492#endif /* __TESTPANICS__ */
4493 }
4494 ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
4495 ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
4496 } else if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
4497 /* only fake entries are present in the queue */
4498 updatebits |= PTH_RWS_CV_PBIT;
4499 }
4500 if (updatep != NULL)
4501 *updatep = updatebits;
4502}
4503
4504void
4505psynch_zoneinit(void)
4506{
4507 kwq_zone = (zone_t)zinit(sizeof(struct ksyn_wait_queue), 8192 * sizeof(struct ksyn_wait_queue), 4096, "ksyn_waitqueue zone");
4508 kwe_zone = (zone_t)zinit(sizeof(struct ksyn_waitq_element), 8192 * sizeof(struct ksyn_waitq_element), 4096, "ksyn_waitq_element zone");
4509}
b0d623f7 4510#endif /* PSYNCH */
mirror of XNU